birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Integrating github/staging through commit ab87ed9

Browse Source
This commit is contained in:
alexpete
2021-04-09 11:27:37 -07:00
parent ae62a97894
commit 1044dc3da1
1582 changed files with 29374 additions and 519051 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tools/LyTestTools/ly_test_tools/_internal/managers/abstract_resource_locator.py
+40 -30
View File
@@ -12,33 +12,31 @@ Utility class to resolve Lumberyard directory paths & file mappings.
"""
import os
import pathlib
import warnings
from abc import ABCMeta, abstractmethod
from ly_test_tools.lumberyard.asset_processor import ASSET_PROCESSOR_PLATFORM_MAP
from ly_test_tools.environment.file_system import find_ancestor_file
def _find_engine_root(initial_path):
# type: (str) -> tuple
# type: (str) -> str
"""
Attempts to find the root engine directory to set the values for engine_root and dev_path.
Attempts to find the root engine directory to set the values for engine_root
Assumes it exists at or above the provided "initial_path", and not in a separate directory tree
ex. for the directory "C:\\root_dir\\dev\\":
"C:\\root_dir\\" is the engine_root
"C:\\root_dir\\dev\\" is the dev_path
ex. for the directory "C:\\root_dir\\":
If it contains an "engine.json" file then it is the engine root "C:\\root_dir\\"
:param initial_path: The initial directory to search for root from
:return: a tuple of 2 strings representing the engine_root and dev_path
:return: a string representing the engine_root
"""
root_file = "engineroot.txt"
root_file = "engine.json"
current_dir = initial_path
# Look upward a handful of levels, before assuming a missing root directory
# Assumes folder structure similar to: engine_root/dev/Tools/.../ly_test_tools/builtin
for _ in range(15):
if os.path.exists(os.path.join(current_dir, root_file)):
# The parent of the directory containing the engineroot.txt is the root directory
engine_root = os.path.abspath(os.path.join(current_dir, os.path.pardir))
dev_path = current_dir
return engine_root, dev_path
engine_root = current_dir
return engine_root
# Using an explicit else to avoid aberrant behavior from following filesystem links
else:
current_dir = os.path.abspath(os.path.join(current_dir, os.path.pardir))
@@ -46,6 +44,19 @@ def _find_engine_root(initial_path):
raise OSError(f"Unable to find engine root directory. Verify root file '{root_file}' exists")
def _find_project_json(engine_root, project):
# type (None) -> str
"""
Find the project.json file for this project.
:return: Full path to the project.json file
"""
project_json = find_ancestor_file('project.json')
if not project_json:
project_json = os.path.join(engine_root, project, 'project.json')
return project_json
class AbstractResourceLocator(object):
__metaclass__ = ABCMeta
@@ -53,13 +64,14 @@ class AbstractResourceLocator(object):
# type: (str, str) -> AbstractResourceLocator
"""
:param build_directory: The path to the build directory (i.e. <engine_root>/dev/windows_vs2017/bin/profile)
:param project: The game project (i.e. AutomatedTesting or StarterGame)
:param project: The game project (i.e. AutomatedTesting)
"""
engine_root, dev_path = _find_engine_root(os.path.abspath(__file__))
initial_search_path = str(pathlib.Path(__file__).resolve()) # __file__ is lowercase, this restores casing
engine_root = _find_engine_root(initial_search_path)
self._build_directory = build_directory
self._engine_root = engine_root
self._dev_path = dev_path
self._project_json = _find_project_json(engine_root, project)
self._project = project
self._cache_override = None
self._db_override = None
@@ -74,14 +86,6 @@ class AbstractResourceLocator(object):
"""
return self._engine_root
def dev(self):
"""
Returns the path to the dev directory
ex. <engine_root>\\dev
:return: dev_path
"""
return self._dev_path
def third_party(self):
"""
Return path to 3rdParty directory
@@ -118,11 +122,18 @@ class AbstractResourceLocator(object):
def project(self):
"""
Return path to the project directory
ex. engine_root/dev/AutomatedTesting
:return: path to <engine_root>/dev/Project
ex. engine_root/dev/AutomatedTesting for included projects or some_dir/project for external projects.
:return: path to the project directory
"""
return os.path.join(self.dev(), self._project)
return os.path.dirname(self._project_json)
def project_settings(self):
"""
Return full path to the project.json file for this project.
:return: Full path to the project.json file for this project.
"""
return self._project_json
def asset_processor(self):
"""
Return path for the AssetProcessor executable.
@@ -210,7 +221,6 @@ class AbstractResourceLocator(object):
"""
return os.path.join(self.ap_log_dir(), 'JobLogs')
def ap_batch_log(self):
"""
Return path to AssetProcessorBatch's log file using the project bin dir
@@ -241,10 +251,10 @@ class AbstractResourceLocator(object):
return os.path.join(self.build_directory(), 'CrySCompileServer')
def bootstrap_config_file(self):
return os.path.join(self.dev(), 'bootstrap.cfg')
return os.path.join(self.engine_root(), 'bootstrap.cfg')
def asset_processor_config_file(self):
return os.path.join(self.dev(), 'AssetProcessorPlatformConfig.setreg')
return os.path.join(self.engine_root(), 'AssetProcessorPlatformConfig.setreg')
def autoexec_file(self):
return os.path.join(
@@ -257,7 +267,7 @@ class AbstractResourceLocator(object):
Return the path to the TestResults directory containing test artifacts.
:return: path to TestResults dir
"""
return os.path.join(self.dev(), "TestResults")
return os.path.join(self.engine_root(), "TestResults")
def devices_file(self):
"""
Tools/LyTestTools/ly_test_tools/_internal/managers/platforms/mac.py
+1 -1
View File
@@ -36,7 +36,7 @@ class _MacResourceLocator(AbstractResourceLocator):
ex. engine_root/dev/system_osx_osx_gl.cfg
:return: path to the platform config file
"""
return os.path.join(self.dev(), CONFIG_FILE)
return os.path.join(self.engine_root(), CONFIG_FILE)
def platform_cache(self):
"""
Tools/LyTestTools/ly_test_tools/_internal/managers/platforms/windows.py
+1 -1
View File
@@ -42,7 +42,7 @@ class _WindowsResourceLocator(AbstractResourceLocator):
ex. engine_root/dev/system_osx_osx_gl.cfg
:return: path to the platform config file
"""
return os.path.join(self.dev(), CONFIG_FILE)
return os.path.join(self.engine_root(), CONFIG_FILE)
def platform_cache(self):
"""
Tools/LyTestTools/ly_test_tools/builtin/helpers.py
-27
View File
@@ -62,33 +62,6 @@ def create_builtin_workspace(
return instance
def setup_bootstrap_project(workspace, project):
"""
Sets up the bootstrap.cfg file to be used for the given project
:param workspace: workspace to use
:param project: Lumberyard project to set as target
:return: None
"""
bootstrap_cfg = os.path.join(workspace.paths.dev(), "bootstrap.cfg")
os.chmod(bootstrap_cfg, stat.S_IWRITE)
lines = None
with open(bootstrap_cfg) as f:
lines = f.readlines()
found_gamefolder = False
for i, line in enumerate(lines):
if line.lstrip().startswith("project_path"):
lines[i] = f"project_path={project}\n"
found_gamefolder = True
break
assert found_gamefolder, "'project_path' not found in bootstrap.cfg"
with open(bootstrap_cfg, "w") as f:
f.writelines(lines)
def setup_builtin_workspace(workspace, test_name, artifact_folder_count):
# type: (internal_workspace.AbstractWorkspaceManager, str, int) -> internal_workspace.AbstractWorkspaceManager
"""
Tools/LyTestTools/ly_test_tools/environment/file_system.py
+31
View File
@@ -420,3 +420,34 @@ def reduce_file_name_length(file_name, max_length):
file_name = file_name[:-reduce_amount]
return file_name
def find_ancestor_file(target_file_name, start_path=os.getcwd()):
"""
Find a file with the given name in the ancestor directories by walking up the starting path until the file is found.
:param target_file_name: Name of the file to find.
:param start_path: Optional path to start looking for the file.
:return: Path to the file or None if not found.
"""
current_path = os.path.normpath(start_path)
candidate_path = os.path.join(current_path, target_file_name)
# Limit the number of directories to traverse, to avoid infinite loop in path cycles
for _ in range(15):
if not os.path.exists(candidate_path):
parent_path = os.path.dirname(current_path)
if parent_path == current_path:
# Only true when we are at the directory root, can't keep searching
break
candidate_path = os.path.join(parent_path, target_file_name)
current_path = parent_path
else:
# Found the file we wanted
break
if not os.path.exists(candidate_path):
logger.warning(f'The candidate path {candidate_path} does not exist.')
return None
return candidate_path
Tools/LyTestTools/ly_test_tools/launchers/platforms/android/launcher.py
+1 -1
View File
@@ -122,7 +122,7 @@ class AndroidLauncher(Launcher):
self._device_id = None
self.launch_proc = None
self.android_vfs_setreg_path = None
self.package_name = get_package_name(os.path.join(self.workspace.paths.dev(),
self.package_name = get_package_name(os.path.join(self.workspace.paths.engine_root(),
self.workspace.project))
self._device_id = self.get_device_config(config_file=self.workspace.paths.devices_file(),
device_section='android',
Tools/LyTestTools/ly_test_tools/launchers/platforms/win/launcher.py
+1 -1
View File
@@ -165,7 +165,7 @@ class WinLauncher(Launcher):
"""
# Update settings via the settings registry to avoid modifying the bootstrap.cfg
host_ip = '127.0.0.1'
self.args.append(f'--regset="/Amazon/AzCore/Bootstrap/project_path={self.workspace.project}"')
self.args.append(f'--regset="/Amazon/AzCore/Bootstrap/project_path={self.workspace.paths.project()}"')
self.args.append(f'--regset="/Amazon/AzCore/Bootstrap/remote_ip={host_ip}"')
self.args.append('--regset="/Amazon/AzCore/Bootstrap/wait_for_connect=1"')
self.args.append(f'--regset="/Amazon/AzCore/Bootstrap/allowed_list={host_ip}"')
Tools/LyTestTools/ly_test_tools/lumberyard/asset_processor.py
+10 -17
View File
@@ -17,20 +17,14 @@ import logging
import subprocess
import socket
import time
import itertools
from timeit import default_timer
import pytest
import platform
import random
import tempfile
import shutil
import stat
from typing import Dict, List, Tuple, Optional, Callable
from typing import List
import psutil
import ly_test_tools
import ly_test_tools.environment.waiter as waiter
import ly_test_tools.environment.process_utils as process_utils
import ly_test_tools.environment.file_system as file_system
import ly_test_tools.lumberyard.pipeline_utils as utils
from ly_test_tools.lumberyard.ap_log_parser import APLogParser
@@ -49,7 +43,7 @@ ASSET_PROCESSOR_PLATFORM_MAP = {
'windows': 'pc',
}
ASSET_PROCESSOR_SETTINGS_ROOT_KEY='/Amazon/AssetProcessor/Settings'
ASSET_PROCESSOR_SETTINGS_ROOT_KEY = '/Amazon/AssetProcessor/Settings'
class AssetProcessorError(Exception):
""" Indicates that the AssetProcessor raised an error """
@@ -70,7 +64,7 @@ class AssetProcessor(object):
self._ap_proc = None
self._temp_asset_directory = None
self._temp_asset_root = None
self._project_path = self._workspace.project
self._project_path = self._workspace.paths.project()
self._override_scan_folders = []
self._test_assets_source_folder = None
self._cache_folder = None
@@ -136,7 +130,6 @@ class AssetProcessor(object):
try:
self._control_connection.sendall(message.encode())
logger.info(f"Sent input {message}")
return True
except IOError as e:
@@ -485,8 +478,7 @@ class AssetProcessor(object):
logger.warning(f"Cannot capture output when leaving AP connection open.")
logger.info(f"Launching AP with command: {command}")
self._ap_proc = subprocess.Popen(command,
cwd=ap_exe_path)
self._ap_proc = subprocess.Popen(command, cwd=ap_exe_path)
if accept_input and not quitonidle:
self.connect_control()
@@ -668,10 +660,11 @@ class AssetProcessor(object):
make_dir = os.path.join(self._temp_asset_root, copy_dir)
if not os.path.isdir(make_dir):
os.makedirs(make_dir)
for copyfile_name in ['bootstrap.cfg', 'AssetProcessorPlatformConfig.setreg',
for copyfile_name in ['bootstrap.cfg',
'AssetProcessorPlatformConfig.setreg',
os.path.join(self._workspace.project, "project.json"),
os.path.join('Engine', 'exclude.filetag')]:
shutil.copyfile(os.path.join(self._workspace.paths.dev(), copyfile_name),
shutil.copyfile(os.path.join(self._workspace.paths.engine_root(), copyfile_name),
os.path.join(self._temp_asset_root, copyfile_name))
def delete_temp_asset_root(self):
@@ -756,7 +749,7 @@ class AssetProcessor(object):
if not self._temp_asset_root:
logger.warning(f"Can't create scan folder, no temporary asset workspace has been created")
return
scan_folder = os.path.join(self._temp_asset_root if self._temp_asset_root else self._workspace.paths.dev(),
scan_folder = os.path.join(self._temp_asset_root if self._temp_asset_root else self._workspace.paths.engine_root(),
folder_name)
if not os.path.isdir(scan_folder):
os.makedirs(scan_folder)
@@ -901,7 +894,7 @@ class AssetProcessor(object):
asset root if not supplied
:return: None
"""
source_root = source_root or self._workspace.paths.dev()
source_root = source_root or self._workspace.paths.engine_root()
if not self._temp_asset_root:
logger.warning(f"Can't add relative source asset, no temporary asset root created")
return
@@ -966,7 +959,7 @@ class AssetProcessor(object):
supplied
:return: path to project cache
"""
source_folder = os.path.join(self._workspace.paths.dev(), relative_source)
source_folder = os.path.join(self._workspace.paths.engine_root(), relative_source)
dest_relative = relative_dest or relative_source
dest_folder = os.path.join(self._temp_asset_root, dest_relative)
shutil.copytree(source_folder, dest_folder)
Tools/LyTestTools/ly_test_tools/lumberyard/asset_processor_config_util.py
+40 -112
View File
@@ -13,38 +13,14 @@ AssetProcessorPlatformConfig.setreg
"""
import json
import logging
import os.path as path
from ly_test_tools.lumberyard.settings import RegistrySettings
from ly_test_tools.lumberyard.asset_processor import ASSET_PROCESSOR_SETTINGS_ROOT_KEY
logger = logging.getLogger(__name__)
AssetProcessorConfig = "AssetProcessorPlatformConfig.setreg"
def load_asset_processor_platform_config(file_location):
"""
Loads the AssetProcessorPlatformConfig.setreg file removing any comments
:param file_location: The file path of AssetProcessorPlatformConfig.setreg
"""
json_dict = {}
with open(file_location, 'r') as json_file:
cleaned_lines = []
for line in json_file.readlines():
lineIndex = line.lstrip().startswith('//')
cleaned_lines.append(line if lineIndex == -1 else line[:lineIndex])
json_dict = json.loads(''.join(cleaned_lines))
return json_dict
def save_asset_processor_platform_config(file_location, json_dict):
"""
Saves the json_dict to the AssetProcessorPlatformConfig.setreg file
:param file_location: The file path of AssetProcessorPlatformConfig.setreg
"""
with open(file_location, 'w') as json_file:
json.dump(json_dict, json_file, indent=4)
return json_dict
def platform_exists(config_setreg_path, platform):
"""
Checks to see if a specific Platform Key is in Platforms Section
@@ -53,19 +29,15 @@ def platform_exists(config_setreg_path, platform):
:param platform: Name of the Platform Key that you're checking exists
:return: The boolean value of the existance of the platform
"""
logger.debug("Checking for Platform '{0}' in Platforms Section of '{1}"
.format(platform, AssetProcessorConfig))
file_location = path.join(config_setreg_path, AssetProcessorConfig)
json_dict = load_asset_processor_platform_config(file_location)
try:
platformExist = platform in json_dict['Amazon']['AssetProcessor']['Settings']['Platforms']
return platformExist
except KeyError as err:
logger.error(f'KeyError when attempting to query platform existance of {platform} in file {file_location}: {err}')
return False
with RegistrySettings(file_location) as settings:
platformExist = settings.get_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/Platforms/{platform}', False)
return platformExist
def is_platform_enabled(config_setreg_path, platform):
@@ -77,20 +49,15 @@ def is_platform_enabled(config_setreg_path, platform):
See asset_processor.SUPPORTED_PLATFORMS for listed of supported platforms
:return: The boolean value of the enabled state of the platform
"""
logger.debug("Checking if Platform '{0}' is enabled in Platform Section of '{1}"
.format(platform, AssetProcessorConfig))
file_location = path.join(config_setreg_path, AssetProcessorConfig)
# load AssetProcessorPlatformConfig.setreg removing any comments
json_dict = load_asset_processor_platform_config(file_location)
try:
enabled = json_dict['Amazon']['AssetProcessor']['Settings']['Platforms'][platform] == 'enabled'
return enabled
except KeyError as err:
logger.error(f'KeyError when attempting to check if platform {platform} is enabled in file {file_location}: {err}')
return False
with RegistrySettings(file_location) as settings:
enabled = settings.get_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/Platforms/{platform}', None) == 'enabled'
return enabled
def enable_platform(config_setreg_path, platform):
@@ -103,15 +70,11 @@ def enable_platform(config_setreg_path, platform):
:assert: Assert if the platform is not enabled
:return: None
"""
logger.debug("Enabling platform '{0}' in '{1}'".format(platform, AssetProcessorConfig))
file_location = path.join(config_setreg_path, AssetProcessorConfig)
json_dict = load_asset_processor_platform_config(file_location)
# Enable platform in settings registry
json_dict.setdefault('Amazon', {}).setdefault('AssetProcessor', {}).setdefault('Settings',{}) \
.setdefault('Platforms',{})[platform] = 'enabled'
save_asset_processor_platform_config(file_location)
with RegistrySettings(file_location) as settings:
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/Platforms/{platform}', 'enabled')
def enable_all_platforms(config_setreg_path):
@@ -121,7 +84,6 @@ def enable_all_platforms(config_setreg_path):
:param config_setreg_path: The file path to the location of AssetProcessorPlatformConfig.setreg
:return: None
"""
logger.debug("Enabling all supported platforms in '{0}'.".format(AssetProcessorConfig))
for platform in SUPPORTED_PLATFORMS:
@@ -140,15 +102,13 @@ def disable_platform(config_setreg_path, platform):
:assert: Assert if the platform is not disabled
:return: None
"""
logger.debug("Disabling platform '{0}' in '{1}'".format(platform, AssetProcessorConfig))
file_location = path.join(config_setreg_path, AssetProcessorConfig)
# Disable platform in settings registry
json_dict = load_asset_processor_platform_config(file_location)
json_dict.setdefault('Amazon', {}).setdefault('AssetProcessor', {}).setdefault('Settings',{}) \
.setdefault('Platforms',{})[platform] = 'disabled'
save_asset_processor_platform_config(file_location)
file_location = path.join(config_setreg_path, AssetProcessorConfig)
with RegistrySettings(file_location) as settings:
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/Platforms/{platform}', 'disabled')
def disable_all_platforms(config_setreg_path):
@@ -158,7 +118,6 @@ def disable_all_platforms(config_setreg_path):
:param config_setreg_path: The file path to the location of AssetProcessorPlatformConfig.setreg
:return: None
"""
logger.debug("Disabling all platforms in '{0}'".format(AssetProcessorConfig))
for platform in SUPPORTED_PLATFORMS:
@@ -174,18 +133,15 @@ def enable_scanfolder_engine(config_setreg_path):
:param config_setreg_path: The file path to the location of AssetProcessorPlatformConfig.setreg
:return: None
"""
file_location = path.join(config_setreg_path, AssetProcessorConfig)
section = 'ScanFolder Engine'
logger.debug("Enabling Asset Processor scanning of the Engine folder")
json_dict = load_asset_processor_platform_config(file_location)
json_dict.setdefault('Amazon', {}).setdefault('AssetProcessor', {}).setdefault('Settings',{}) \
.setdefault(section,{})['watch'] = '@ENGINEROOT@/Engine'
json_dict['Amazon']['AssetProcessor']['Settings'][section]['recursive'] = '1'
json_dict['Amazon']['AssetProcessor']['Settings'][section]['order'] = '20000'
save_asset_processor_platform_config(file_location)
with RegistrySettings(file_location) as settings:
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/watch', '@ENGINEROOT@/Engine')
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/recursive', '1')
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/order', '20000')
def disable_scanfolder_engine(config_setreg_path):
@@ -195,18 +151,12 @@ def disable_scanfolder_engine(config_setreg_path):
:param config_setreg_path: The file path to the location of AssetProcessorPlatformConfig.setreg
:return: None
"""
file_location = path.join(config_setreg_path, AssetProcessorConfig)
section = 'ScanFolder Engine'
logger.debug("Disabling Asset Processor scanning of the Engine folder")
json_dict = load_asset_processor_platform_config(file_location)
try:
del json_dict['Amazon']['AssetProcessor']['Settings'][section]
save_asset_processor_platform_config(file_location)
except KeyError as err:
logger.debug(f'No-op: f{section} key does not exist in file {file_location}')
with RegistrySettings(file_location) as settings:
settings.remove_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}')
def enable_scanfolder_editor(config_setreg_path):
@@ -216,21 +166,17 @@ def enable_scanfolder_editor(config_setreg_path):
:param config_setreg_path: The file path to the location of AssetProcessorPlatformConfig.setreg
:return: None
"""
file_location = path.join(config_setreg_path, AssetProcessorConfig)
section = 'ScanFolder Editor'
logger.debug("Enabling Asset Processor scanning of the Editor folder")
json_dict = load_asset_processor_platform_config(file_location)
json_dict.setdefault('Amazon', {}).setdefault('AssetProcessor', {}).setdefault('Settings',{}) \
.setdefault(section,{})['watch'] = '@ENGINEROOT@/Editor'
json_dict['Amazon']['AssetProcessor']['Settings'][section]['output'] = 'editor'
json_dict['Amazon']['AssetProcessor']['Settings'][section]['recursive'] = '1'
json_dict['Amazon']['AssetProcessor']['Settings'][section]['order'] = '30000'
json_dict['Amazon']['AssetProcessor']['Settings'][section]['include'] = 'tools,renderer'
save_asset_processor_platform_config(file_location)
with RegistrySettings(file_location) as settings:
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/watch', '@ENGINEROOT@/Editor')
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/output', 'editor')
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/recursive', '1')
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/order', '30000')
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/include', 'tools,renderer')
def disable_scanfolder_editor(config_setreg_path):
@@ -240,17 +186,13 @@ def disable_scanfolder_editor(config_setreg_path):
:param config_setreg_path: The file path to the location of AssetProcessorPlatformConfig.setreg
:return: None
"""
file_location = path.join(config_setreg_path, AssetProcessorConfig)
section = 'ScanFolder Editor'
logger.debug("Disabling Asset Processor scanning of the Editor folder")
try:
del json_dict['Amazon']['AssetProcessor']['Settings'][section]
save_asset_processor_platform_config(file_location)
except KeyError as err:
logger.debug(f'No-op: f{section} key does not exist in file {file_location}')
with RegistrySettings(file_location) as settings:
settings.remove_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}')
def enable_scanfolder_root(config_setreg_path):
@@ -260,18 +202,15 @@ def enable_scanfolder_root(config_setreg_path):
:param config_setreg_path: The file path to the location of AssetProcessorPlatformConfig.setreg
:return: None
"""
file_location = path.join(config_setreg_path, AssetProcessorConfig)
section = 'ScanFolder Root'
logger.debug("Enabling Asset Processor scanning of the Root folder")
json_dict = load_asset_processor_platform_config(file_location)
json_dict.setdefault('Amazon', {}).setdefault('AssetProcessor', {}).setdefault('Settings',{}) \
.setdefault(section,{})['watch'] = '@ROOT@'
json_dict['Amazon']['AssetProcessor']['Settings'][section]['recursive'] = '0'
json_dict['Amazon']['AssetProcessor']['Settings'][section]['order'] = '10000'
save_asset_processor_platform_config(file_location)
with RegistrySettings(file_location) as settings:
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/watch', '@ROOT@')
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/recursive', '0')
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}/order', '10000')
def disable_scanfolder_root(config_setreg_path):
@@ -281,17 +220,13 @@ def disable_scanfolder_root(config_setreg_path):
:param config_setreg_path: The file path to the location of AssetProcessorPlatformConfig.setreg
:return: None
"""
file_location = path.join(config_setreg_path, AssetProcessorConfig)
section = 'ScanFolder Root'
logger.debug("Disabling Asset Processor scanning of the Root folder")
try:
del json_dict['Amazon']['AssetProcessor']['Settings'][section]
save_asset_processor_platform_config(file_location)
except KeyError as err:
logger.debug(f'No-op: f{section} key does not exist in file {file_location}')
with RegistrySettings(file_location) as settings:
settings.remove_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{section}')
def update_pattern(config_setreg_path, pattern, key, value):
@@ -304,15 +239,12 @@ def update_pattern(config_setreg_path, pattern, key, value):
:param value: The value to set the key to
:return: None
"""
file_location = path.join(config_setreg_path, AssetProcessorConfig)
rc_pattern = 'RC ' + pattern
logger.debug("Modifying pattern for '{0}'.".format(rc_pattern))
json_dict = load_asset_processor_platform_config(file_location)
json_dict.setdefault('Amazon', {}).setdefault('AssetProcessor', {}).setdefault('Settings',{}) \
.setdefault(rc_pattern,{})[key] = value
save_asset_processor_platform_config(file_location)
with RegistrySettings(file_location) as settings:
settings.set_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{rc_pattern}/{key}', value)
def get_pattern_key(config_setreg_path, pattern, key):
@@ -328,11 +260,7 @@ def get_pattern_key(config_setreg_path, pattern, key):
file_location = path.join(config_setreg_path, AssetProcessorConfig)
rc_pattern = 'RC ' + pattern
logger.debug("Retrieving the key '{0}' from pattern '{1}'.".format(key, rc_pattern))
json_dict = load_asset_processor_platform_config(file_location)
try:
value = json_dict['Amazon']['AssetProcessor']['Settings'][rc_pattern][key]
return value
except KeyError as err:
logger.error(f'KeyError attempting to query value for key "/Amazon/AssetProcessor/Settings/{rc_pattern}/{key}" in file {file_location}: {err}')
with RegistrySettings(file_location) as settings:
value = settings.get_key(f'{ASSET_PROCESSOR_SETTINGS_ROOT_KEY}/{rc_pattern}/{key}', None)
return None
return value
Tools/LyTestTools/ly_test_tools/lumberyard/pipeline_utils.py
-37
View File
@@ -215,43 +215,6 @@ def backup_asset_processor_logs(bin_directory: str, backup_directory: str) -> No
shutil.copytree(ap_logs, destination)
def platform_enabled(workspace: pytest.fixture, platform: str) -> bool:
"""
Checks to see if the platform specified is enabled for the current build of LY.
:param workspace: The current testing workspace
:param platform: The name of the platform to lookup.
:return: True if the platform is enabled, False if not.
"""
user_settings_file = os.path.join(workspace.paths.dev(), "_WAF_", "user_settings.options")
assert os.path.exists(user_settings_file), f"User settings file not found at {user_settings_file}"
parser = ConfigParser()
parser.read(user_settings_file)
section = platform.title() + " Options" # The Platform Options section
option = "enable_" + platform.lower()
# Make sure the platform has an options section
assert parser.has_section(section), f"Section {section} was not found in {user_settings_file}"
if parser.has_option(section, option):
entry = parser.get(section, option)
if entry.lower() == "true":
# Found 'true'
return True
elif entry.lower() == "false":
# Found 'false'
return False
else:
# Found something unexpected
# fmt:off
logger.warning(f"Found unexpected value '{entry}' in {user_settings_file} - {section}:{option}. "
f"Using default value of 'False'")
# fmt:on
else:
logger.info(f"No option '{option}' was found in the section '{section}', defaulting to 'False'")
return False
def safe_subprocess(command: str or List[str], **kwargs: Dict) -> ProcessOutput:
"""
Forwards arguments to subprocess.Popen to have a processes output
Tools/LyTestTools/ly_test_tools/lumberyard/settings.py
+149
View File
@@ -12,6 +12,7 @@ LySettings provides an API for modifying settings files and creating/restoring b
"""
import fileinput
import json
import logging
import re
import os
@@ -89,6 +90,24 @@ class LySettings(object):
def restore_shader_compiler_settings(self, backup_path=None):
self._restore_settings(self._resource_locator.shader_compiler_config_file(), backup_path)
def backup_json_settings(self, json_settings_file, backup_path=None):
"""
Creates a backup of a Json/Registry Settings file in the backup_path. If no path is provided, it will store in
the workspace temp path (the contents of the workspace temp directory are removed during workspace teardown)
:param json_settings_file:
:param backup_path:
:return:
"""
self._backup_settings(json_settings_file, backup_path)
def restore_json_settings(self, json_settings_file, backup_path=None):
"""
Restores a Json/Registry settings file from its backup.
The backup is stored in the backup_path.
If no backup_path is provided, it will attempt to retrieve the backup from the workspace temp path.
"""
self._restore_settings(json_settings_file, backup_path)
def _backup_settings(self, settings_file, backup_path):
"""
Creates a backup of the settings file in the backup_path. If no path is
@@ -109,6 +128,136 @@ class LySettings(object):
ly_test_tools.environment.file_system.restore_backup(settings_file, backup_path)
class JsonSettings(object):
"""
JsonSettings provides an API for reading, modifying and writing Json settings files.
Can be used as a Context Manager.
"""
def __init__(self, file_path, line_clean_func=None):
self._data = None
self._file_path = file_path
self._line_clean_func = line_clean_func
self._modified = False
self._path_separator = '/'
def __enter__(self):
self.load()
return self
def __exit__(self, exception_type, exception_value, traceback):
if self._modified:
self.dump()
def _get_path_tokens(self, key_path):
"""
Splits the path using the separator and returns a list of tokens.
The first empty token is skipped, since it points to the root of the document.
:param key_path: Path to the key
:return: List of path tokens
"""
path_tokens = key_path.split(self._path_separator)
# Skip first empty token, it points to the root of the document
path_tokens = path_tokens[1:]
return path_tokens
def load(self):
"""
Load the Json file into memory
:return: None
"""
with open(self._file_path, 'r') as contents:
if not self._line_clean_func:
self._data = json.load(contents)
else:
cleaned_lines = []
for line in contents:
clean_line = self._line_clean_func(line)
cleaned_lines.append(clean_line)
self._data = json.loads(cleaned_lines)
def dump(self):
"""
Write the content into the Json file
:return: None
"""
with open(self._file_path, 'w') as out_file:
json.dump(self._data, out_file, indent=4)
def get_key(self, key_path, default_value=None):
"""
Retrieve a key by path from the Json
:param key_path: Path to the key
:param default_value: Default value to return if the key is not found
:return: Value of the key or default_value if not found
"""
path_tokens = self._get_path_tokens(key_path)
obj = self._data
try:
for token in path_tokens:
obj = obj[token]
return obj
except KeyError as err:
logger.error(f'KeyError when retrieving {key_path} from file {self._file_path}: {err}')
return default_value
def set_key(self, key_path, value):
"""
Set a key to a value
:param key_path: Path to the key
:param value: The value
:return: None
"""
path_tokens = self._get_path_tokens(key_path)
obj = self._data
try:
for token in path_tokens[0:-1]:
obj = obj.setdefault(token, {})
obj[path_tokens[-1]] = value
self._modified = True
except KeyError as err:
logger.error(f'KeyError when setting {key_path} from file {self._file_path}: {err}')
def remove_key(self, key_path):
"""
Remove a key by path from the Json
:param key_path: Path to the key
:return: None
"""
path_tokens = self._get_path_tokens(key_path)
obj = self._data
parent = None
key = None
try:
for token in path_tokens:
key = token
parent = obj
obj = obj[token]
if parent:
del parent[key]
self._modified = True
else:
logger.warning(f'Could not remove key {key} from file {self._file_path}')
except KeyError as err:
logger.error(f'KeyError when removing {key_path} from file {self._file_path}: {err}')
class RegistrySettings(JsonSettings):
"""
JsonSettings provides an API for reading, modifying and writing Registry settings files.
Files must be Json-compatible with optional line comments starting with '//'.
"""
def __init__(self, file_path):
super(RegistrySettings, self).__init__(file_path, RegistrySettings._clean_line)
@staticmethod
def _clean_line(line):
comment_index = line.lstrip.startswith('//')
return line if comment_index == -1 else line[:comment_index]
def _edit_text_settings_file(settings_file, setting, value, comment_char=""):
"""
Find and set a specific setting in a text based settings file. Uses "setting = value" syntax to identify setting,
Tools/LyTestTools/tests/unit/test_abstract_resource_locator.py
+7 -17
View File
@@ -32,14 +32,11 @@ class TestFindEngineRoot(object):
@mock.patch('ly_test_tools._internal.managers.abstract_resource_locator.os.path.exists')
def test_FindEngineRoot_InitialPathExists_ReturnsTuple(self, mock_path_exists, mock_abspath):
mock_path_exists.return_value = True
mock_abspath.return_value = mock_engine_root
engine_root, dev_path = abstract_resource_locator._find_engine_root(mock_initial_path)
engine_root = abstract_resource_locator._find_engine_root(mock_engine_root)
assert engine_root == mock_engine_root
assert dev_path == mock_initial_path
mock_path_exists.assert_called_once()
mock_abspath.assert_called_once()
@mock.patch('ly_test_tools._internal.managers.abstract_resource_locator.os.path.abspath')
@mock.patch('ly_test_tools._internal.managers.abstract_resource_locator.os.path.exists')
@@ -54,7 +51,7 @@ class TestFindEngineRoot(object):
@mock.patch('ly_test_tools._internal.managers.abstract_resource_locator.os.path.abspath',
mock.MagicMock(return_value=mock_initial_path))
@mock.patch('ly_test_tools._internal.managers.abstract_resource_locator._find_engine_root',
mock.MagicMock(return_value=(mock_engine_root, mock_dev_path)))
mock.MagicMock(return_value=mock_engine_root))
class TestAbstractResourceLocator(object):
def test_Init_HasEngineRoot_SetsAttrs(self):
@@ -63,7 +60,6 @@ class TestAbstractResourceLocator(object):
assert mock_abstract_resource_locator._build_directory == mock_build_directory
assert mock_abstract_resource_locator._engine_root == mock_engine_root
assert mock_abstract_resource_locator._dev_path == mock_dev_path
assert mock_abstract_resource_locator._project == mock_project
def test_BasePath_IsCalled_ReturnsBasePath(self):
@@ -72,12 +68,6 @@ class TestAbstractResourceLocator(object):
assert mock_abstract_resource_locator.engine_root() == mock_engine_root
def test_Dev_IsCalled_ReturnsDevPath(self):
mock_abstract_resource_locator = abstract_resource_locator.AbstractResourceLocator(
mock_build_directory, mock_project)
assert mock_abstract_resource_locator.dev() == mock_dev_path
@mock.patch('ly_test_tools._internal.managers.abstract_resource_locator.os.path.isfile')
def test_3rdParty_IsCalledHasTxtFile_Returns3rdPartyPath(self, mock_isfile):
mock_isfile.return_value = True
@@ -106,7 +96,7 @@ class TestAbstractResourceLocator(object):
def test_Project_IsCalled_ReturnsProjectPath(self):
mock_abstract_resource_locator = abstract_resource_locator.AbstractResourceLocator(
mock_build_directory, mock_project)
expected_path = os.path.join(mock_abstract_resource_locator.dev(), mock_project)
expected_path = os.path.join(mock_abstract_resource_locator.engine_root(), mock_project)
assert mock_abstract_resource_locator.project() == expected_path
@@ -171,21 +161,21 @@ class TestAbstractResourceLocator(object):
def test_BootstrapConfigFile_IsCalled_ReturnBootstrapConfigFilePath(self):
mock_abstract_resource_locator = abstract_resource_locator.AbstractResourceLocator(
mock_build_directory, mock_project)
expected_path = os.path.join(mock_abstract_resource_locator.dev(), 'bootstrap.cfg')
expected_path = os.path.join(mock_abstract_resource_locator.engine_root(), 'bootstrap.cfg')
assert mock_abstract_resource_locator.bootstrap_config_file() == expected_path
def test_AssetProcessorConfigFile_IsCalled_ReturnsAssetProcessorConfigFilePath(self):
mock_abstract_resource_locator = abstract_resource_locator.AbstractResourceLocator(
mock_build_directory, mock_project)
expected_path = os.path.join(mock_abstract_resource_locator.dev(), 'AssetProcessorPlatformConfig.setreg')
expected_path = os.path.join(mock_abstract_resource_locator.engine_root(), 'AssetProcessorPlatformConfig.setreg')
assert mock_abstract_resource_locator.asset_processor_config_file() == expected_path
def test_AutoexecFile_IsCalled_ReturnsAutoexecFilePath(self):
mock_abstract_resource_locator = abstract_resource_locator.AbstractResourceLocator(
mock_build_directory, mock_project)
expected_path = os.path.join(mock_abstract_resource_locator.dev(),
expected_path = os.path.join(mock_abstract_resource_locator.engine_root(),
mock_abstract_resource_locator._project,
'autoexec.cfg')
@@ -194,7 +184,7 @@ class TestAbstractResourceLocator(object):
def test_TestResults_IsCalled_TestResultsPath(self):
mock_abstract_resource_locator = abstract_resource_locator.AbstractResourceLocator(
mock_build_directory, mock_project)
expected_path = os.path.join(mock_abstract_resource_locator.dev(), 'TestResults')
expected_path = os.path.join(mock_abstract_resource_locator.engine_root(), 'TestResults')
assert mock_abstract_resource_locator.test_results() == expected_path
Tools/LyTestTools/tests/unit/test_asset_processor.py
+21 -11
View File
@@ -27,6 +27,7 @@ mock_engine_root = "mock_engine_root"
mock_dev_path = "mock_dev_path"
mock_build_directory = 'mock_build_directory'
mock_project = 'mock_project'
mock_project_path = os.path.join('some', 'dir', mock_project)
@mock.patch('ly_test_tools._internal.managers.abstract_resource_locator.os.path.abspath',
@@ -52,7 +53,8 @@ class TestAssetProcessor(object):
mock_ap_path = 'mock_ap_path'
mock_workspace.asset_processor_platform = 'foo'
mock_workspace.paths.asset_processor.return_value = mock_ap_path
mock_workspace.project = 'AutomatedTesting'
mock_workspace.project = mock_project
mock_workspace.paths.project.return_value = mock_project_path
under_test = ly_test_tools.lumberyard.asset_processor.AssetProcessor(mock_workspace)
under_test.enable_asset_processor_platform = mock.MagicMock()
under_test.wait_for_idle = mock.MagicMock()
@@ -60,7 +62,8 @@ class TestAssetProcessor(object):
under_test.start(connect_to_ap=True)
assert under_test._ap_proc is not None
mock_popen.assert_called_once_with([mock_ap_path, '--zeroAnalysisMode', '--regset="/Amazon/AzCore/Bootstrap/project_path=AutomatedTesting"',
mock_popen.assert_called_once_with([mock_ap_path, '--zeroAnalysisMode',
f'--regset="/Amazon/AzCore/Bootstrap/project_path={mock_project_path}"',
'--logDir', under_test.log_root(),
'--acceptInput', '--platforms', 'bar'], cwd=os.path.dirname(mock_ap_path))
mock_connect.assert_called()
@@ -107,20 +110,24 @@ class TestAssetProcessor(object):
@mock.patch('subprocess.run')
def test_BatchProcess_NoFastscanBatchCompletes_Success(self, mock_run, mock_workspace):
mock_workspace.project = None
mock_workspace.paths.project.return_value = mock_project_path
under_test = ly_test_tools.lumberyard.asset_processor.AssetProcessor(mock_workspace)
apb_path = mock_workspace.paths.asset_processor_batch()
mock_run.return_value.returncode = 0
result, _ = under_test.batch_process(1, False)
assert result
mock_run.assert_called_once_with([apb_path, '--logDir', under_test.log_root()],
mock_run.assert_called_once_with([apb_path,
f'--regset="/Amazon/AzCore/Bootstrap/project_path={mock_project_path}"',
'--logDir', under_test.log_root()],
close_fds=True, capture_output=False,
timeout=1)
@mock.patch('ly_test_tools._internal.managers.workspace.AbstractWorkspaceManager')
@mock.patch('subprocess.run')
def test_BatchProcess_FastscanBatchCompletes_Success(self, mock_run, mock_workspace):
mock_workspace.project = 'AutomatedTesting'
mock_workspace.project = mock_project
mock_workspace.paths.project.return_value = mock_project_path
under_test = ly_test_tools.lumberyard.asset_processor.AssetProcessor(mock_workspace)
apb_path = mock_workspace.paths.asset_processor_batch()
mock_run.return_value.returncode = 0
@@ -128,18 +135,18 @@ class TestAssetProcessor(object):
result = under_test.batch_process(1, True)
assert result
mock_run.assert_called_once_with(
[apb_path, '--zeroAnalysisMode', '--regset="/Amazon/AzCore/Bootstrap/project_path=AutomatedTesting"',
'--logDir',
under_test.log_root()],
mock_run.assert_called_once_with([apb_path, '--zeroAnalysisMode',
f'--regset="/Amazon/AzCore/Bootstrap/project_path={mock_project_path}"',
'--logDir', under_test.log_root()],
close_fds=True, capture_output=False,
timeout=1)
close_fds=True, capture_output=False,
timeout=1)
@mock.patch('ly_test_tools._internal.managers.workspace.AbstractWorkspaceManager')
@mock.patch('subprocess.run')
def test_BatchProcess_ReturnCodeFail_Failure(self, mock_run, mock_workspace):
mock_workspace.project = None
mock_workspace.paths.project.return_value = mock_project_path
under_test = ly_test_tools.lumberyard.asset_processor.AssetProcessor(mock_workspace)
apb_path = mock_workspace.paths.asset_processor_batch()
mock_run.return_value.returncode = 1
@@ -147,9 +154,12 @@ class TestAssetProcessor(object):
result, _ = under_test.batch_process(None, False)
assert not result
mock_run.assert_called_once_with([apb_path, '--logDir', under_test.log_root()],
mock_run.assert_called_once_with([apb_path,
f'--regset="/Amazon/AzCore/Bootstrap/project_path={mock_project_path}"',
'--logDir', under_test.log_root()],
close_fds=True, capture_output=False, timeout=28800.0)
@mock.patch('ly_test_tools._internal.managers.workspace.AbstractWorkspaceManager')
def test_EnableAssetProcessorPlatform_AssetProcessorObject_Updated(self, mock_workspace):
under_test = ly_test_tools.lumberyard.asset_processor.AssetProcessor(mock_workspace)
Tools/LyTestTools/tests/unit/test_builtin_helpers.py
+1 -2
View File
@@ -102,7 +102,7 @@ class TestBuiltinHelpers(object):
under_test = ly_test_tools._internal.managers.abstract_resource_locator._find_engine_root(
initial_path='mock_dev_dir')
assert under_test == ('mock_base_dir', 'mock_dev_dir')
assert under_test == ('mock_dev_dir')
@mock.patch('os.path.abspath', mock.MagicMock(return_value='mock_base_dir'))
@mock.patch('os.path.exists', mock.MagicMock(return_value=False))
@@ -113,7 +113,6 @@ class TestBuiltinHelpers(object):
@mock.patch('ly_test_tools._internal.managers.workspace.AbstractWorkspaceManager.setup')
@mock.patch('ly_test_tools._internal.managers.artifact_manager.NullArtifactManager', mock.MagicMock())
@mock.patch('ly_test_tools.builtin.helpers.setup_bootstrap_project', mock.MagicMock(return_value=None))
@mock.patch('os.path.exists', mock.MagicMock(return_value=True))
def test_SetupBuiltinWorkspace_ValidWorkspaceSetup_ReturnsWorkspaceObject(self, mock_setup):
mock_test_name = 'mock_test_name'
Tools/LyTestTools/tests/unit/test_launcher_android.py
+1 -1
View File
@@ -39,7 +39,7 @@ class MockedWorkspace(object):
self.shader_compiler = mock.MagicMock()
self.settings = mock.MagicMock()
self.paths.dev.return_value = 'dev_path'
self.paths.engine_root.return_value = 'engine_path'
self.paths.build_directory.return_value = 'build_directory'
self.paths.autoexec_file.return_value = 'autoexec.cfg'
Tools/LyTestTools/tests/unit/test_manager_platforms_mac.py
+1 -1
View File
@@ -45,7 +45,7 @@ class TestMacResourceLocator(object):
def test_PlatformConfigFile_HasPath_ReturnsPath(self):
expected = os.path.join(
mac_resource_locator.dev(),
mac_resource_locator.engine_root(),
CONFIG_FILE)
assert mac_resource_locator.platform_config_file() == expected
Tools/LyTestTools/tests/unit/test_manager_platforms_windows.py
+1 -1
View File
@@ -51,7 +51,7 @@ class TestWindowsResourceLocator(object):
def test_PlatformConfigFile_HasPath_ReturnsPath(self):
expected = os.path.join(
windows_resource_locator.dev(),
windows_resource_locator.engine_root(),
CONFIG_FILE)
assert windows_resource_locator.platform_config_file() == expected
Tools/LyTestTools/tests/unit/test_settings.py
+92
View File
@@ -158,3 +158,95 @@ class TestReplaceLineInFile(unittest.TestCase):
ly_test_tools.lumberyard.settings._edit_text_settings_file(self.file_name, 'Setting5', 'NewSetting!')
mock_stdout.assert_has_calls(expected_print_lines)
class TestJsonSettings(unittest.TestCase):
def setUp(self):
self.test_file_name = 'something.json'
self.mock_file_content = """
{
"name": "Foo",
"weight": 30,
"scale": {
"x": 1,
"y": 2,
"z": 3
},
" ":"secret",
"": 0
}"""
def test_ReadJson_RetrieveKey_Success(self,):
mock_open = mock.mock_open(read_data=self.mock_file_content)
with mock.patch('builtins.open', mock_open):
with ly_test_tools.lumberyard.settings.JsonSettings(self.test_file_name) as js:
# get the whole document
value = js.get_key('')
assert len(value) == 5
# get a nested key
value = js.get_key('/scale/x')
assert value == 1
# get the " " key ad the root level
value = js.get_key('/ ')
assert value == 'secret'
# get the "" key at the root level
value = js.get_key('/')
assert value == 0
def test_ReadJson_RetrieveMissingKey_DefaultReturned(self):
mock_open = mock.mock_open(read_data=self.mock_file_content)
default_value = -10
with mock.patch('builtins.open', mock_open):
with ly_test_tools.lumberyard.settings.JsonSettings(self.test_file_name) as js:
value = js.get_key('/scale/w', default_value)
assert value == default_value
def test_ReadJson_ModifyKey_KeyModified(self):
mock_open = mock.mock_open(read_data=self.mock_file_content)
expected = 100
with mock.patch('builtins.open', mock_open):
with ly_test_tools.lumberyard.settings.JsonSettings(self.test_file_name) as js:
js.set_key('/scale/x', expected)
value = js.get_key('/scale/x')
assert value == expected
@mock.patch('json.dump')
def test_WriteJson_ModifyKey_KeyModified(self, json_dump):
mock_open = mock.mock_open(read_data=self.mock_file_content)
expected = "this is the new value"
new_dict_content = None
def _mock_dump(content, file_path, indent):
nonlocal new_dict_content
new_dict_content = content
json_dump.side_effect = _mock_dump
with mock.patch('builtins.open', mock_open):
with ly_test_tools.lumberyard.settings.JsonSettings(self.test_file_name) as js:
js.set_key('/name', expected)
assert expected == new_dict_content['name']
@mock.patch('json.dump')
def test_WriteJson_RemoveKey_KeyRemoved(self, json_dump):
mock_open = mock.mock_open(read_data=self.mock_file_content)
new_dict_content = None
def _mock_dump(content, file_path, indent):
nonlocal new_dict_content
new_dict_content = content
json_dump.side_effect = _mock_dump
with mock.patch('builtins.open', mock_open):
with ly_test_tools.lumberyard.settings.JsonSettings(self.test_file_name) as js:
js.remove_key('/scale/z')
assert len(new_dict_content['scale']) == 2
Tools/PakShaders/gen_shaders.py
-214
View File
@@ -1,214 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
import importlib
import os
import pkgutil
import shutil
import subprocess
import sys
ROOT_DEV_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))
if ROOT_DEV_PATH not in sys.path:
sys.path.append(ROOT_DEV_PATH)
class _Configuration:
def __init__(self, platform, asset_platform, core_compiler):
self.platform = platform
self.asset_platform = asset_platform
self.compiler = '{}-{}'.format(platform, core_compiler)
def __str__(self):
return '{} ({})'.format(self.platform, self.asset_platform)
class _ShaderType:
def __init__(self, name, base_compiler):
self.name = name
self.core_compiler = '{}-{}'.format(base_compiler, name)
self.configurations = []
def add_configuration(self, platform, asset_platform):
self.configurations.append(_Configuration(platform, asset_platform, self.core_compiler))
def find_shader_type(shader_type_name, shader_types):
return next((shader for shader in shader_types if shader.name == shader_type_name), None)
def find_shader_configuration(platform, assets, shader_configurations):
if platform:
check_func = lambda config: config.platform == platform and config.asset_platform == assets
else:
check_func = lambda config: config.asset_platform == assets
return next((config for config in shader_configurations if check_func(config)), None)
def error(msg):
print(msg)
exit(1)
def is_windows():
if os.name == 'nt':
return True
else:
return False
def gen_shaders(game_name, shader_type, shader_config, shader_list, bin_folder, game_path, engine_path, verbose):
"""
Generates the shaders for a specific platform and shader type using a list of shaders using ShaderCacheGen.
The generated shaders will be output at Cache/<game_name>/<asset_platform>/user/cache/Shaders/Cache/<shader_type>
"""
platform = shader_config.platform
asset_platform = shader_config.asset_platform
compiler = shader_config.compiler
asset_cache_root = os.path.join(game_path, 'Cache', game_name, asset_platform)
# Make sure that the Cache/.../user folder exists
cache_user_folder = os.path.join(asset_cache_root, 'user')
if not os.path.isdir(cache_user_folder):
try:
os.makedirs(cache_user_folder)
except os.error as err:
error("Unable to create the required cache folder '{}': {}".format(cache_user_folder, err))
cache_shader_list = os.path.join(cache_user_folder, 'cache', 'shaders', 'shaderlist.txt')
if shader_list is None:
if is_windows():
shader_compiler_platform = 'x64'
else:
shader_compiler_platform = 'osx'
shader_list_path = os.path.join(engine_path, 'Tools', 'CrySCompileServer', shader_compiler_platform, 'profile', 'Cache', game_name, compiler, 'ShaderList_{}.txt'.format(shader_type))
if not os.path.isfile(shader_list_path):
shader_list_path = cache_shader_list
print("Source Shader List not specified, using {} by default".format(shader_list_path))
else:
shader_list_path = os.path.join(game_path, shader_list)
normalized_shaderlist_path = os.path.normpath(os.path.normcase(os.path.realpath(shader_list_path)))
normalized_cache_shader_list = os.path.normpath(os.path.normcase(os.path.realpath(cache_shader_list)))
if normalized_shaderlist_path != normalized_cache_shader_list:
cache_shader_list_basename = os.path.split(cache_shader_list)[0]
if not os.path.exists(cache_shader_list_basename):
os.makedirs(cache_shader_list_basename)
print("Copying shader_list from {} to {}".format(shader_list_path, cache_shader_list))
shutil.copy2(shader_list_path, cache_shader_list)
platform_shader_cache_path = os.path.join(cache_user_folder, 'cache', 'shaders', 'cache', shader_type.lower())
shutil.rmtree(platform_shader_cache_path, ignore_errors=True)
shadergen_path = os.path.join(engine_path, bin_folder, 'ShaderCacheGen')
if is_windows():
shadergen_path += '.exe'
if not os.path.isfile(shadergen_path):
error("ShaderCacheGen could not be found at {}".format(shadergen_path))
else:
command_arguments = [
shadergen_path,
'/BuildGlobalCache',
'/ShadersPlatform={}'.format(shader_type),
'/TargetPlatform={}'.format(asset_platform)
]
if verbose:
print('Running: {}'.format(' '.join(command_arguments)))
subprocess.call(command_arguments)
def add_shaders_types():
"""
Add the shader types for the non restricted platforms.
The compiler argument is used for locating the shader_list file.
"""
shaders = []
d3d11 = _ShaderType('D3D11', 'D3D11_FXC')
d3d11.add_configuration('PC', 'pc')
shaders.append(d3d11)
gl4 = _ShaderType('GL4', 'GLSL_HLSLcc')
gl4.add_configuration('PC', 'pc')
shaders.append(gl4)
gles3 = _ShaderType('GLES3', 'GLSL_HLSLcc')
gles3.add_configuration('Android', 'es3')
shaders.append(gles3)
metal = _ShaderType('METAL', 'METAL_LLVM_DXC')
metal.add_configuration('Mac', 'osx_gl')
metal.add_configuration('iOS', 'ios')
shaders.append(metal)
restricted_path = os.path.join(ROOT_DEV_PATH, 'restricted')
if os.path.exists(restricted_path):
restricted_platforms = os.listdir(restricted_path)
for platform in restricted_platforms:
try:
imported_module = importlib.import_module(f'restricted.{platform}.Tools.PakShaders.gen_shaders')
except ImportError:
continue
restricted_func = getattr(imported_module, 'get_restricted_platform_shader', lambda: iter(()))
for shader_type, shader_compiler, platform_name, asset_platform in restricted_func():
shader = find_shader_type(shader_type, shaders)
if shader is None:
shader = _ShaderType(shader_type, shader_compiler)
shaders.append(shader)
shader.add_configuration(platform_name, asset_platform)
return shaders
def check_arguments(args, parser, shader_types):
"""
Check that the platform and shader type arguments are correct.
"""
shader_names = [shader.name for shader in shader_types]
shader_found = find_shader_type(args.shader_type, shader_types)
if shader_found is None:
parser.error('Invalid shader type {}. Must be one of [{}]'.format(args.shader_type, ' '.join(shader_names)))
else:
config_found = find_shader_configuration(args.shader_platform, args.asset_platform, shader_found.configurations)
if config_found is None:
parser.error('Invalid configuration for shader type "{}". It must be one of the following: {}'.format(shader_found.name, ', '.join(str(config) for config in shader_found.configurations)))
args.game_path = args.game_path or args.engine_path
parser = argparse.ArgumentParser(description='Generates the shaders for a specific platform and shader type.')
parser.add_argument('game_name', type=str, help="Name of the game")
parser.add_argument('asset_platform', type=str, help="The asset cache sub folder to use for shader generation")
parser.add_argument('shader_type', type=str, help="The shader type to use")
parser.add_argument('-p', '--shader_platform', type=str, required=False, default='', help="The target platform to generate shaders for.")
parser.add_argument('-b', '--bin_folder', type=str, help="Folder where the ShaderCacheGen executable lives. This is used along the game_path (game_path/bin_folder/ShaderCacheGen)")
parser.add_argument('-e', '--engine_path', type=str, help="Path to the engine root folder. This the same as game_path for non external projects")
parser.add_argument('-g', '--game_path', type=str, required=False, help="Path to the game root folder. This the same as engine_path for non external projects")
parser.add_argument('-s', '--shader_list', type=str, required=False, help="Optional path to the list of shaders. If not provided will use the list generated by the local shader compiler.")
parser.add_argument('-v', '--verbose', action="store_true", required=False, help="Increase the logging output")
args = parser.parse_args()
shader_types = add_shaders_types()
check_arguments(args, parser, shader_types)
print('Generating shaders for {} (shaders={}, platform={}, assets={})'.format(args.game_name, args.shader_type, args.shader_platform, args.asset_platform))
shader = find_shader_type(args.shader_type, shader_types)
shader_config = find_shader_configuration(args.shader_platform, args.asset_platform, shader.configurations)
gen_shaders(args.game_name, args.shader_type, shader_config, args.shader_list, args.bin_folder, args.game_path, args.engine_path, args.verbose)
print('Finish generating shaders')
Tools/PakShaders/get_shader_list.py
-67
View File
@@ -1,67 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
import os
import shutil
import subprocess
def error(msg):
print(msg)
exit(1)
def is_windows():
if os.name == 'nt':
return True
else:
return False
def get_shader_list(game_name, asset_platform, shader_type, shader_platform, shadergen_path):
"""
Gets the shader list for a specific platform using ShaderCacheGen.
Right now the shader list will always output at Cache/<game_name>/<asset_platform>/user/cache/shaders
That will change when this is updated to take a destination path
"""
shadergen_path = os.path.join(shadergen_path, 'ShaderCacheGen')
if is_windows():
shadergen_path += '.exe'
command_args = [
shadergen_path,
'/GetShaderList',
'/ShadersPlatform={}'.format(shader_type),
'/TargetPlatform={}'.format(asset_platform)
]
if not os.path.isfile(shadergen_path):
error("[ERROR] ShaderCacheGen could not be found at {}".format(shadergen_path))
else:
command = ' '.join(command_args)
print('[INFO] get_shader_list: Running command - {}'.format(command))
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
error('[ERROR] Failed to get the shader list for {}'.format(shader_type))
parser = argparse.ArgumentParser(description='Gets the shader list for a specific platform from the current shader compiler server')
parser.add_argument('game_name', type=str, help="Name of the game")
parser.add_argument('asset_platform', type=str, help="The asset cache sub folder to use for shader generation")
parser.add_argument('shader_type', type=str, help="The shader type to use")
parser.add_argument('-p', '--shader_platform', type=str, required=False, default='', help="The target platform to generate shaders for.")
parser.add_argument('-s', '--shadergen_path', type=str, help="Path to where the the ShaderCacheGen executable lives")
args = parser.parse_args()
print('Getting shader list for {}'.format(args.asset_platform))
get_shader_list(args.game_name, args.asset_platform, args.shader_type, args.shader_platform, args.shadergen_path)
print('Finish getting shader list')
Tools/PakShaders/pak_shaders.py
-142
View File
@@ -1,142 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
import fnmatch
import glob
import os
import re
import sys
import zipfile
def _create_zip(source_path, zip_file_path, append, filter_list, ignore_list, compression_level, zip_internal_path = ''):
"""
Internal function for creating a zip file containing the files that match the list of filters provided.
File matching is case insensitive.
"""
ignore_list = [filter.lower() for filter in ignore_list]
filter_list = [filter.lower() for filter in filter_list]
ignore_list = r'|'.join([fnmatch.translate(x) for x in ignore_list])
try:
mode = 'a' if append else 'w'
with zipfile.ZipFile(zip_file_path, mode, compression_level) as myzip:
# Go through all the files in the source path.
for root, dirnames, filenames in os.walk(source_path):
# Remove files that match the ignore list.
files = [os.path.relpath(os.path.join(root, file), source_path).lower() for file in filenames if not re.match(ignore_list, file.lower())]
# Match the files we have found against the filters.
for filter in filter_list:
for filename in fnmatch.filter(files, filter):
# Add the file to the zip using the specified internal destination.
myzip.write(os.path.join(source_path, filename), os.path.join(zip_internal_path, filename))
except IOError as error:
print("I/O error({0}) while creating zip file {1}: {2}".format(error.errno, zip_file_path, error.strerror))
return False
except:
print("Unexpected error while creating zip file {0}: {1}".format(zip_file_path, sys.exc_info()[0]))
return False
return True
# Create or append a pak file with all the shaders found in source_path.
def pak_shaders_in_folder(source_path, output_folder, shader_type, append):
"""
Creates the shadercache.pak and the shadercachestartup.pak using the shader files located at source_path.
"""
ignore_list = ['shaderlist.txt', 'shadercachemisses.txt']
shaders_cache_startup_filters = ['Common.cfib', 'FXConstantDefs.cfib', 'FXSamplerDefs.cfib', 'FXSetupEnvVars.cfib',
'FXStreamDefs.cfib', 'fallback.cfxb', 'fallback.fxb', 'FixedPipelineEmu.cfxb',
'FixedPipelineEmu.fxb', 'Stereo.cfxb', 'Stereo.fxb', 'lookupdata.bin',
'Video.cfxb', 'Video.fxb',
os.path.join('CGPShaders', 'FixedPipelineEmu@*'),
os.path.join('CGVShaders', 'FixedPipelineEmu@*'),
os.path.join('CGPShaders', 'FixedPipelineEmu', '*'),
os.path.join('CGVShaders', 'FixedPipelineEmu', '*'),
os.path.join('CGPShaders', 'Stereo@*'),
os.path.join('CGVShaders', 'Stereo@*'),
os.path.join('CGPShaders', 'Stereo', '*'),
os.path.join('CGVShaders', 'Stereo', '*'),
os.path.join('CGPShaders', 'Video@*'),
os.path.join('CGVShaders', 'Video@*'),
os.path.join('CGPShaders', 'Video', '*'),
os.path.join('CGVShaders', 'Video', '*')
]
print('Packing shader source folder {}'.format(source_path))
result = True
if os.path.exists(source_path):
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# We want the files to be added to the "shaders/cache/$shader_type" path inside the pak file.
zip_interal_path = os.path.join('shaders', 'cache', shader_type)
result &= _create_zip(source_path, os.path.join(output_folder, 'shadercache.pak'), append, ['*.*'], ignore_list, zipfile.ZIP_STORED, zip_interal_path)
result &= _create_zip(source_path, os.path.join(output_folder, 'shadercachestartup.pak'), append, shaders_cache_startup_filters, ignore_list, zipfile.ZIP_STORED, zip_interal_path)
else:
print('[Error] Shader source folder is not available at {}. Shader type: {}.'.format(source_path, shader_type))
result = False
return result
# Generate a shaders pak file with all the shader types indicated.
# NOTE: A shader type can specify an specific source path or not. Examples:
# - 'metal,specific/path/to/shaders': Use the folder specified as the source path to all metal shaders.
# - 'metal': Use source_path/metal as source path to all metal shaders. Wildcard usage is allowed, for example 'gles3*'.
def pak_shaders(source_path, output_folder, shader_types):
shader_flavors_packed = 0
for shader_info in shader_types:
# First element is the type, the second (if present) is the specific source
shader_type = shader_info[0]
if len(shader_info) > 1:
shader_type_source = shader_info[1]
if pak_shaders_in_folder(shader_type_source, output_folder, shader_type, shader_flavors_packed > 0):
shader_flavors_packed = shader_flavors_packed + 1
else:
return False
else:
# No specific source path for this shader type so use the global source path. Wildcard allowed.
listing = glob.glob(os.path.join(source_path, shader_type))
for shader_type_source in listing:
if os.path.isdir(shader_type_source):
# Since the shader_type can use wildcard we have to obtain the actual shader type found by removing source_path
# Example: If shader_type is 'gl4*' then now will be 'gl4_4'
shader_type = shader_type_source[len(source_path)+1:]
if pak_shaders_in_folder(shader_type_source, output_folder, shader_type, shader_flavors_packed > 0):
shader_flavors_packed = shader_flavors_packed + 1
else:
return False
if shader_flavors_packed == 0:
print('Failed to pack any shader type')
return shader_flavors_packed > 0
def pair_arg(arg):
return [str(x) for x in arg.split(',')]
parser = argparse.ArgumentParser(description='Pack the provided shader files into paks.')
parser.add_argument("output", type=str, help="specify the output folder")
parser.add_argument('-r', '--source', type=str, required=False, help="specify global input folder")
parser.add_argument('-s', '--shaders_types', required=True, nargs='+', type=pair_arg,
help='list of shader types with optional source path')
args = parser.parse_args()
print('Packing shaders...')
if not pak_shaders(args.source, args.output, args.shaders_types):
print('Failed to pack shaders')
exit(1)
print('Packs have been placed at "{}"'.format(args.output))
print('To use them, deploy them in your assets folder.')
print('Finish packing shaders')
Tools/build/JenkinsScripts/build/PackageEnv.py
+1 -1
View File
@@ -68,7 +68,7 @@ class PackageEnv(Params):
def validate_engine_root(engine_root):
if not os.path.isdir(engine_root):
return False
return os.path.exists(os.path.join(engine_root, 'engineroot.txt'))
return os.path.exists(os.path.join(engine_root, 'engine.json'))
# Jenkins only
workspace = os.getenv('WORKSPACE')
Tools/build/JenkinsScripts/build/Platform/Android/build_and_run_unit_tests.cmd
+2 -1
View File
@@ -27,7 +27,8 @@ IF NOT EXIST "%LY_ANDROID_SDK%" (
ECHO [ci_build] FAIL: LY_ANDROID_SDK=!LY_ANDROID_SDK!
GOTO :error
)
SET ANDROID_SDK_ROOT=%LY_ANDROID_SDK%
ECHO "ANDROID_SDK_ROOT=!ANDROID_SDK_ROOT!"
SET PYTHON=python\python.cmd
ECHO [ci_build] %PYTHON% Tools\build\JenkinsScripts\build\Platform\Android\run_test_on_android_simulator.py --android-sdk-path %LY_ANDROID_SDK% --build-path %OUTPUT_DIRECTORY% --build-config %CONFIGURATION%
CALL %PYTHON% Tools\build\JenkinsScripts\build\Platform\Android\run_test_on_android_simulator.py --android-sdk-path %LY_ANDROID_SDK% --build-path %OUTPUT_DIRECTORY% --build-config %CONFIGURATION%
Tools/build/JenkinsScripts/build/Platform/Android/build_config.json
+16 -17
View File
@@ -1,11 +1,18 @@
{
"clean": {
"TAGS": [],
"COMMAND": "../Windows/clean_windows.cmd",
"PARAMETERS": {
"OUTPUT_DIRECTORY": "build",
"CMAKE_LY_PROJECTS": "AutomatedTesting;AtomTest;AtomSampleViewer"
}
},
"profile_pipe": {
"TAGS": [
"default"
],
"steps": [
"profile",
"gradle"
"profile"
]
},
"metrics": {
@@ -77,7 +84,7 @@
"CMAKE_TARGET":"AssetProcessorBatch",
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
"ASSET_PROCESSOR_BINARY": "bin\\profile\\AssetProcessorBatch.exe",
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode",
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode --regset=\"/Amazon/AssetProcessor/Settings/Exclude Android/pattern=.*/DiffuseGlobalIllumination/.*precompiledshader\"",
"ASSET_PROCESSOR_PLATFORMS":"es3"
}
},
@@ -85,21 +92,13 @@
"TAGS":[
"nightly"
],
"PIPELINE_ENV":{
"CLEAN_OUTPUT_DIRECTORY":"1"
"PIPELINE_ENV": {
"CLEAN_ASSETS": "1"
},
"COMMAND":"../Windows/build_asset_windows.cmd",
"PARAMETERS": {
"CONFIGURATION":"profile",
"OUTPUT_DIRECTORY":"build\\windows_vs2019",
"CMAKE_OPTIONS":"-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
"CMAKE_LY_PROJECTS":"AutomatedTesting",
"CMAKE_TARGET":"AssetProcessorBatch",
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
"ASSET_PROCESSOR_BINARY": "bin\\profile\\AssetProcessorBatch.exe",
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode",
"ASSET_PROCESSOR_PLATFORMS":"es3"
}
"steps": [
"clean",
"asset_profile"
]
},
"release": {
"TAGS":[
Tools/build/JenkinsScripts/build/Platform/Android/gradle_windows.cmd
-9
View File
@@ -66,15 +66,6 @@ IF NOT EXIST "%LY_ANDROID_NDK%" (
GOTO :error
)
REM Delete output directory if CLEAN_OUTPUT_DIRECTORY env variable is set
IF "%CLEAN_OUTPUT_DIRECTORY%"=="true" (
IF EXIST %OUTPUT_DIRECTORY% (
ECHO [ci_build] CLEAN_OUTPUT_DIRECTORY option set with value "%CLEAN_OUTPUT_DIRECTORY%"
ECHO [ci_build] Deleting "%OUTPUT_DIRECTORY%"
DEL /s /q /f %OUTPUT_DIRECTORY% 1>nul
)
)
IF NOT EXIST %OUTPUT_DIRECTORY% (
mkdir %OUTPUT_DIRECTORY%
)
Tools/build/JenkinsScripts/build/Platform/Linux/asset_linux.sh
-12
View File
@@ -12,18 +12,6 @@
set -o errexit # exit on the first failure encountered
# Delete output directory if CLEAN_OUTPUT_DIRECTORY env variable is set
if [[ $CLEAN_OUTPUT_DIRECTORY == "true" ]]; then
for project in $(echo $CMAKE_LY_PROJECTS | sed "s/;/ /g")
do
if [[ -d "$project/Cache" ]]; then
echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set with value \"${CLEAN_OUTPUT_DIRECTORY}\""
echo "[ci_build] Deleting \"$project/Cache\""
rm -rf $project/Cache
fi
done
fi
if [[ ! -d $OUTPUT_DIRECTORY ]]; then
echo [ci_build] Error: $OUTPUT_DIRECTORY was not found
exit 1
Tools/build/JenkinsScripts/build/Platform/Linux/build_config.json
+13 -12
View File
@@ -1,4 +1,12 @@
{
"clean": {
"TAGS": [],
"COMMAND": "clean_linux.sh",
"PARAMETERS": {
"OUTPUT_DIRECTORY": "build",
"CMAKE_LY_PROJECTS": "AutomatedTesting;AtomTest;AtomSampleViewer"
}
},
"profile_pipe": {
"TAGS": [
"default"
@@ -97,19 +105,12 @@
"nightly"
],
"PIPELINE_ENV": {
"CLEAN_OUTPUT_DIRECTORY": "1"
"CLEAN_ASSETS": "1"
},
"COMMAND": "build_asset_linux.sh",
"PARAMETERS": {
"CONFIGURATION": "profile",
"OUTPUT_DIRECTORY": "build/linux",
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
"CMAKE_LY_PROJECTS": "AutomatedTesting",
"CMAKE_TARGET": "AssetProcessorBatch",
"ASSET_PROCESSOR_BINARY": "bin/profile/AssetProcessorBatch",
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode",
"ASSET_PROCESSOR_PLATFORMS": "pc,server"
}
"steps": [
"clean",
"asset_profile"
]
},
"periodic_test_profile": {
"TAGS": [
Tools/build/JenkinsScripts/build/Platform/Linux/build_linux.sh
-9
View File
@@ -15,15 +15,6 @@ set -o errexit # exit on the first failure encountered
BASEDIR=$(dirname "$0")
source $BASEDIR/env_linux.sh
# Delete output directory if CLEAN_OUTPUT_DIRECTORY env variable is set
if [[ $CLEAN_OUTPUT_DIRECTORY == "true" ]]; then
if [[ -d $OUTPUT_DIRECTORY ]]; then
echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set with value \"${CLEAN_OUTPUT_DIRECTORY}\""
echo "[ci_build] Deleting \"${OUTPUT_DIRECTORY}\""
rm -rf ${OUTPUT_DIRECTORY}
fi
fi
mkdir -p ${OUTPUT_DIRECTORY}
SOURCE_DIRECTORY=${PWD}
pushd $OUTPUT_DIRECTORY
Tools/build/JenkinsScripts/build/Platform/Linux/clean_linux.sh
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set -o errexit # exit on the first failure encountered
if [[ -n "$CLEAN_ASSETS" ]]; then
echo "[ci_build] CLEAN_ASSETS option set"
for project in $(echo $CMAKE_LY_PROJECTS | sed "s/;/ /g")
do
if [[ -d "$project/Cache" ]]; then
echo "[ci_build] Deleting \"$project/Cache\""
rm -rf $project/Cache
fi
done
fi
if [[ -n "$CLEAN_OUTPUT_DIRECTORY" ]]; then
echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set"
if [[ -d $OUTPUT_DIRECTORY ]]; then
echo "[ci_build] Deleting \"${OUTPUT_DIRECTORY}\""
rm -rf ${OUTPUT_DIRECTORY}
fi
fi
Tools/build/JenkinsScripts/build/Platform/Mac/asset_mac.sh
-12
View File
@@ -12,18 +12,6 @@
set -o errexit # exit on the first failure encountered
# Delete output directory if CLEAN_OUTPUT_DIRECTORY env variable is set
if [[ $CLEAN_OUTPUT_DIRECTORY == "true" ]]; then
for project in $(echo $CMAKE_LY_PROJECTS | sed "s/;/ /g")
do
if [[ -d "$project/Cache" ]]; then
echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set with value \"${CLEAN_OUTPUT_DIRECTORY}\""
echo "[ci_build] Deleting \"$project/Cache\""
rm -rf $project/Cache
fi
done
fi
if [[ ! -d $OUTPUT_DIRECTORY ]]; then
echo [ci_build] Error: $OUTPUT_DIRECTORY was not found
exit 1
Tools/build/JenkinsScripts/build/Platform/Mac/build_config.json
+13 -12
View File
@@ -1,4 +1,12 @@
{
"clean": {
"TAGS": [],
"COMMAND": "clean_mac.sh",
"PARAMETERS": {
"OUTPUT_DIRECTORY": "build",
"CMAKE_LY_PROJECTS": "AutomatedTesting;AtomTest;AtomSampleViewer"
}
},
"profile_pipe": {
"TAGS": [
"nightly"
@@ -81,19 +89,12 @@
"nightly"
],
"PIPELINE_ENV": {
"CLEAN_OUTPUT_DIRECTORY": "1"
"CLEAN_ASSETS": "1"
},
"COMMAND": "build_asset_mac.sh",
"PARAMETERS": {
"CONFIGURATION": "profile",
"OUTPUT_DIRECTORY": "build/mac",
"CMAKE_OPTIONS": "-G Xcode -DLY_UNITY_BUILD=TRUE",
"CMAKE_LY_PROJECTS": "AutomatedTesting",
"CMAKE_TARGET": "AssetProcessorBatch",
"ASSET_PROCESSOR_BINARY": "bin/profile/AssetProcessorBatch",
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode",
"ASSET_PROCESSOR_PLATFORMS": "osx_gl"
}
"steps": [
"clean",
"asset_profile"
]
},
"periodic_test_profile": {
"TAGS": [
Tools/build/JenkinsScripts/build/Platform/Mac/build_mac.sh
+2 -11
View File
@@ -15,15 +15,6 @@ set -o errexit # exit on the first failure encountered
BASEDIR=$(dirname "$0")
source $BASEDIR/env_mac.sh
# Delete output directory if CLEAN_OUTPUT_DIRECTORY env variable is set
if [[ $CLEAN_OUTPUT_DIRECTORY == "true" ]]; then
if [[ -d $OUTPUT_DIRECTORY ]]; then
echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set with value \"${CLEAN_OUTPUT_DIRECTORY}\""
echo "[ci_build] Deleting \"${OUTPUT_DIRECTORY}\""
rm -rf ${OUTPUT_DIRECTORY}
fi
fi
mkdir -p ${OUTPUT_DIRECTORY}
SOURCE_DIRECTORY=${PWD}
pushd $OUTPUT_DIRECTORY
@@ -52,7 +43,7 @@ if [[ ! -z "$RUN_CONFIGURE" ]]; then
echo "${CONFIGURE_CMD}" > ${LAST_CONFIGURE_CMD_FILE}
fi
echo [ci_build] cmake --build . --target ${CMAKE_TARGET} --config ${CONFIGURATION} -j $(sysctl -n hw.ncpu) -- ${CMAKE_NATIVE_BUILD_ARGS} -UseModernBuildSystem=NO
cmake --build . --target ${CMAKE_TARGET} --config ${CONFIGURATION} -j $(sysctl -n hw.ncpu) -- ${CMAKE_NATIVE_BUILD_ARGS} -UseModernBuildSystem=NO
echo [ci_build] cmake --build . --target ${CMAKE_TARGET} --config ${CONFIGURATION} -j $(sysctl -n hw.ncpu) -- ${CMAKE_NATIVE_BUILD_ARGS}
cmake --build . --target ${CMAKE_TARGET} --config ${CONFIGURATION} -j $(sysctl -n hw.ncpu) -- ${CMAKE_NATIVE_BUILD_ARGS}
popd
Tools/build/JenkinsScripts/build/Platform/Mac/clean_mac.sh
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set -o errexit # exit on the first failure encountered
if [[ -n "$CLEAN_ASSETS" ]]; then
echo "[ci_build] CLEAN_ASSETS option set"
for project in $(echo $CMAKE_LY_PROJECTS | sed "s/;/ /g")
do
if [[ -d "$project/Cache" ]]; then
echo "[ci_build] Deleting \"$project/Cache\""
rm -rf $project/Cache
fi
done
fi
if [[ -n "$CLEAN_OUTPUT_DIRECTORY" ]]; then
echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set"
if [[ -d $OUTPUT_DIRECTORY ]]; then
echo "[ci_build] Deleting \"${OUTPUT_DIRECTORY}\""
rm -rf ${OUTPUT_DIRECTORY}
fi
fi
Tools/build/JenkinsScripts/build/Platform/Windows/asset_windows.cmd
-11
View File
@@ -12,17 +12,6 @@ REM
SETLOCAL EnableDelayedExpansion
REM Delete output directory if CLEAN_OUTPUT_DIRECTORY env variable is set
IF "%CLEAN_OUTPUT_DIRECTORY%"=="true" (
FOR %%P in (%CMAKE_LY_PROJECTS%) do (
IF EXIST %%P\Cache (
ECHO [ci_build] CLEAN_OUTPUT_DIRECTORY option set with value "%CLEAN_OUTPUT_DIRECTORY%"
ECHO [ci_build] Deleting "%%P\Cache"
DEL /s /q /f %%P\Cache 1>nul
)
)
)
IF NOT EXIST %OUTPUT_DIRECTORY% (
ECHO [ci_build] Error: %OUTPUT_DIRECTORY% was not found
GOTO :error
Tools/build/JenkinsScripts/build/Platform/Windows/build_config.json
+14 -13
View File
@@ -1,9 +1,18 @@
{
"clean": {
"TAGS": [],
"COMMAND": "clean_windows.cmd",
"PARAMETERS": {
"OUTPUT_DIRECTORY": "build",
"CMAKE_LY_PROJECTS": "AutomatedTesting"
}
},
"validation_pipe": {
"TAGS": [
"default"
],
"steps": [
"scrubbing",
"validation"
]
},
@@ -186,20 +195,12 @@
"nightly"
],
"PIPELINE_ENV": {
"CLEAN_OUTPUT_DIRECTORY": "1"
"CLEAN_ASSETS": "1"
},
"COMMAND": "build_asset_windows.cmd",
"PARAMETERS": {
"CONFIGURATION": "profile",
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
"CMAKE_LY_PROJECTS": "AutomatedTesting",
"CMAKE_TARGET": "AssetProcessorBatch",
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
"ASSET_PROCESSOR_BINARY": "bin\\profile\\AssetProcessorBatch.exe",
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode",
"ASSET_PROCESSOR_PLATFORMS": "pc,server"
}
"steps": [
"clean",
"asset_profile_vs2019"
]
},
"periodic_test_profile_vs2019": {
"TAGS": [
Tools/build/JenkinsScripts/build/Platform/Windows/build_windows.cmd
-9
View File
@@ -14,15 +14,6 @@ SETLOCAL EnableDelayedExpansion
CALL %~dp0env_windows.cmd
REM Delete output directory if CLEAN_OUTPUT_DIRECTORY env variable is set
IF "%CLEAN_OUTPUT_DIRECTORY%"=="true" (
IF EXIST %OUTPUT_DIRECTORY% (
ECHO [ci_build] CLEAN_OUTPUT_DIRECTORY option set with value "%CLEAN_OUTPUT_DIRECTORY%"
ECHO [ci_build] Deleting "%OUTPUT_DIRECTORY%"
DEL /s /q /f %OUTPUT_DIRECTORY% 1>nul
)
)
IF NOT EXIST "%OUTPUT_DIRECTORY%" (
MKDIR %OUTPUT_DIRECTORY%.
)
Tools/build/JenkinsScripts/build/Platform/Windows/clean_windows.cmd
+31
View File
@@ -0,0 +1,31 @@
@ECHO OFF
REM
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
REM its licensors.
REM
REM For complete copyright and license terms please see the LICENSE at the root of this
REM distribution (the "License"). All use of this software is governed by the License,
REM or, if provided, by the license below or the license accompanying this file. Do not
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
REM
SETLOCAL EnableDelayedExpansion
IF DEFINED CLEAN_ASSETS (
ECHO [ci_build] CLEAN_ASSETS option set
FOR %%P in (%CMAKE_LY_PROJECTS%) do (
IF EXIST %%P\Cache (
ECHO [ci_build] Deleting "%%P\Cache"
DEL /s /q /f %%P\Cache 1>nul
)
)
)
IF DEFINED CLEAN_OUTPUT_DIRECTORY (
ECHO [ci_build] CLEAN_OUTPUT_DIRECTORY option set
IF EXIST %OUTPUT_DIRECTORY% (
ECHO [ci_build] Deleting "%OUTPUT_DIRECTORY%"
DEL /s /q /f %OUTPUT_DIRECTORY% 1>nul
)
)
Tools/build/JenkinsScripts/build/Platform/iOS/build_config.json
+13 -12
View File
@@ -1,4 +1,12 @@
{
"clean": {
"TAGS": [],
"COMMAND": "../Mac/clean_mac.sh",
"PARAMETERS": {
"OUTPUT_DIRECTORY": "build",
"CMAKE_LY_PROJECTS": "AutomatedTesting;AtomTest;AtomSampleViewer"
}
},
"metrics": {
"TAGS": [
"weekly"
@@ -77,19 +85,12 @@
"nightly"
],
"PIPELINE_ENV": {
"CLEAN_OUTPUT_DIRECTORY": "1"
"CLEAN_ASSETS": "true"
},
"COMMAND": "../Mac/build_asset_mac.sh",
"PARAMETERS": {
"CONFIGURATION": "profile",
"OUTPUT_DIRECTORY": "build/mac",
"CMAKE_OPTIONS": "-G Xcode -DLY_UNITY_BUILD=TRUE",
"CMAKE_LY_PROJECTS": "AutomatedTesting",
"CMAKE_TARGET": "AssetProcessorBatch",
"ASSET_PROCESSOR_BINARY": "bin/profile/AssetProcessorBatch",
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode",
"ASSET_PROCESSOR_PLATFORMS": "ios"
}
"steps": [
"clean",
"asset_profile"
]
},
"release": {
"TAGS": [