birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into LoadPipelineFromGitHub

Browse Source
This commit is contained in:
Brian Herrera
2021-03-26 17:05:57 -07:00
parent e8bbb5a0d5 cfd7789a49
commit 39f53338bf
765 changed files with 11513 additions and 20246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmake/Tools/layout_tool.py
+185 -201
View File
@@ -11,14 +11,13 @@
import argparse
import datetime
import hashlib
import logging
import os
import pathlib
import platform
import re
import shutil
import stat
import subprocess
import sys
import tempfile
@@ -26,9 +25,9 @@ import timeit
# Resolve the common python module
ROOT_DEV_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))
if ROOT_DEV_PATH not in sys.path:
sys.path.append(ROOT_DEV_PATH)
ROOT_ENGINE_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))
if ROOT_ENGINE_PATH not in sys.path:
sys.path.append(ROOT_ENGINE_PATH)
from cmake.Tools import common
@@ -51,21 +50,20 @@ PAK_ONLY_BUILD_CONFIGS = ['RELEASE']
PLATFORM_NAME = platform.system()
# List of files to blacklist from copying to the layout folder
COPY_ASSET_FILE_GENERAL_BLACKLIST_FILES = [
# List of files to deny from copying to the layout folder
COPY_ASSET_FILE_GENERAL_DENYLIST_FILES = [
'aztest_bootstrap.json',
'editor.cfg',
'assetprocessorplatformconfig.ini',
'assetprocessorplatformconfig.setreg',
]
def verify_layout(layout_dir, platform_name, game_name, asset_mode, asset_type):
def verify_layout(layout_dir, platform_name, project_path, asset_mode, asset_type):
"""
Verify a layout folder (WRT to assets and configs) against the bootstrap and system config files
@param layout_dir: The layout path to validate the asset mode against the bootstrap and system configs
@param platform_name: The name of the platform the deployment is for
@param game_name: The game (project) name being deployed
@param project_path: The path to the project being deployed
@param asset_mode: The desired asset mode (PAK, LOOSE, VFS)
@param asset_type: The asset type
@return: The number of possible errors in the configuration files based on the asset mode and type
@@ -98,8 +96,15 @@ def verify_layout(layout_dir, platform_name, game_name, asset_mode, asset_type):
warning_count = 0
# Look up the project_path from the project.json file
project_name = common.read_project_name_from_project_json(project_path)
# If the project-name could not be read from the project.json, then the supplied project path does not
# point to a valid project
if not project_name:
return 1
platform_name_lower = platform_name.lower()
game_name_lower = game_name.lower()
project_name_lower = project_path.lower()
layout_path = pathlib.Path(layout_dir)
# Validate bootstrap.cfg exists
@@ -108,9 +113,7 @@ def verify_layout(layout_dir, platform_name, game_name, asset_mode, asset_type):
warning_count += _warn(f"'bootstrap.cfg' is missing from {str(layout_path)}")
bootstrap_values = None
else:
bootstrap_values = common.get_config_file_values(str(bootstrap_file), ['sys_game_folder',
'sys_game_name',
f'{platform_name_lower}_remote_filesystem',
bootstrap_values = common.get_config_file_values(str(bootstrap_file), [f'{platform_name_lower}_remote_filesystem',
f'{platform_name_lower}_connect_to_remote',
f'{platform_name_lower}_wait_for_connect',
f'{platform_name_lower}_assets',
@@ -120,9 +123,9 @@ def verify_layout(layout_dir, platform_name, game_name, asset_mode, asset_type):
])
# Validate the system_{platform}_{asset type}.cfg exists
platform_system_cfg_file = layout_path / f'system_{platform_name}_{asset_type}.cfg'
platform_system_cfg_file = layout_path / f'system_{platform_name_lower}_{asset_type}.cfg'
if not platform_system_cfg_file.is_file():
warning_count += _warn(f"'system_{platform_name}_{asset_type}.cfg' is missing from {str(layout_path)}")
warning_count += _warn(f"'system_{platform_name_lower}_{asset_type}.cfg' is missing from {str(layout_path)}")
system_config_values = None
else:
system_config_values = common.get_config_file_values(str(platform_system_cfg_file), ['r_ShadersRemoteCompiler',
@@ -133,14 +136,7 @@ def verify_layout(layout_dir, platform_name, game_name, asset_mode, asset_type):
if bootstrap_values:
remote_ip = bootstrap_values.get(f'{platform_name_lower}_remote_ip') or bootstrap_values.get('remote_ip') or LOCAL_HOST
remote_connect = bootstrap_values.get(f'{platform_name}_connect_to_remote') or '0'
# Validate that the game name matches in bootstrap.cfg
bootstrap_game = bootstrap_values.get('sys_game_folder') or bootstrap_values.get('sys_game_name')
if not bootstrap_game:
warning_count += _warn("'bootstrap.cfg' is missing the game name set in 'sys_game_folder")
elif bootstrap_game != game_name:
warning_count += _warn(f"The game specified in bootstrap.cfg ({bootstrap_game}) does not match the game name specified for this deployment ({game_name})")
remote_connect = bootstrap_values.get(f'{platform_name_lower}_connect_to_remote') or '0'
# Validate that the asset type for the platform matches the one set for the build
bootstrap_asset_type = bootstrap_values.get(f'{platform_name_lower}_assets') or bootstrap_values.get('assets')
@@ -152,9 +148,9 @@ def verify_layout(layout_dir, platform_name, game_name, asset_mode, asset_type):
# Validate that if '<platform>_connect_to_remote is enabled, that the 'remote_ip' is not set to local host
warning_count += _validate_remote_ap(remote_ip, remote_connect, None)
game_asset_path = layout_path / game_name_lower
if not game_asset_path.is_dir():
warning_count += _warn(f"Asset folder for game {game_name} is missing from the deployment layout.")
project_asset_path = layout_path / project_name_lower
if not project_asset_path.is_dir():
warning_count += _warn(f"Asset folder for project {project_name} is missing from the deployment layout.")
elif system_config_values is not None:
@@ -183,9 +179,9 @@ def verify_layout(layout_dir, platform_name, game_name, asset_mode, asset_type):
# Validate that we have pak files
pak_count = 0
has_shader_pak = False
game_paks = game_asset_path.glob("*.pak")
for game_pak in game_paks:
if game_pak.name == 'shadercachestartup.pak':
project_paks = project_asset_path.glob("*.pak")
for project_pak in project_paks:
if project_pak.name == 'shadercachestartup.pak':
has_shader_pak = True
pak_count += 1
@@ -196,7 +192,7 @@ def verify_layout(layout_dir, platform_name, game_name, asset_mode, asset_type):
# If the shader paks are set, make sure that the remote shader compiler connection settings are set
# or that it is going through AP
if shaders_remote_compiler == '1':
warning_count += _warn(f"Shader paks are set for game {game_name} but remote shader compiling "
warning_count += _warn(f"Shader paks are set for project {project_name} but remote shader compiling "
f"(r_ShadersRemoteCompiler) is still enabled "
f"for it in system_{platform_name_lower}_{asset_type}.cfg.")
else:
@@ -205,7 +201,7 @@ def verify_layout(layout_dir, platform_name, game_name, asset_mode, asset_type):
warning_count += _validate_remote_ap(remote_ip, remote_connect, False)
if shaders_allow_compilation is not None and shaders_allow_compilation == '1':
warning_count += _warn(f"Shader paks are set for game {game_name} but shader compiling "
warning_count += _warn(f"Shader paks are set for project {project_name} but shader compiling "
f"(r_ShadersAllowCompilation) is still enabled "
f"for it in system_{platform_name_lower}_{asset_type}.cfg.")
@@ -226,26 +222,23 @@ def verify_layout(layout_dir, platform_name, game_name, asset_mode, asset_type):
return warning_count
def copy_asset_files_to_layout(game_name, game_asset_folder, target_platform, layout_target):
def copy_asset_files_to_layout(project_asset_folder, target_platform, layout_target):
"""
Perform the specific rules for copying files to the root level of the layout.
:param game_name: The name of the game
:param game_asset_folder: The source game asset folder to copy the files. (Will not traverse deeper than this folder)
:param target_platform: The target platform of the layout
:param layout_target: The target path of the target layout folder.
:param project_asset_folder: The source project asset folder to copy the files. (Will not traverse deeper than this folder)
:param target_platform: The target platform of the layout
:param layout_target: The target path of the target layout folder.
"""
src_asset_contents = os.listdir(game_asset_folder)
src_asset_contents = os.listdir(project_asset_folder)
allowed_system_config_prefix = 'system_{}'.format(target_platform.lower())
for src_file in src_asset_contents:
# For each source file found in the root of the source game asset folder, apply various rules to determine
# For each source file found in the root of the source project asset folder, apply various rules to determine
# if we will copy the file to the layout destination or not
if src_file in COPY_ASSET_FILE_GENERAL_BLACKLIST_FILES:
# The source file is black-listed from being copied
if src_file in COPY_ASSET_FILE_GENERAL_DENYLIST_FILES:
# The source file is denied from being copied
continue
if src_file.startswith('system_'):
@@ -255,7 +248,7 @@ def copy_asset_files_to_layout(game_name, game_asset_folder, target_platform, la
continue
# Resolve the absolute paths for source and destination to perform more specific checks
abs_src = os.path.join(game_asset_folder, src_file)
abs_src = os.path.join(project_asset_folder, src_file)
abs_dst = os.path.join(layout_target, src_file)
if os.path.isdir(abs_src):
@@ -280,35 +273,33 @@ def copy_asset_files_to_layout(game_name, game_asset_folder, target_platform, la
src_file,
src_hash)
continue
if os.path.basename(abs_src) == 'bootstrap.cfg':
logging.debug("Copying (%s) %s -> %s", game_name, abs_src, abs_dst)
common.transform_bootstrap_for_game(game_name, abs_src, abs_dst)
else:
logging.debug("Copying %s -> %s", abs_src, abs_dst)
shutil.copy2(abs_src, abs_dst)
def remove_link(link):
def remove_link(link:pathlib.PurePath):
"""
Helper function to either remove a symlink, or remove a folder
"""
link = pathlib.PurePath(link)
if os.path.isdir(link):
try:
os.unlink(link)
except:
if PLATFORM_NAME == 'Windows':
rmdir_cmd = ['cmd', '/c', 'rmdir', '/J', '/S', link]
else:
rmdir_cmd = ['rm', '-rf', link]
except OSError:
# If unlink fails use shutil.rmtree
def remove_readonly(func, path, _):
"Clear the readonly bit and reattempt the removal"
os.chmod(path, stat.S_IWRITE)
func(path)
try:
logging.debug('Executing call %s', ' '.join(rmdir_cmd))
subprocess.check_call(rmdir_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError as e:
raise common.LmbrCmdError("Error trying remove directory {}: {}".format(link, e),
e.returncode)
shutil.rmtree(link, onerror=remove_readonly)
except shutil.Error as shutil_error:
raise common.LmbrCmdError(f'Error trying remove directory {link}: {shutil_error}', shutil_error.errno)
def create_link(src, tgt, copy):
def create_link(src:pathlib.Path, tgt:pathlib.Path, copy):
"""
Helper function to create a directory link or copy a directory. On windows, this will be a directory junction, and on mac/linux
this will be a soft link
@@ -317,68 +308,74 @@ def create_link(src, tgt, copy):
:param tgt: The target of the new link
:param copy: Perform a directory copy instead of a link
"""
src = pathlib.Path(src)
tgt = pathlib.Path(tgt)
if copy:
if os.path.isdir(tgt):
os.rmdir(tgt)
logging.debug("Copying from %s to %s", src, tgt)
shutil.copytree(src, tgt, symlinks=False)
else:
logging.debug('Creating internal junction %s => %s in %s', src, tgt)
if PLATFORM_NAME == 'Windows':
link_type = 'junction'
junction_cmd = ['cmd', '/c', 'mklink', '/J', tgt, src]
# Remove the exist target
if tgt.is_symlink():
tgt.unlink()
else:
link_type = 'soft link'
junction_cmd = ['ln', '-s', src, tgt]
def remove_readonly(func, path, _):
"Clear the readonly bit and reattempt the removal"
os.chmod(path, stat.S_IWRITE)
func(path)
shutil.rmtree(tgt, onerror=remove_readonly)
logging.debug(f'Copying from {src} to {tgt}')
shutil.copytree(str(src), str(tgt), symlinks=False)
else:
link_type = "symlink"
logging.debug(f'Creating symlink {src} =>{tgt}')
try:
logging.debug('Executing call %s', ' '.join(junction_cmd))
subprocess.check_call(junction_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError as e:
raise common.LmbrCmdError(f"Error trying to create {link_type} {src} => {tgt} : {str(e)}", e.returncode)
if PLATFORM_NAME == "Windows":
link_type = "junction"
import _winapi
_winapi.CreateJunction(str(src), str(tgt))
else:
src.symlink_to(tgt, target_is_directory=True)
except OSError as e:
raise common.LmbrCmdError(f"Error trying to create {link_type} {src} => {tgt} : {e}", e.errno)
def construct_and_validate_cache_game_asset_folder(dev_root, game_name, asset_type, warn_on_missing_game_cache):
def construct_and_validate_cache_project_asset_folder(project_path, asset_type, warn_on_missing_project_cache):
"""
Given the parameters for a game (name, dev root, asset type), construct and validate the absolute path
Given the parameters for a project (project_path, asset type), construct and validate the absolute path
of where the built assets are (for LOOSE and VFS modes)
:param dev_root: The dev root to base the Cache folder search
:param game_name: The name of the game
:param asset_type: The type of asset
:param warn_on_missing_game_cache: Option to warn if the path is missing vs raising an exception
:return: The validated constructed cache game asset folder if it exists, None if not
:param project_path: The path to the project
:param asset_type: The type of asset
:param warn_on_missing_project_cache: Option to warn if the path is missing vs raising an exception
:return: The validated constructed cache project asset folder if it exists, None if not
"""
# Locate the Cache root folder
cache_game_folder_root = os.path.join(dev_root, CACHE_FOLDER_NAME, game_name)
if not os.path.isdir(cache_game_folder_root) and not warn_on_missing_game_cache:
cache_project_folder_root = os.path.join(project_path, CACHE_FOLDER_NAME)
if not os.path.isdir(cache_project_folder_root) and not warn_on_missing_project_cache:
raise common.LmbrCmdError(
"Missing Cache folder for the game in the current dev root. Make sure that assets have been built "
"for the game '{}'".format(game_name),
f"Missing Cache folder for the project at path {project_path}. Make sure that assets have been built ",
common.ERROR_CODE_ERROR_DIRECTORY)
# Locate based on the game's built asset type
cache_game_asset_folder = os.path.join(cache_game_folder_root, asset_type)
if os.path.isdir(cache_game_asset_folder):
# Locate based on the project's built asset type
cache_project_asset_folder = os.path.join(cache_project_folder_root, asset_type)
if os.path.isdir(cache_project_asset_folder):
# TODO: Note, this is only checking the existence of the folder, not for any content validation
return cache_game_asset_folder
return cache_project_asset_folder
# Expected source of the game assets was not found
if not warn_on_missing_game_cache:
# Expected source of the project assets was not found
if not warn_on_missing_project_cache:
raise common.LmbrCmdError(
"Missing compiled assets folder for the game. Make sure that assets for '{}' have been built "
"for the game '{}'".format(asset_type, game_name),
f'Missing compiled assets folder for the project at path {project_path}."'
f' Make sure that assets for "{asset_type}" have been built',
common.ERROR_CODE_ERROR_DIRECTORY)
return None
def sync_layout_vfs(dev_root, target_platform, game, asset_type, warning_on_missing_assets, layout_target, override_pak_folder, copy):
def sync_layout_vfs(target_platform, project_path, asset_type, warning_on_missing_assets, layout_target, override_pak_folder, copy):
"""
Perform the logic to sync the layout folder with assets in VFS mode
:param dev_root: The configured dev root
:param target_platform: The target platform the layout is based on
:param game: The name of the game being synced
:param project_path: The path to the project being synced
:param asset_type: The asset type being synced
:param warning_on_missing_assets: If the built assets cannot be located (LOOSE or PAKs), then optionally warn vs raising an error
:param layout_target: The target layout folder to perform the sync on
@@ -386,71 +383,68 @@ def sync_layout_vfs(dev_root, target_platform, game, asset_type, warning_on_miss
:param copy: Option to copy instead of attempting to symlink/junction
"""
logging.debug("Syncing VFS layout for game '%s' to layout path '%s'", game, layout_target)
logging.debug(f'Syncing VFS layout for project at path "{project_path}" to layout path "{layout_target}"')
game_asset_folder = construct_and_validate_cache_game_asset_folder(dev_root=dev_root,
game_name=game,
project_asset_folder = construct_and_validate_cache_project_asset_folder(project_path=project_path,
asset_type=asset_type,
warn_on_missing_game_cache=warning_on_missing_assets)
warn_on_missing_project_cache=warning_on_missing_assets)
game_folder = game.lower()
vfs_asset_source = os.path.join(game_asset_folder, game_folder, 'config')
vfs_asset_source = os.path.join(project_asset_folder, 'config')
if not os.path.isdir(vfs_asset_source):
raise common.LmbrCmdError("Cache folder for the game '{}' missing 'config' folder".format(game),
raise common.LmbrCmdError("Cache folder for the project '{}' missing 'config' folder".format(project_path),
common.ERROR_CODE_ERROR_DIRECTORY)
# create a temporary folder that will serve as a working junction point into the layout
hasher = hashlib.md5()
hasher.update(dev_root.encode('UTF-8'))
hasher.update(game_folder.encode('UTF-8'))
hasher.update(project_path.encode('UTF-8'))
result = hasher.hexdigest()
temp_dir = tempfile.gettempdir()
temp_vfs_layout_path = os.path.join(temp_dir, 'ly-layout-{}'.format(result), 'vfs')
temp_vfs_layout_game_path = os.path.join(temp_vfs_layout_path, game_folder)
temp_vfs_layout_project_path = temp_vfs_layout_path
temp_vfs_layout_game_config_path = os.path.join(temp_vfs_layout_game_path, 'config')
temp_vfs_layout_project_config_path = os.path.join(temp_vfs_layout_project_path, 'config')
# If the temporary folder was created previously, always reset it
if os.path.isdir(temp_vfs_layout_game_path):
if os.path.isdir(temp_vfs_layout_game_config_path):
os.rmdir(temp_vfs_layout_game_config_path)
shutil.rmtree(temp_vfs_layout_game_path)
os.makedirs(temp_vfs_layout_game_path, exist_ok=True)
if os.path.isdir(temp_vfs_layout_project_path):
if os.path.isdir(temp_vfs_layout_project_config_path):
os.rmdir(temp_vfs_layout_project_config_path)
shutil.rmtree(temp_vfs_layout_project_path)
os.makedirs(temp_vfs_layout_project_path, exist_ok=True)
# Create the 'project asset platform cache' junction before copying configuration files at the engine root to it
layout_project_folder_target = layout_target
# Remove previous layout folder if it is a directory
if os.path.isdir(layout_project_folder_target):
remove_link(layout_project_folder_target)
if os.path.isdir(temp_vfs_layout_project_path):
create_link(temp_vfs_layout_project_path, layout_project_folder_target, copy)
# Create the link
create_link(vfs_asset_source, temp_vfs_layout_game_config_path, copy)
create_link(vfs_asset_source, temp_vfs_layout_project_config_path, copy)
# Create the assets to the layout
copy_asset_files_to_layout(game_name=game,
game_asset_folder=game_asset_folder,
copy_asset_files_to_layout(project_asset_folder=project_asset_folder,
target_platform=target_platform,
layout_target=layout_target)
# Reset the 'gems' junction if any in the layout
layout_gems_folder_src = os.path.join(game_asset_folder, 'gems')
layout_gems_folder_src = os.path.join(project_asset_folder, 'gems')
layout_gems_folder_target = os.path.join(layout_target, 'gems')
if os.path.isdir(layout_gems_folder_target):
remove_link(layout_gems_folder_target)
if os.path.isdir(layout_gems_folder_src):
create_link(layout_gems_folder_src, layout_gems_folder_target, copy)
# Reset the <game_folder> junction
layout_game_folder_target = os.path.join(layout_target, game_folder)
if os.path.isdir(layout_game_folder_target):
remove_link(layout_game_folder_target)
if os.path.isdir(temp_vfs_layout_game_path):
create_link(temp_vfs_layout_game_path, layout_game_folder_target, copy)
def sync_layout_non_vfs(mode, target_platform, dev_root, game, asset_type, warning_on_missing_assets, layout_target, override_pak_folder, copy):
def sync_layout_non_vfs(mode, target_platform, project_path, asset_type, warning_on_missing_assets, layout_target, override_pak_folder, copy):
"""
Perform the logic to sync the layout folder with assets in non-VFS mode (LOOSE or PAK)
:param mode: 'LOOSE' or 'PAK' mode
:param target_platform: The target platform the layout is based on
:param dev_root: The configured dev root
:param game: The name of the game being synced
:param project_path: The path to the project being synced
:param asset_type: The asset type being synced
:param warning_on_missing_assets: If the built assets cannot be located (LOOSE or PAKs), then optionally warn vs raising an error
:param layout_target: The target layout folder to perform the sync on
@@ -460,66 +454,67 @@ def sync_layout_non_vfs(mode, target_platform, dev_root, game, asset_type, warni
assert mode in (ASSET_MODE_PAK, ASSET_MODE_LOOSE)
game_folder = game.lower()
project_name = common.read_project_name_from_project_json(project_path)
if not project_name:
raise common.LmbrCmdError(f'Project at path {project_path} does not have a valid project.json')
project_name_lower = project_name.lower()
layout_gems_folder_target = os.path.join(layout_target, 'gems')
if os.path.isdir(layout_gems_folder_target):
remove_link(layout_gems_folder_target)
layout_game_folder_target = os.path.join(layout_target, game_folder)
if os.path.isdir(layout_game_folder_target):
remove_link(layout_game_folder_target)
if mode == ASSET_MODE_PAK:
target_pak_folder_name = '{}_{}_paks'.format(game_folder, asset_type)
game_asset_folder = os.path.join(dev_root, override_pak_folder or PAK_FOLDER_NAME, target_pak_folder_name)
if not os.path.isdir(game_asset_folder):
target_pak_folder_name = '{}_{}_paks'.format(project_name_lower, asset_type)
project_asset_folder = os.path.join(project_path, override_pak_folder or PAK_FOLDER_NAME, target_pak_folder_name)
if not os.path.isdir(project_asset_folder):
if warning_on_missing_assets:
logging.warning("Pak folder for the game '{}' is missing (expected at '{}'). Skipping layout sync".format(game, game_asset_folder))
logging.warning(f'Pak folder for the project at path "{project_path}" is missing'
f' (expected at "{project_asset_folder}"). Skipping layout sync')
return
else:
raise common.LmbrCmdError("Pak folder for the game '{}' is missing (expected at '{}')".format(game, game_asset_folder),
raise common.LmbrCmdError(f'Pak folder for the project at path "{project_path}" is missing (expected at'
f' "{project_asset_folder}")',
common.ERROR_CODE_ERROR_DIRECTORY)
elif mode == ASSET_MODE_LOOSE:
game_asset_folder = construct_and_validate_cache_game_asset_folder(dev_root=dev_root,
game_name=game,
project_asset_folder = construct_and_validate_cache_project_asset_folder(project_path=project_path,
asset_type=asset_type,
warn_on_missing_game_cache=warning_on_missing_assets)
if not game_asset_folder:
warn_on_missing_project_cache=warning_on_missing_assets)
if not project_asset_folder:
logging.warning(
"Cannot locate built assets for game '{}' (expected at '{}'). Skipping layout sync".format(game,
game_asset_folder))
f'Cannot locate built assets for project at path "{project_path}" (expected at "{project_asset_folder}").'
f' Skipping layout sync')
return
else:
assert False, "Invalid Mode {}".format(mode)
# Create the 'project asset platform cache' junction before copying additional files to it
layout_project_folder_src = project_asset_folder
# Remove previous layout folder if it is a directory
if os.path.isdir(layout_target):
remove_link(layout_target)
if os.path.isdir(layout_project_folder_src):
create_link(layout_project_folder_src, layout_target, copy)
# Create the assets to the layout
copy_asset_files_to_layout(game_name=game,
game_asset_folder=game_asset_folder,
copy_asset_files_to_layout(project_asset_folder=project_asset_folder,
target_platform=target_platform,
layout_target=layout_target)
# Reset the 'gems' junction if any in the layout (only in loose mode).
layout_gems_folder_src = os.path.join(game_asset_folder, 'gems')
layout_gems_folder_src = os.path.join(project_asset_folder, 'gems')
# The gems link only is valid in LOOSE mode. If in PAK, then dont re-link
if mode == ASSET_MODE_LOOSE and os.path.isdir(layout_gems_folder_src):
if os.path.isdir(layout_gems_folder_src):
create_link(layout_gems_folder_src, layout_gems_folder_target, copy)
# Reset the <game_folder> junction
layout_game_folder_src = os.path.join(game_asset_folder, game_folder)
if os.path.isdir(layout_game_folder_src):
create_link(layout_game_folder_src, layout_game_folder_target, copy)
def sync_layout_pak(dev_root, target_platform, game, asset_type, warning_on_missing_assets, layout_target,
def sync_layout_pak(target_platform, project_path, asset_type, warning_on_missing_assets, layout_target,
override_pak_folder, copy):
sync_layout_non_vfs(mode=ASSET_MODE_PAK,
target_platform=target_platform,
dev_root=dev_root,
game=game,
project_path=project_path,
asset_type=asset_type,
warning_on_missing_assets=warning_on_missing_assets,
layout_target=layout_target,
@@ -527,12 +522,11 @@ def sync_layout_pak(dev_root, target_platform, game, asset_type, warning_on_miss
copy=copy)
def sync_layout_loose(dev_root, target_platform, game, asset_type, warning_on_missing_assets, layout_target,
def sync_layout_loose(target_platform, project_path, asset_type, warning_on_missing_assets, layout_target,
override_pak_folder, copy):
sync_layout_non_vfs(mode=ASSET_MODE_LOOSE,
target_platform=target_platform,
dev_root=dev_root,
game=game,
project_path=project_path,
asset_type=asset_type,
warning_on_missing_assets=warning_on_missing_assets,
layout_target=layout_target,
@@ -548,13 +542,10 @@ ASSET_SYNC_MODE_FUNCTION = {
def main(args):
parser = argparse.ArgumentParser(description="Synchronize a game's assets to a layout folder")
parser = argparse.ArgumentParser(description="Synchronize a project's assets to a layout folder")
parser.add_argument('--dev-root',
help='The path to the dev root',
required=True)
parser.add_argument('-g', '--game',
help='Name of the game whose assets we will sync.',
parser.add_argument('--project-path',
help='The project path whose assets we will sync.',
required=True)
parser.add_argument('-p', '--platform',
help='Target platform for the layout.',
@@ -567,7 +558,7 @@ def main(args):
help='Enable debug logs.')
parser.add_argument('--warn-on-missing-assets',
action='store_true',
help='If the game does not have any built assets, warn rather than return an error')
help='If the project does not have any built assets, warn rather than return an error')
parser.add_argument('-m', '--mode',
type=str,
choices=ALL_ASSET_MODES,
@@ -582,7 +573,7 @@ def main(args):
parser.add_argument('--override-pak-folder',
default='',
help='(optional) If provided, use this path as the path to the pak folder when creating layouts '
'in PAK mode. Otherwise, use the dev-root/pak/${game}_${asset_type}_pak as the source pak folder')
'in PAK mode. Otherwise, use the {project_path}/pak/${project}_${asset_type}_pak as the source pak folder')
parser.add_argument('--build-config',
default='',
help='(optional) If provided, will adjust the asset mode if the provided build-config is "release"')
@@ -599,11 +590,36 @@ def main(args):
parsed_args = parser.parse_args(args)
# Validate the dev_root exists
if not os.path.exists(parsed_args.dev_root):
raise common.LmbrCmdError("Invalid dev root folder. '{}' does not exist".format(parsed_args.dev_root),
# Prepare the logging
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG if parsed_args.debug else logging.INFO)
# Validate the asset mode
input_asset_mode = parsed_args.mode.upper()
if input_asset_mode not in ALL_ASSET_MODES:
raise common.LmbrCmdError("Invalid asset mode '{}'. Must be one of : '{}'.".format(input_asset_mode, ','.join(ALL_ASSET_MODES)),
common.ERROR_CODE_INVALID_PARAMETER)
# Check if the build config is set, if so, check if its release
build_config = parsed_args.build_config.upper()
if build_config in PAK_ONLY_BUILD_CONFIGS:
input_asset_mode = ASSET_MODE_PAK
logging.info("Starting (%s) Asset Synchronization in %s mode and project %s", parsed_args.asset_type, input_asset_mode, parsed_args.project_path)
start_time = timeit.default_timer()
ASSET_SYNC_MODE_FUNCTION[input_asset_mode](target_platform=parsed_args.platform,
project_path=parsed_args.project_path,
asset_type=parsed_args.asset_type,
warning_on_missing_assets=parsed_args.warn_on_missing_assets,
layout_target=os.path.normpath(parsed_args.layout_root),
override_pak_folder=parsed_args.override_pak_folder,
copy=parsed_args.copy)
duration = timeit.default_timer() - start_time
logging.info("Asset Synchronization complete {:.2f} seconds".format(duration))
# Remove broken symlinks/junctions to the layout folder
if os.path.isdir(parsed_args.layout_root) and not os.path.exists(parsed_args.layout_root):
remove_link(parsed_args.layout_root)
if not os.path.isdir(parsed_args.layout_root):
# If the layout target doesnt exist, check if we want to create it
if parsed_args.create_layout_root:
@@ -617,42 +633,10 @@ def main(args):
raise common.LmbrCmdError("Invalid layout folder (--layout-root): '{}'".format(parsed_args.layout_root),
common.ERROR_CODE_ERROR_DIRECTORY)
# Prepare the logging
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG if parsed_args.debug else logging.INFO)
# Validate the dev root
for check_file in ('bootstrap.cfg', 'engine.json'):
if not os.path.isfile(os.path.join(parsed_args.dev_root, check_file)):
raise common.LmbrCmdError("Invalid value for --dev-root. Path '{}' missing file '{}'".format(parsed_args.dev_root, check_file),
common.ERROR_CODE_INVALID_PARAMETER)
# Validate the asset mode
input_asset_mode = parsed_args.mode.upper()
if input_asset_mode not in ALL_ASSET_MODES:
raise common.LmbrCmdError("Invalid asset mode '{}'. Must be one of : '{}'.".format(input_asset_mode, ','.join(ALL_ASSET_MODES)),
common.ERROR_CODE_INVALID_PARAMETER)
# Check if the build config is set, if so, check if its release
build_config = parsed_args.build_config.upper()
if build_config in PAK_ONLY_BUILD_CONFIGS:
input_asset_mode = ASSET_MODE_PAK
logging.info("Starting (%s) Asset Synchronization in %s mode and game %s", parsed_args.asset_type, input_asset_mode, parsed_args.game)
start_time = timeit.default_timer()
ASSET_SYNC_MODE_FUNCTION[input_asset_mode](dev_root=os.path.normpath(parsed_args.dev_root),
target_platform=parsed_args.platform,
game=parsed_args.game,
asset_type=parsed_args.asset_type,
warning_on_missing_assets=parsed_args.warn_on_missing_assets,
layout_target=os.path.normpath(parsed_args.layout_root),
override_pak_folder=parsed_args.override_pak_folder,
copy=parsed_args.copy)
duration = timeit.default_timer() - start_time
logging.info("Asset Synchronization complete {:.2f} seconds".format(duration))
if parsed_args.verify:
warnings = verify_layout(layout_dir=os.path.normpath(parsed_args.layout_root),
platform_name=parsed_args.platform,
game_name=parsed_args.game,
project_path=parsed_args.project_path,
asset_mode=input_asset_mode,
asset_type=parsed_args.asset_type)
if warnings > 0: