birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'development' into Prism/ShowRepoList

Browse Source
This commit is contained in:
nggieber
2021-10-11 09:20:06 -07:00
parent 70053f055a 189aa5f3ac
commit 3698c3c3a7
889 changed files with 22679 additions and 14037 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build/Jenkins/Jenkinsfile
Vendored
+20 -15
View File
@@ -793,27 +793,32 @@ catch(Exception e) {
}
finally {
try {
if(env.SNS_TOPIC) {
snsPublish(
topicArn: env.SNS_TOPIC,
subject:'Build Result',
message:"${currentBuild.currentResult}:${BUILD_URL}:${env.RECREATE_VOLUME}:${env.CLEAN_OUTPUT_DIRECTORY}:${env.CLEAN_ASSETS}"
)
}
node('controller') {
if("${currentBuild.currentResult}" == "SUCCESS") {
buildFailure = ""
emailBody = "${BUILD_URL}\nSuccess!"
} else {
buildFailure = tm('${BUILD_FAILURE_ANALYZER}')
emailBody = "${BUILD_URL}\n${buildFailure}!"
if(env.SNS_TOPIC_BUILD_FAILURE) {
message_json = ["build_url":env.BUILD_URL, "repository_name":env.REPOSITORY_NAME, "branch_name":env.BRANCH_NAME, "build_failure":buildFailure]
snsPublish(
topicArn: env.SNS_TOPIC_BUILD_FAILURE,
subject:'Build Failure',
message:JsonOutput.toJson(message_json)
)
}
}
if(env.POST_AR_BUILD_SNS_TOPIC) {
message_json = [
"build_url": env.BUILD_URL,
"build_number": env.BUILD_NUMBER,
"repository_name": env.REPOSITORY_NAME,
"branch_name": env.BRANCH_NAME,
"build_result": "${currentBuild.currentResult}",
"build_failure": buildFailure,
"recreate_volume": env.RECREATE_VOLUME,
"clean_output_directory": env.CLEAN_OUTPUT_DIRECTORY,
"clean_assets": env.CLEAN_ASSETS
]
snsPublish(
topicArn: env.POST_AR_BUILD_SNS_TOPIC,
subject:'Build Result',
message:JsonOutput.toJson(message_json)
)
}
emailext (
body: "${emailBody}",
scripts/build/Platform/Windows/build_config.json
-1
View File
@@ -31,7 +31,6 @@
],
"steps": [
"profile_vs2019",
"test_impact_analysis_profile_vs2019",
"asset_profile_vs2019",
"test_cpu_profile_vs2019"
]
scripts/build/Platform/Windows/deploy_cdk_applications.cmd
+1 -1
View File
@@ -57,7 +57,7 @@ IF ERRORLEVEL 1 (
exit /b 1
)
CALL :DeployCDKApplication AWSCore --all
CALL :DeployCDKApplication AWSCore --all "-c disable_access_log=true"
IF ERRORLEVEL 1 (
exit /b 1
)
scripts/build/bootstrap/incremental_build_util.py
+23 -7
View File
@@ -18,6 +18,8 @@ from contextlib import contextmanager
import threading
import _thread
from botocore.config import Config
DEFAULT_REGION = 'us-west-2'
DEFAULT_DISK_SIZE = 300
DEFAULT_DISK_TYPE = 'gp2'
@@ -173,10 +175,27 @@ def get_region_name():
def get_ec2_client(region):
client = boto3.client('ec2', region_name=region)
client_config = Config(
region_name=region,
retries={
'mode': 'standard'
}
)
client = boto3.client('ec2', config=client_config)
return client
def get_ec2_resource(region):
resource_config = Config(
region_name=region,
retries={
'mode': 'standard'
}
)
resource = boto3.resource('ec2', config=resource_config)
return resource
def get_ec2_instance_id():
try:
instance_id = urllib.request.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read()
@@ -395,14 +414,11 @@ def detach_volume_from_ec2_instance(volume, ec2_instance_id, force, timeout_dura
def mount_ebs(snapshot_hint, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type):
session = boto3.session.Session()
region = session.region_name
if region is None:
region = DEFAULT_REGION
region = get_region_name()
ec2_client = get_ec2_client(region)
ec2_instance_id = get_ec2_instance_id()
ec2_availability_zone = get_availability_zone()
ec2_resource = boto3.resource('ec2', region_name=region)
ec2_resource = get_ec2_resource(region)
ec2_instance = ec2_resource.Instance(ec2_instance_id)
for volume in ec2_instance.volumes.all():
@@ -469,7 +485,7 @@ def mount_ebs(snapshot_hint, repository_name, project, pipeline, branch, platfor
def unmount_ebs():
region = get_region_name()
ec2_instance_id = get_ec2_instance_id()
ec2_resource = boto3.resource('ec2', region_name=region)
ec2_resource = get_ec2_resource(region)
ec2_instance = ec2_resource.Instance(ec2_instance_id)
if os.path.isfile('envinject.properties'):
scripts/o3de/o3de/engine_template.py
+43 -4
View File
@@ -59,6 +59,14 @@ binary_file_ext = {
'.motionset'
}
cpp_file_ext = {
'.cpp',
'.h',
'.hpp',
'.hxx',
'.inl'
}
expect_license_info_ext = {
'.cpp',
'.h',
@@ -402,7 +410,7 @@ def create_template(source_path: pathlib.Path,
# if no template path, error
if not template_path:
logger.info(f'Template path empty. Using source name {source_name}')
template_path = source_name
template_path = pathlib.Path(source_name)
if not template_path.is_absolute():
default_templates_folder = manifest.get_registered(default_folder='templates')
template_path = default_templates_folder / template_path
@@ -518,21 +526,52 @@ def create_template(source_path: pathlib.Path,
replacements.append((source_name.upper(), '${NameUpper}'))
replacements.append((source_name, '${Name}'))
replacements.append((sanitized_source_name, '${SanitizedCppName}'))
sanitized_name_index = len(replacements) - 1
def _transform_into_template(s_data: object) -> (bool, str):
def _is_cpp_file(file_path: pathlib.Path) -> bool:
"""
Internal helper method to check if a file is a C++ file based
on its extension, so we can determine if we need to prefer
the ${SanitizedCppName}
:param file_path: The input file path
:return: bool: Whether or not the input file path has a C++ extension
"""
name, ext = os.path.splitext(file_path)
return ext.lower() in cpp_file_ext
def _transform_into_template(s_data: object,
prefer_sanitized_name: bool = False) -> (bool, str):
"""
Internal function to transform any data into templated data
:param s_data: the input data, this could be file data or file name data
:param prefer_sanitized_name: Optionally swap the sanitized name with the normal name
This can be necessary when creating the template, the source
name and sanitized source name might be the same, but C++
files will need to prefer the sanitized version, or else
there might be compile errors (e.g. '-' characters in the name)
:return: bool: whether or not the returned data MAY need to be transformed to instantiate it
t_data: potentially transformed data 0 for success or non 0 failure code
"""
def swap_sanitized_name_and_normal():
replacements[sanitized_name_index-1], replacements[sanitized_name_index] = \
replacements[sanitized_name_index], replacements[sanitized_name_index-1]
# copy the src data to the transformed data, then operate only on transformed data
t_data = str(s_data)
# If we need to prefer the sanitized name, then swap it for the normal
if prefer_sanitized_name:
swap_sanitized_name_and_normal()
# run all the replacements
for replacement in replacements:
t_data = t_data.replace(replacement[0], replacement[1])
# Once we are done running the replacements, reset the list if we had modified it
if prefer_sanitized_name:
swap_sanitized_name_and_normal()
if not keep_license_text:
t_data = _replace_license_text(t_data)
@@ -704,7 +743,7 @@ def create_template(source_path: pathlib.Path,
# open the file and attempt to transform it
with open(entry_abs, 'r') as s:
source_data = s.read()
templated, source_data = _transform_into_template(source_data)
templated, source_data = _transform_into_template(source_data, _is_cpp_file(entry_abs))
# if the file type is a file that we expect to fins license header and we don't find any
# warn that the we didn't find the license info, this makes it easy to make sure we didn't
@@ -840,7 +879,7 @@ def create_template(source_path: pathlib.Path,
# open the file and attempt to transform it
with open(entry_abs, 'r') as s:
source_data = s.read()
templated, source_data = _transform_into_template(source_data)
templated, source_data = _transform_into_template(source_data, _is_cpp_file(entry_abs))
# if the file type is a file that we expect to fins license header and we don't find any
# warn that the we didn't find the license info, this makes it easy to make sure we didn't
scripts/o3de/o3de/register.py
+4 -3
View File
@@ -487,14 +487,14 @@ def register_repo(json_data: dict,
if remove:
logger.warn(f'Removing repo uri {repo_uri}.')
return 0
repo_sha256 = hashlib.sha256(url.encode())
cache_file = manifest.get_o3de_cache_folder() / str(repo_sha256.hexdigest() + '.json')
result = utils.download_file(url, cache_file)
result = utils.download_file(parsed_uri, cache_file)
if result == 0:
json_data['repos'].insert(0, repo_uri.as_posix())
json_data['repos'].insert(0, repo_uri)
repo_set = set()
result = repo.process_add_o3de_repo(cache_file, repo_set)
return result
@@ -621,6 +621,7 @@ def register(engine_path: pathlib.Path = None,
return 1
result = result or register_gem_path(json_data, gem_path, remove,
external_subdir_engine_path, external_subdir_project_path)
if isinstance(external_subdir_path, pathlib.PurePath):
if not external_subdir_path:
logger.error(f'External Subdirectory path is None.')
scripts/o3de/o3de/repo.py
+48 -10
View File
@@ -12,6 +12,7 @@ import pathlib
import shutil
import urllib.parse
import urllib.request
import hashlib
from o3de import manifest, utils, validation
@@ -24,7 +25,6 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
file_name = pathlib.Path(file_name).resolve()
if not validation.valid_o3de_repo_json(file_name):
return 1
cache_folder = manifest.get_o3de_cache_folder()
with file_name.open('r') as f:
@@ -34,11 +34,30 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
logger.error(f'{file_name} failed to load: {str(e)}')
return 1
for o3de_object_uris, manifest_json in [(repo_data['engines'], 'engine.json'),
(repo_data['projects'], 'project.json'),
(repo_data['gems'], 'gem.json'),
(repo_data['template'], 'template.json'),
(repo_data['restricted'], 'restricted.json')]:
# A repo may not contain all types of object.
manifest_download_list = []
try:
manifest_download_list.append((repo_data['engines'], 'engine.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['projects'], 'project.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['gems'], 'gem.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['templates'], 'template.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['restricted'], 'restricted.json'))
except KeyError:
pass
for o3de_object_uris, manifest_json in manifest_download_list:
for o3de_object_uri in o3de_object_uris:
manifest_json_uri = f'{o3de_object_uri}/{manifest_json}'
manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode())
@@ -49,7 +68,27 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
if download_file_result != 0:
return download_file_result
repo_set |= repo_data['repos']
# Having a repo is also optional
repo_list = []
try:
repo_list.add(repo_data['repos'])
except KeyError:
pass
for repo in repo_list:
if repo not in repo_set:
repo_set.add(repo)
for o3de_object_uri in o3de_object_uris:
parsed_uri = urllib.parse.urlparse(f'{repo}/repo.json')
manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
if cache_file.is_file():
cache_file.unlink()
download_file_result = utils.download_file(parsed_uri, cache_file)
if download_file_result != 0:
return download_file_result
return process_add_o3de_repo(parsed_uri.geturl(), repo_set)
return 0
@@ -70,11 +109,10 @@ def refresh_repos() -> int:
if repo_uri not in repo_set:
repo_set.add(repo_uri)
repo_uri = f'{repo_uri}/repo.json'
repo_sha256 = hashlib.sha256(repo_uri.encode())
parsed_uri = urllib.parse.urlparse(f'{repo_uri}/repo.json')
repo_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json')
if not cache_file.is_file():
parsed_uri = urllib.parse.urlparse(repo_uri)
download_file_result = utils.download_file(parsed_uri, cache_file)
if download_file_result != 0:
return download_file_result
scripts/o3de/o3de/utils.py
+6 -2
View File
@@ -13,6 +13,10 @@ import uuid
import pathlib
import shutil
import urllib.request
import logging
logger = logging.getLogger()
logging.basicConfig()
def validate_identifier(identifier: str) -> bool:
"""
@@ -97,11 +101,11 @@ def download_file(parsed_uri, download_path: pathlib.Path) -> int:
if download_path.is_file():
logger.warn(f'File already downloaded to {download_path}.')
elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']:
with urllib.request.urlopen(url) as s:
with urllib.request.urlopen(parsed_uri.geturl()) as s:
with download_path.open('wb') as f:
shutil.copyfileobj(s, f)
else:
origin_file = pathlib.Path(url).resolve()
origin_file = pathlib.Path(parsed_uri.geturl()).resolve()
if not origin_file.is_file():
return 1
shutil.copy(origin_file, download_path)
scripts/o3de/tests/unit_test_engine_template.py
+14 -14
View File
@@ -31,17 +31,17 @@ TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE = """
#include <AzCore/EBus/EBus.h>
#include <AzCore/Interface/Interface.h>
namespace ${Name}
namespace ${SanitizedCppName}
{
class ${Name}Requests
class ${SanitizedCppName}Requests
{
public:
AZ_RTTI(${Name}Requests, "{${Random_Uuid}}");
virtual ~${Name}Requests() = default;
AZ_RTTI(${SanitizedCppName}Requests, "{${Random_Uuid}}");
virtual ~${SanitizedCppName}Requests() = default;
// Put your public methods here
};
class ${Name}BusTraits
class ${SanitizedCppName}BusTraits
: public AZ::EBusTraits
{
public:
@@ -52,31 +52,31 @@ namespace ${Name}
//////////////////////////////////////////////////////////////////////////
};
using ${Name}RequestBus = AZ::EBus<${Name}Requests, ${Name}BusTraits>;
using ${Name}Interface = AZ::Interface<${Name}Requests>;
using ${SanitizedCppName}RequestBus = AZ::EBus<${SanitizedCppName}Requests, ${SanitizedCppName}BusTraits>;
using ${SanitizedCppName}Interface = AZ::Interface<${SanitizedCppName}Requests>;
} // namespace ${Name}
} // namespace ${SanitizedCppName}
"""
TEST_TEMPLATED_CONTENT_WITH_LICENSE = CPP_LICENSE_TEXT + TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE
TEST_CONCRETE_TESTTEMPLATE_CONTENT_WITHOUT_LICENSE = string.Template(
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'Name': "TestTemplate"})
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'SanitizedCppName': "TestTemplate"})
TEST_CONCRETE_TESTTEMPLATE_CONTENT_WITH_LICENSE = string.Template(
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'Name': "TestTemplate"})
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'SanitizedCppName': "TestTemplate"})
TEST_CONCRETE_TESTPROJECT_TEMPLATE_CONTENT_WITHOUT_LICENSE = string.Template(
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'Name': "TestProject"})
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'SanitizedCppName': "TestProject"})
TEST_CONCRETE_TESTPROJECT_TEMPLATE_CONTENT_WITH_LICENSE = string.Template(
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'Name': "TestProject"})
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'SanitizedCppName': "TestProject"})
TEST_CONCRETE_TESTGEM_TEMPLATE_CONTENT_WITHOUT_LICENSE = string.Template(
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'Name': "TestGem"})
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'SanitizedCppName': "TestGem"})
TEST_CONCRETE_TESTGEM_TEMPLATE_CONTENT_WITH_LICENSE = string.Template(
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'Name': "TestGem"})
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'SanitizedCppName': "TestGem"})
TEST_TEMPLATE_JSON_CONTENTS = """\
{