birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make scrubber/validator not depend on legacy packaging scripts (#6053)

Browse Source 
* Make validator not depend on legacy packaging scripts

Signed-off-by: Shirang Jia <shiranj@amazon.com>

* Remove unused glob_to_regex.py

Signed-off-by: Shirang Jia <shiranj@amazon.com>

* Remove unsued import path

Signed-off-by: Shirang Jia <shiranj@amazon.com>
This commit is contained in:
Shirang Jia
2021-11-30 16:46:03 -08:00
committed by GitHub
parent 2809661794
commit 58be7c27ed
3 changed files with 12 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build/package/glob_to_regex.py
-130
View File
@@ -1,130 +0,0 @@
#
# Copyright (c) Contributors to the Open 3D Engine Project.
# For complete copyright and license terms please see the LICENSE at the root of this distribution.
#
# SPDX-License-Identifier: Apache-2.0 OR MIT
#
#
from __future__ import absolute_import
import os
import re
import json
import sys
try:
import six
except ImportError:
import pip
pip.main(['install', 'six', '--ignore-installed', '-q'])
import six
from pathlib import Path
this_file_path = os.path.dirname(os.path.realpath(__file__))
# resolve symlinks and eliminate ".." components
engine_root_path = Path(__file__).resolve().parents[3]
def convert_glob_pattern_to_regex_pattern(glob_pattern):
# switch to forward slashes because way easier to pattern match against
pattern = re.sub(r'\\', r'/', glob_pattern)
# Replace the dots and question marks
pattern = re.sub(r'\.', r'\\.', pattern)
pattern = re.sub(r'\?', r'.', pattern)
# Handle the * vs ** expansions
pattern = re.sub(r'([^*])\*($|[^*])', r'\1[^/\\\\]*\2', pattern)
pattern = re.sub(r'\*\*/', r'(.*/)?', pattern)
pattern = re.sub(r'\*\*', r'.*', pattern)
# replace the forward slashes with [/\\] so it works on PC/unix
pattern = re.sub(r'([^^])/', r'\1[/\\\\]', pattern)
return pattern
# Convert the package json into a pair of regexes we can use to look for includes and excludes
def convert_glob_list_to_regex_list(filelist, prefix):
includes = []
excludes = []
for key, value in six.iteritems(filelist):
glob_pattern = os.path.join(prefix, key)
if isinstance(value, dict):
(sub_includes, sub_excludes) = convert_glob_list_to_regex_list(value, glob_pattern)
includes.extend(sub_includes)
excludes.extend(sub_excludes)
else:
# Simulate what glob would do with file walking to scope the * within a directory
# and ** across directories
regex_pattern = convert_glob_pattern_to_regex_pattern(os.path.normpath(glob_pattern))
# Deal with the commands. include/exclude are straight forward. Moves/renames are to be considered
# includes, and we will stick with validating the original contents for now
if value == "#include":
includes.append(regex_pattern)
elif value == "#exclude":
excludes.append(regex_pattern)
elif value.startswith('#move:'):
includes.append(regex_pattern)
elif value.startswith('#rename:'):
includes.append(regex_pattern)
else:
pass
return (includes, excludes)
def generate_excludes_for_platform(root, platform):
if platform == 'all':
platform_exclusions_filename = os.path.join(this_file_path, 'platform_exclusions.json')
with open(platform_exclusions_filename, 'r') as platform_exclusions_file:
platform_exclusions = json.load(platform_exclusions_file)
else:
# Use real path in case root is a symlink path
if os.name == 'posix' and os.path.islink(root):
root = os.readlink(root)
# "root" is the root of the folder structure we're validating
# "engine_root_path" is the engine root where the restricted platform folder is linked
relative_folder = os.path.relpath(this_file_path, engine_root_path)
platform_exclusions_filename = os.path.join(engine_root_path, 'restricted', platform, relative_folder, platform.lower() + '_exclusions.json')
with open(platform_exclusions_filename, 'r') as platform_exclusions_file:
platform_exclusions = json.load(platform_exclusions_file)
if platform not in platform_exclusions:
raise KeyError('No {} found in {}'.format(platform, platform_exclusions_filename))
if '@lyengine' not in platform_exclusions[platform]:
raise KeyError('No {}/@lyengine found in {}'.format(platform, package_file_list))
(_, excludes) = convert_glob_list_to_regex_list(platform_exclusions[platform]['@lyengine'], root)
del _
return excludes
def generate_include_exclude_regexes(package_platform, package_type, root, prohibited_platforms):
# The general contents will be indicated by the package file
if package_type == 'all':
package_file_list = os.path.join(this_file_path, 'package_filelists', 'all.json')
else:
# Search non-restricted platform first
package_file_list = os.path.join(this_file_path, 'Platform', package_platform, 'package_filelists', f'{package_type}.json')
if not os.path.exists(filelist):
# Use real path in case root is a symlink path
if os.name == 'posix' and os.path.islink(root):
root = os.readlink(root)
# "root" is the root of the folder structure we're validating
# "engine_root_path" is the engine root where the restricted platform folder is linked
rel_path = os.path.relpath(this_file_path, engine_root_path)
package_file_list = os.path.join(engine_root_path, 'restricted', package_platform, rel_path, 'package_filelists',
f'{package_type}.json')
with open(package_file_list, 'r') as package_file:
package = json.load(package_file)
if '@lyengine' not in package:
raise KeyError('No @lyengine found in {}'.format(package_file_list))
(includes_list, excludes_list) = convert_glob_list_to_regex_list(package['@lyengine'], root)
prohibited_platforms.append('all')
# Add the exclusions of each prohibited platform
for p in prohibited_platforms:
excludes_list.extend(generate_excludes_for_platform(root, p))
includes = re.compile('|'.join(includes_list), re.IGNORECASE)
excludes = re.compile('|'.join(excludes_list), re.IGNORECASE)
return (includes, excludes)
def generate_exclude_regexes_for_platform(root, platform):
return re.compile('|'.join(generate_excludes_for_platform(root, platform)), re.IGNORECASE)
scripts/scrubbing/scrubbing_job.py
-29
View File
@@ -1,29 +0,0 @@
#
# Copyright (c) Contributors to the Open 3D Engine Project.
# For complete copyright and license terms please see the LICENSE at the root of this distribution.
#
# SPDX-License-Identifier: Apache-2.0 OR MIT
#
#
import os
import sys
cur_dir = cur_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.abspath(f'{cur_dir}/../build/package'))
import util
# Run validator
success = True
validator_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'validator.py')
engine_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
if sys.platform == 'win32':
python = os.path.join(engine_root, 'python', 'python.cmd')
else:
python = os.path.join(engine_root, 'python', 'python.sh')
args = [python, validator_path, '--package_platform', 'Windows', '--package_type', 'all', engine_root]
return_code = util.safe_execute_system_call(args)
if return_code != 0:
success = False
if not success:
util.error('Restricted file validator failed.')
print('Restricted file validator completed successfully.')
scripts/scrubbing/validator.py
+12 -25
View File
@@ -29,8 +29,6 @@ else:
from io import StringIO
import validator_data_LEGAL_REVIEW_REQUIRED # pull in the data we need to configure this tool
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'build', 'package'))
from glob_to_regex import generate_include_exclude_regexes
class Validator(object):
"""Class to contain the validator program"""
@@ -212,9 +210,6 @@ class Validator(object):
# TODO: Perhaps the directories to skip should become a parameter so we can use the validator
# on non-Lumberyard trees.
def validate_directory_tree(self, root, platform):
prohibited_platforms = validator_data_LEGAL_REVIEW_REQUIRED.get_prohibited_platforms_for_package(self.options.package_platform)
(includes, excludes) = generate_include_exclude_regexes(self.options.package_platform, self.options.package_type, root, prohibited_platforms)
"""Walk from root to find all files to validate and call the validator on each file.
Return 0 if no problems where found, and 1 if any validation failures occured."""
counter = 0
@@ -227,28 +222,22 @@ class Validator(object):
# First deal with the files in the current directory
for filename in filenames:
filepath = os.path.join(dirname, filename)
include_match = includes.match(filepath)
exclude_match = excludes.match(filepath)
allowed = include_match and not exclude_match
if self.options.all or allowed:
scanned += 1
file_failed = self.validate_file(os.path.normpath(filepath))
if file_failed:
platform_failed = file_failed
else:
validations += 1
counter += 1
scanned += 1
file_failed = self.validate_file(os.path.normpath(filepath))
if file_failed:
platform_failed = file_failed
else:
validations += 1
# Trim out allowlisted subdirectories in the current directory if allowed
for name in bypassed_directories:
if name in dirnames:
dirnames.remove(name)
if counter == 0 or scanned == 0:
if scanned == 0:
logging.error('No files scanned at target search directory: %s', root)
platform_failed = 1
else:
print('validated {} of {} package files ({} non-package files skipped)'.format(validations, scanned, counter - scanned))
print('validated {} of {} files'.format(validations, scanned))
return platform_failed
@@ -387,8 +376,6 @@ def parse_options():
choices=platform_choices,
dest='package_platform',
help='Package platform to validate. Must be one of {}.'.format(platform_choices))
parser.add_option('--package_type', action='store', type='string', default='all', dest='package_type',
help='Package type to validate.')
parser.add_option('-s', '--store-exceptions', action='store', type='string', default='',
dest='exception_file',
help='Store list of lines that the validator gave exceptions to by matching accepted use patterns. These can be diffed with prior runs to see what is changing.')
@@ -430,7 +417,6 @@ def main():
package_failed = 0
package_platform = validator.options.package_platform
package_type = validator.options.package_type
prohibited_platforms = validator_data_LEGAL_REVIEW_REQUIRED.get_prohibited_platforms_for_package(package_platform)
if validator.options.exception_file != '':
@@ -441,19 +427,20 @@ def main():
sys.exit(1)
for platform in prohibited_platforms:
print('validating {} against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type))
print('validating {} against {} for package platform {}'.format(args[0], platform, package_platform))
platform_failed = validator.validate(platform)
if platform_failed:
print('{} FAILED validation against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type))
print('{} FAILED validation against {} for package platform {}'.format(args[0], platform, package_platform))
package_failed = platform_failed
else:
print('{} is VALIDATED against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type))
print('{} is VALIDATED against {} for package platform {}'.format(args[0], platform, package_platform))
if validator.options.exception_file != '':
validator.exceptions_output.close()
return package_failed
if __name__ == '__main__':
# pylint: disable-msg=C0103
main_results = main()