From 58be7c27edec0319cc76b22f5389e1f77cda398b Mon Sep 17 00:00:00 2001 From: Shirang Jia Date: Tue, 30 Nov 2021 16:46:03 -0800 Subject: [PATCH] Make scrubber/validator not depend on legacy packaging scripts (#6053) * Make validator not depend on legacy packaging scripts Signed-off-by: Shirang Jia * Remove unused glob_to_regex.py Signed-off-by: Shirang Jia * Remove unsued import path Signed-off-by: Shirang Jia --- scripts/build/package/glob_to_regex.py | 130 ------------------------- scripts/scrubbing/scrubbing_job.py | 29 ------ scripts/scrubbing/validator.py | 37 +++---- 3 files changed, 12 insertions(+), 184 deletions(-) delete mode 100755 scripts/build/package/glob_to_regex.py delete mode 100755 scripts/scrubbing/scrubbing_job.py diff --git a/scripts/build/package/glob_to_regex.py b/scripts/build/package/glob_to_regex.py deleted file mode 100755 index 53ad3d850d..0000000000 --- a/scripts/build/package/glob_to_regex.py +++ /dev/null @@ -1,130 +0,0 @@ -# -# Copyright (c) Contributors to the Open 3D Engine Project. -# For complete copyright and license terms please see the LICENSE at the root of this distribution. -# -# SPDX-License-Identifier: Apache-2.0 OR MIT -# -# -from __future__ import absolute_import -import os -import re -import json -import sys -try: - import six -except ImportError: - import pip - pip.main(['install', 'six', '--ignore-installed', '-q']) - import six -from pathlib import Path - -this_file_path = os.path.dirname(os.path.realpath(__file__)) - -# resolve symlinks and eliminate ".." components -engine_root_path = Path(__file__).resolve().parents[3] - -def convert_glob_pattern_to_regex_pattern(glob_pattern): - # switch to forward slashes because way easier to pattern match against - pattern = re.sub(r'\\', r'/', glob_pattern) - - # Replace the dots and question marks - pattern = re.sub(r'\.', r'\\.', pattern) - pattern = re.sub(r'\?', r'.', pattern) - - # Handle the * vs ** expansions - pattern = re.sub(r'([^*])\*($|[^*])', r'\1[^/\\\\]*\2', pattern) - pattern = re.sub(r'\*\*/', r'(.*/)?', pattern) - pattern = re.sub(r'\*\*', r'.*', pattern) - - # replace the forward slashes with [/\\] so it works on PC/unix - pattern = re.sub(r'([^^])/', r'\1[/\\\\]', pattern) - return pattern - -# Convert the package json into a pair of regexes we can use to look for includes and excludes -def convert_glob_list_to_regex_list(filelist, prefix): - includes = [] - excludes = [] - for key, value in six.iteritems(filelist): - glob_pattern = os.path.join(prefix, key) - if isinstance(value, dict): - (sub_includes, sub_excludes) = convert_glob_list_to_regex_list(value, glob_pattern) - includes.extend(sub_includes) - excludes.extend(sub_excludes) - else: - # Simulate what glob would do with file walking to scope the * within a directory - # and ** across directories - regex_pattern = convert_glob_pattern_to_regex_pattern(os.path.normpath(glob_pattern)) - - # Deal with the commands. include/exclude are straight forward. Moves/renames are to be considered - # includes, and we will stick with validating the original contents for now - if value == "#include": - includes.append(regex_pattern) - elif value == "#exclude": - excludes.append(regex_pattern) - elif value.startswith('#move:'): - includes.append(regex_pattern) - elif value.startswith('#rename:'): - includes.append(regex_pattern) - else: - pass - return (includes, excludes) - -def generate_excludes_for_platform(root, platform): - if platform == 'all': - platform_exclusions_filename = os.path.join(this_file_path, 'platform_exclusions.json') - with open(platform_exclusions_filename, 'r') as platform_exclusions_file: - platform_exclusions = json.load(platform_exclusions_file) - else: - # Use real path in case root is a symlink path - if os.name == 'posix' and os.path.islink(root): - root = os.readlink(root) - # "root" is the root of the folder structure we're validating - # "engine_root_path" is the engine root where the restricted platform folder is linked - relative_folder = os.path.relpath(this_file_path, engine_root_path) - platform_exclusions_filename = os.path.join(engine_root_path, 'restricted', platform, relative_folder, platform.lower() + '_exclusions.json') - with open(platform_exclusions_filename, 'r') as platform_exclusions_file: - platform_exclusions = json.load(platform_exclusions_file) - - if platform not in platform_exclusions: - raise KeyError('No {} found in {}'.format(platform, platform_exclusions_filename)) - if '@lyengine' not in platform_exclusions[platform]: - raise KeyError('No {}/@lyengine found in {}'.format(platform, package_file_list)) - (_, excludes) = convert_glob_list_to_regex_list(platform_exclusions[platform]['@lyengine'], root) - del _ - return excludes - -def generate_include_exclude_regexes(package_platform, package_type, root, prohibited_platforms): - # The general contents will be indicated by the package file - if package_type == 'all': - package_file_list = os.path.join(this_file_path, 'package_filelists', 'all.json') - else: - # Search non-restricted platform first - package_file_list = os.path.join(this_file_path, 'Platform', package_platform, 'package_filelists', f'{package_type}.json') - if not os.path.exists(filelist): - # Use real path in case root is a symlink path - if os.name == 'posix' and os.path.islink(root): - root = os.readlink(root) - # "root" is the root of the folder structure we're validating - # "engine_root_path" is the engine root where the restricted platform folder is linked - rel_path = os.path.relpath(this_file_path, engine_root_path) - package_file_list = os.path.join(engine_root_path, 'restricted', package_platform, rel_path, 'package_filelists', - f'{package_type}.json') - with open(package_file_list, 'r') as package_file: - package = json.load(package_file) - - if '@lyengine' not in package: - raise KeyError('No @lyengine found in {}'.format(package_file_list)) - - (includes_list, excludes_list) = convert_glob_list_to_regex_list(package['@lyengine'], root) - prohibited_platforms.append('all') - - # Add the exclusions of each prohibited platform - for p in prohibited_platforms: - excludes_list.extend(generate_excludes_for_platform(root, p)) - - includes = re.compile('|'.join(includes_list), re.IGNORECASE) - excludes = re.compile('|'.join(excludes_list), re.IGNORECASE) - return (includes, excludes) - -def generate_exclude_regexes_for_platform(root, platform): - return re.compile('|'.join(generate_excludes_for_platform(root, platform)), re.IGNORECASE) diff --git a/scripts/scrubbing/scrubbing_job.py b/scripts/scrubbing/scrubbing_job.py deleted file mode 100755 index 11dbf5ea5a..0000000000 --- a/scripts/scrubbing/scrubbing_job.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) Contributors to the Open 3D Engine Project. -# For complete copyright and license terms please see the LICENSE at the root of this distribution. -# -# SPDX-License-Identifier: Apache-2.0 OR MIT -# -# - -import os -import sys -cur_dir = cur_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.abspath(f'{cur_dir}/../build/package')) -import util - -# Run validator -success = True -validator_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'validator.py') -engine_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) -if sys.platform == 'win32': - python = os.path.join(engine_root, 'python', 'python.cmd') -else: - python = os.path.join(engine_root, 'python', 'python.sh') -args = [python, validator_path, '--package_platform', 'Windows', '--package_type', 'all', engine_root] -return_code = util.safe_execute_system_call(args) -if return_code != 0: - success = False -if not success: - util.error('Restricted file validator failed.') -print('Restricted file validator completed successfully.') diff --git a/scripts/scrubbing/validator.py b/scripts/scrubbing/validator.py index 39f1a40a18..3b53417755 100755 --- a/scripts/scrubbing/validator.py +++ b/scripts/scrubbing/validator.py @@ -29,8 +29,6 @@ else: from io import StringIO import validator_data_LEGAL_REVIEW_REQUIRED # pull in the data we need to configure this tool -sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'build', 'package')) -from glob_to_regex import generate_include_exclude_regexes class Validator(object): """Class to contain the validator program""" @@ -212,9 +210,6 @@ class Validator(object): # TODO: Perhaps the directories to skip should become a parameter so we can use the validator # on non-Lumberyard trees. def validate_directory_tree(self, root, platform): - prohibited_platforms = validator_data_LEGAL_REVIEW_REQUIRED.get_prohibited_platforms_for_package(self.options.package_platform) - (includes, excludes) = generate_include_exclude_regexes(self.options.package_platform, self.options.package_type, root, prohibited_platforms) - """Walk from root to find all files to validate and call the validator on each file. Return 0 if no problems where found, and 1 if any validation failures occured.""" counter = 0 @@ -227,28 +222,22 @@ class Validator(object): # First deal with the files in the current directory for filename in filenames: filepath = os.path.join(dirname, filename) - include_match = includes.match(filepath) - exclude_match = excludes.match(filepath) - allowed = include_match and not exclude_match - - if self.options.all or allowed: - scanned += 1 - file_failed = self.validate_file(os.path.normpath(filepath)) - if file_failed: - platform_failed = file_failed - else: - validations += 1 - counter += 1 + scanned += 1 + file_failed = self.validate_file(os.path.normpath(filepath)) + if file_failed: + platform_failed = file_failed + else: + validations += 1 # Trim out allowlisted subdirectories in the current directory if allowed for name in bypassed_directories: if name in dirnames: dirnames.remove(name) - if counter == 0 or scanned == 0: + if scanned == 0: logging.error('No files scanned at target search directory: %s', root) platform_failed = 1 else: - print('validated {} of {} package files ({} non-package files skipped)'.format(validations, scanned, counter - scanned)) + print('validated {} of {} files'.format(validations, scanned)) return platform_failed @@ -387,8 +376,6 @@ def parse_options(): choices=platform_choices, dest='package_platform', help='Package platform to validate. Must be one of {}.'.format(platform_choices)) - parser.add_option('--package_type', action='store', type='string', default='all', dest='package_type', - help='Package type to validate.') parser.add_option('-s', '--store-exceptions', action='store', type='string', default='', dest='exception_file', help='Store list of lines that the validator gave exceptions to by matching accepted use patterns. These can be diffed with prior runs to see what is changing.') @@ -430,7 +417,6 @@ def main(): package_failed = 0 package_platform = validator.options.package_platform - package_type = validator.options.package_type prohibited_platforms = validator_data_LEGAL_REVIEW_REQUIRED.get_prohibited_platforms_for_package(package_platform) if validator.options.exception_file != '': @@ -441,19 +427,20 @@ def main(): sys.exit(1) for platform in prohibited_platforms: - print('validating {} against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type)) + print('validating {} against {} for package platform {}'.format(args[0], platform, package_platform)) platform_failed = validator.validate(platform) if platform_failed: - print('{} FAILED validation against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type)) + print('{} FAILED validation against {} for package platform {}'.format(args[0], platform, package_platform)) package_failed = platform_failed else: - print('{} is VALIDATED against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type)) + print('{} is VALIDATED against {} for package platform {}'.format(args[0], platform, package_platform)) if validator.options.exception_file != '': validator.exceptions_output.close() return package_failed + if __name__ == '__main__': # pylint: disable-msg=C0103 main_results = main()