diff --git a/scripts/license_scanner/license_scanner.py b/scripts/license_scanner/license_scanner.py index c0e3c1f1ba..2936e343f9 100644 --- a/scripts/license_scanner/license_scanner.py +++ b/scripts/license_scanner/license_scanner.py @@ -6,6 +6,7 @@ # import argparse +from collections import OrderedDict import fnmatch import json import os @@ -24,15 +25,19 @@ class LicenseScanner: """ DEFAULT_CONFIG_FILE = 'scanner_config.json' + DEFAULT_EXCLUDE_FILE = '.gitignore' + DEFAULT_PACKAGE_INFO_FILE = 'PackageInfo.json' def __init__(self, config_file=None): self.config_file = config_file self.config_data = self._load_config() - self.license_regex = self._load_license_regex() + self.file_regex = self._load_file_regex(self.config_data['license_patterns']) + self.package_info = self._load_file_regex(self.config_data['package_patterns']) + self.excluded_directories = self._load_file_regex(self.config_data['excluded_directories']) def _load_config(self): """Load config from the provided file. Sets default file if one is not provided.""" - if self.config_file is None: + if not self.config_file: script_directory = os.path.dirname(os.path.abspath(__file__)) # Default file expected in same dir as script self.config_file = os.path.join(script_directory, self.DEFAULT_CONFIG_FILE) @@ -43,45 +48,68 @@ class LicenseScanner: print('Config file cannot be found') raise - def _load_license_regex(self): + def _load_file_regex(self, patterns): """Returns regex object with case-insensitive matching from the list of filename patterns.""" regex_patterns = [] - for pattern in self.config_data['license_patterns']: + for pattern in patterns: regex_patterns.append(fnmatch.translate(pattern)) + + if not regex_patterns: + print(f'Warning: No patterns from {patterns} found') + return None + return re.compile('|'.join(regex_patterns), re.IGNORECASE) - def scan(self, path=os.curdir): - """Scan directory tree for filenames matching license_regex. + def scan(self, paths=os.curdir): + """Scan directory tree for filenames matching file_regex, package info, and exclusion files. - :param path: Path of the directory to run scanner - :return: Package paths and their corresponding license file contents - :rtype: dict + :param paths: Paths of the directory to run scanner + :return: Package paths and their corresponding file contents + :rtype: Ordered dict """ - licenses = 0 - license_files = {} + files = 0 + matching_files = OrderedDict() + excluded_directories = None - for dirpath, dirnames, filenames in os.walk(path): - for file in filenames: - if self.license_regex.match(file): - license_file_content = self._get_license_file_contents(os.path.join(dirpath, file)) - rel_dirpath = os.path.relpath(dirpath, path) # Limit path inside scanned directory - license_files[rel_dirpath] = license_file_content - licenses += 1 - print(f'License file: {os.path.join(dirpath, file)}') + if not self.package_info: + self.package_info = self.DEFAULT_PACKAGE_INFO_FILE - # Remove directories that should not be scanned - for dir in self.config_data['excluded_directories']: - if dir in dirnames: - dirnames.remove(dir) - print(f'{licenses} license files found.') - return license_files + if not self.excluded_directories: + print(f'No excluded directory in config, looking for {self.DEFAULT_EXCLUDE_FILE} instead') - def _get_license_file_contents(self, filepath): + for path in paths: + for dirpath, dirnames, filenames in os.walk(path, topdown=True): + dirnames.sort(key=str.casefold) # Ensure that results are sorted + for file in filenames: + if self.file_regex.match(file) or self.package_info.match(file): + file_path = os.path.join(dirpath, file) + matching_file_content = self._get_file_contents(file_path) + matching_files[file_path] = matching_file_content + files += 1 + print(f'Matching file: {file_path}') + if self.package_info.match(file): + dirnames[:] = [] # Stop scanning subdirectories if package info file found + if self.DEFAULT_EXCLUDE_FILE in file and not self.excluded_directories: + ignore_list = self._get_file_contents(os.path.join(dirpath, file)).splitlines() + ignore_list.append('.git') # .gitignore doesn't usually have .git in its exclusions + excluded_directories = self._load_file_regex(ignore_list) + + # Remove directories that should not be scanned + if self.excluded_directories: + excluded_directories = self.excluded_directories + for dir in dirnames: + if excluded_directories.match(dir): + dirnames.remove(dir) + + print(f'{files} files found.') + return matching_files + + def _get_file_contents(self, filepath): try: with open(filepath, encoding='utf8') as f: return f.read() except UnicodeDecodeError: - print(f'Unable to read license file: {filepath}') + print(f'Unable to read file: {filepath}') pass def create_license_file(self, licenses, filepath='NOTICES.txt'): @@ -89,18 +117,44 @@ class LicenseScanner: :param licenses: Dict with package paths and their corresponding license file contents :param filepath: Path to write the file - """ - package_separator = '------------------------------------' - with open(filepath, 'w', encoding='utf8') as f: + """ + license_separator = '------------------------------------' + with open(filepath, 'w', encoding='utf8') as lf: for directory, license in licenses.items(): - license_output = '\n\n'.join([ - f'{package_separator}', - f'Package path: {directory}', - 'License:', - f'{license}\n' - ]) - f.write(license_output) + if not self.package_info.match(os.path.basename(directory)): + license_output = '\n\n'.join([ + f'{license_separator}', + f'Package path: {os.path.relpath(directory)}', + 'License:', + f'{license}\n' + ]) + lf.write(license_output) return None + + def create_package_file(self, packages, filepath='SPDX-Licenses.json', get_contents=False): + """Creates file with all the provided SPDX package info summaries in json. + Optional dirpath parameter will follow the license file path in the package info and return its contents in a dictionary + + :param licenses: Dict with package info paths and their corresponding file contents + :param filepath: Path to write the file + :param dirpath: Root path for packages + :rtype: Ordered dict + """ + licenses = OrderedDict() + package_json = [] + + with open(filepath, 'w', encoding='utf8') as pf: + for directory, package in packages.items(): + if self.package_info.match(os.path.basename(directory)): + package_obj = json.loads(package) + package_json.append(package_obj) + if get_contents: + license_path = os.path.join(os.path.dirname(directory), pathlib.Path(package_obj['LicenseFile'])) + licenses[license_path] = self._get_file_contents(license_path) + else: + licenses[directory] = package + pf.write(json.dumps(package_json, indent=4)) + return licenses def parse_args(): @@ -108,7 +162,8 @@ def parse_args(): description='Script to run LicenseScanner and generate license file') parser.add_argument('--config-file', '-c', type=pathlib.Path, help='Config file for LicenseScanner') parser.add_argument('--license-file-path', '-l', type=pathlib.Path, help='Create license file in the provided path') - parser.add_argument('--scan-path', '-s', default=os.curdir, type=pathlib.Path, help='Path to scan') + parser.add_argument('--package-file-path', '-p', type=pathlib.Path, help='Create package summary file in the provided path') + parser.add_argument('--scan-path', '-s', default=os.curdir, type=pathlib.Path, nargs='+', help='Path to scan, multiple space separated paths can be used') return parser.parse_args() @@ -116,10 +171,15 @@ def main(): try: args = parse_args() ls = LicenseScanner(args.config_file) - licenses = ls.scan(args.scan_path) + scanned_path_data = ls.scan(args.scan_path) if args.license_file_path: - ls.create_license_file(licenses, args.license_file_path) + ls.create_license_file(scanned_path_data, args.license_file_path) + if args.package_file_path: + ls.create_package_file(scanned_path_data, args.package_file_path) + if args.license_file_path and args.package_file_path: + license_files = ls.create_package_file(scanned_path_data, args.package_file_path, True) + ls.create_license_file(license_files, args.license_file_path) except FileNotFoundError as e: print(f'Type: {type(e).__name__}, Error: {e}') return 1 diff --git a/scripts/license_scanner/scanner_config.json b/scripts/license_scanner/scanner_config.json index b5863a7d31..2e8f5206db 100644 --- a/scripts/license_scanner/scanner_config.json +++ b/scripts/license_scanner/scanner_config.json @@ -8,5 +8,8 @@ "license_patterns": [ "LICENSE*", "COPYING*" + ], + "package_patterns": [ + "PackageInfo.json" ] }