""" Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. SPDX-License-Identifier: Apache-2.0 OR MIT Lumberyard Legacy Renderer to Atom Component Conversion Script What does this script do? ================================================ This script walks through all the .slice, .layer, .ly, and .cry files in a project and attempts to convert the following components into something reasonably similar that renders in Atom: Mesh Actor Do materials get carried over? ================================================ For the mesh component, this script will attempt to create an atom mesh component that uses the same material, pre-supposing that you have already run the LegacyMaterialConverter.py script sto generate Atom .material files out of legacy .mtl files For meshes that only have one sub-mesh, this is straightforward as the mesh will only have one material to apply, and this script will look for a material with the same name but a .material extension. For mult-materials, this is a little tricky since Atom does not follow the same ordered sub-material convention used by the legacy renderer. However, legacy .mtl files that were generated when adding a .fbx to a project use a naming convention that can be used by this script to match with the default materials that come from Atom. So as long as you were using the initial .mtl generated by Lumberyard and have not re-named the submaterials, it should find a match. This applies to both the ActorComponent and the MeshComponent How do I run this script from a command line? ================================================ 1) Check out any .slice, .layer, .ly, and .cry files you want to convert from source control - This script will remove the legacy components entirely, so make sure you have your files backed up before you run this script in case you want to run it again <<<<<<< HEAD 2) From the dev folder, run LegacyComponentConverter.py project= --include_gems ======= 2) From the Lumberyard root folder, run LegacyComponentConverter.py project= --include_gems >>>>>>> main - --include_gems is optional - if you include Gems, it will run all all Gems, not just the ones enabled by your project What are the artifacts of this script? ================================================ The .slice, .layer, .ly, and .cry files will be converted in-place. No new files will be created Is this script destructive? ================================================ Yes! This is a one-way conversion that will clear the old data once converted. You should back up your files before running this conversion in case you want to modify the script and re-run it on your original level. """ CONVERTED_LOG_NAME = "ComponentConversion_ConvertedLegacyFiles.log" UNCONVERTED_LOG_NAME = "ComponentConversion_UnsupportedLegacyFiles.log" STATS_LOG_NAME = "ComponentConversion_LegacyComponentStats.log" BUILD_PATH = None GEMS_PATH = None # Normal imports import sys import xml.etree.ElementTree import time from zipfile import ZipFile import tempfile import subprocess # Local python files from LegacyConversionHelpers import * from LegacyMeshComponentConverter import * from LegacyMaterialComponentConverter import * from LegacyActorComponentConverter import * from LegacyPointLightComponentConverter import * BUILD_PATH = "./" GEMS_PATH = os.path.join(BUILD_PATH, "Gems") class Component_File(object): """ Class to perform any read, write or conversion operations on material (*.mtl) files. """ def __init__(self, filename, projectDir, assetCatalogHelper, statsCollector): self.filename = filename self.normalizedProjectDir = os.path.normpath(projectDir) self.needsConversion = False self.hadException = False self.assetCatalogHelper = assetCatalogHelper self.materialComponentConverter = Material_Component_Converter(assetCatalogHelper)#TODO - I'm pretty sure this is dead code self.xml = None self.statsCollector = statsCollector self.parse_xml() def is_valid_xml(self): """ Performs a simple check to determine if the XML of the mtl is valid. This is to prevent an assert on a material conversion operation and preventing the script from finishing. It's possible for a material's xml to be malformed, which is why this check is needed. """ return True try: if isinstance(self.xml.getroot(), xml.etree.ElementTree.Element): return True else: return False except: return False def parse_xml(self): """ Open and parse the file's xml, storing it for access later. For .ly and .cry files, it will get the xml out of the .zip """ if self.filename.endswith(".cry") or self.filename.endswith(".ly"): zipRead = ZipFile(self.filename, 'r') contents = zipRead.read("levelentities.editor_xml") zipRead.close() # write the contents to a temporary file so we can parse it with ElementTree tmpFile = tempfile.NamedTemporaryFile(delete=False) tmpFile.write(contents) tmpFile.close() self.xml = xml.etree.ElementTree.parse(tmpFile.name) self.gather_elements() os.unlink(tmpFile.name) os.path.exists(tmpFile.name) elif os.path.exists(self.filename): #try: # TODO try-except is supposed to make it so one bad xml doesn't crash the lot # need to clean up stuff so the logging/conversion later doesn't crash # for now, better to crash here so we see where the exception is being thrown self.xml = xml.etree.ElementTree.parse(self.filename) self.gather_elements() #except OSError as err: # print("OS error: {0}".format(err)) # self.xml = None # self.needsConversion = False # self.hadException = True #except ValueError: # print("Could not convert data to an integer.") # self.xml = None # self.needsConversion = False # self.hadException = True #except: # print("Unexpected error:", sys.exc_info()[0]) # self.xml = None # self.needsConversion = False # self.hadException = True def gather_elements(self): """ Once the xml has been parsed, mine through it to find all of the neccessary elements that need to be modified. """ print("starting to parse {0}".format(self.filename)) componentConverters = [] componentConverters.append(Mesh_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir)) componentConverters.append(Actor_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir)) componentConverters.append(Point_Light_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir)) if self.is_valid_xml(): root = self.xml.getroot() if root.tag == "ObjectStream" or True: # First, get a dictionary of child->parent mapping for later use inserting sibling elements self.parent_map = {c:p for p in root.iter('Class') for c in p} # Now go through and look for mesh components for child in root.iter('Class'): # If we run into one of the components we just added, skip it. It doesn't need to be converted, # and it doesn't exist in the pre-built parent_map so it would throw an exception if we tried to access it if child in self.parent_map: parent = self.parent_map[child] for componentConverter in componentConverters: componentConverter.reset() if componentConverter.is_this_the_component_im_looking_for(child, parent): self.needsConversion = True componentConverter.gather_info_for_conversion(child, parent) # TODO - we're about to change the tree structure while iterating, which is apparently undefined but appears to work. Might be better to just build up a list of things that need to be modified, then do a second pass to replace the legacy component # Seems to be okay since we only change or add elements, never remove entirely componentConverter.convert(child, parent) self.xml._setroot(root) print("finished parsing {0}".format(self.filename)) def can_be_converted(self): """ Determines if this material file can be converted by checking if it is using the Illum Shader """ return self.needsConversion def can_write(self): """ Checks to make sure the mtl file is writable. This is to prevent the script from asserting during a save attempt and preventing the script from finishing. """ fullFilePath = self.get_atom_file_path() if os.path.exists(fullFilePath): if os.access(fullFilePath, os.W_OK): return True else: with open(fullFilePath,"a+") as f: f.close() return True return False def get_atom_file_path(self): # This is just a way to optionally create a new file for comparing with the original # TODO: control this via command line atomFileName = self.filename return atomFileName#.replace('.slice', '_atom.slice') def convert(self): """ Creates the new level/slice file """ # TODO - will not work if .slice is part of the path instead of the extension if self.needsConversion: if self.filename.endswith(".cry") or self.filename.endswith(".ly"): # We can't just update the .cry file, we need to rebuild all the contents #Make a temporary file tmpFile, tmpFileName = tempfile.mkstemp(dir=os.path.dirname(self.filename)) os.close(tmpFile) #Create a temporary copy of the .cry file with ZipFile(self.filename, 'r') as zin: with ZipFile(tmpFileName, 'w') as zout: #Loop through the file list and write out every file but level.editor_xml with no modifications #when we hit the level data we want to edit, write it out with the new contents for item in zin.infolist(): if item.filename == "levelentities.editor_xml": xmlString = xml.etree.ElementTree.tostring(self.xml.getroot()) zout.writestr(item, xmlString) else: zout.writestr(item, zin.read(item.filename)) #Remove old cry file and rename the temp file os.remove(self.filename) os.rename(tmpFileName, self.filename) else: self.xml.write(self.get_atom_file_path()) return False def getUpdatedStatsCollector(self): """ Returns the stats collector that was passed in intially, with any modifications that were made """ return self.statsCollector ############################################################################### def main(): '''sys.__name__ wrapper function''' msgStr = "This tool will scan all of your project's level/layer/slice files\n\ convert any compatible legacy components into the equivalent Atom components\n\ This script will overwrite the original files, and will remove the legacy components\n\ upon conversion, decimating the previous contents of those components.\n" commandLineOptions = Common_Command_Line_Options(sys.argv[0], sys.argv[1]) if commandLineOptions.isHelp: print (commandLineOptions.helpString) return start_time = time.time() total_converted = 0 extensionList = [".slice", ".layer", ".ly", ".cry"] fileList = get_file_list(commandLineOptions.projectName, commandLineOptions.includeGems, extensionList, BUILD_PATH, GEMS_PATH) assetCatalogDictionaries = get_asset_catalog_dictionaries(BUILD_PATH, commandLineOptions.projectName) # Create a log file to store converted component file filenames # and to check to see if the component file has already been converted. convertedLogFile = Log_File(filename="{0}\\{1}".format(BUILD_PATH, CONVERTED_LOG_NAME)) # Create a log file to store component file filenames that need conversion # but cannot becuase they are read only. unconvertedLogFile = Log_File(filename="{0}\\{1}".format(BUILD_PATH, UNCONVERTED_LOG_NAME), include_previous = False) statsLogFile = Log_File(filename="{0}\\{1}".format(BUILD_PATH, STATS_LOG_NAME), include_previous = False) statsCollector = Stats_Collector() # Go through each component file to perform the conversion on it print("==============================") componentFileIndex = -1 for componentFileInfo in fileList: componentFileIndex += 1 componentFileName = componentFileInfo.filename copmonentFileProjectDir = componentFileInfo.normalizedProjectDir print(componentFileName) #if convertedLogFile.has_line(componentFileName.lstrip(BUILD_PATH)): # Use this to only convert files that haven't already been converted # print("--> Previously converted, not doing") # continue if commandLineOptions.endsWithStr == "" or componentFileName.lower().endswith(commandLineOptions.endsWithStr.lower()): componentFile = Component_File(componentFileName, copmonentFileProjectDir, assetCatalogDictionaries, statsCollector) if componentFile.can_be_converted(): if commandLineOptions.useP4: subprocess.check_call(['p4', 'edit', componentFileName]) if componentFile.can_write(): componentFile.convert() convertedLogFile.add_line_no_duplicates(componentFile.get_atom_file_path()) print("--> Converted") total_converted += 1 else: unconvertedLogFile.add_line_no_duplicates("{0} - cannot access file (read-only)".format(componentFile.get_atom_file_path())) print("--> Could not write to destination component file (read-only). Not converted.") else: print("--> did not need conversion.") statsCollector = componentFile.getUpdatedStatsCollector() print("\n") # Fill out the stats log statsLogFile.add_line("Mesh/Actor Components without a material overrride: {0}".format(statsCollector.noMaterialOverrideCount)) statsLogFile.add_line("Mesh/Actor Components with a material overrride: {0}".format(statsCollector.materialOverrideCount)) statsLogFile.add_line("Total Mesh/Actor Components: {0}".format(statsCollector.noMaterialOverrideCount + statsCollector.materialOverrideCount)) # Finally, save the log files to disk convertedLogFile.save() unconvertedLogFile.save() statsLogFile.save() total_time = time.time() - start_time log_str = "You can view a list of converted component files in this log file:\n{0}\\{1}".format(BUILD_PATH, CONVERTED_LOG_NAME) unconverted_log_str = "You can view a list of component files that were not converted in this log file:\n{0}\\{1}".format(BUILD_PATH, UNCONVERTED_LOG_NAME) stats_log_str = "You can view a list of component files stats, such as feature and shader usage, in this log file:\n{0}\\{1}".format(BUILD_PATH, STATS_LOG_NAME) print("==============================\n") print("Conversion completed in {0} seconds.\n".format(total_time)) print("Converted {0} component file(s)".format(total_converted)) # Inform the user about the log files print("{0}".format(log_str)) print("{0}".format(unconverted_log_str)) print("{0}".format(stats_log_str)) if __name__ == '__main__': # GLOBAL NOTE: # - All python scripts should execute through a main() function. main()