Remove the legacy renderer and all associated tools (#476)

Remove the legacy renderer and all associated tools:

- Code/CryEngine/Cry3DEngine/*
- Code/CryEngine/RenderDll/*
- Code/Tools/CryFXC/*
- Code/Tools/HLSLCrossCompiler/*
- Code/Tools/HLSLCrossCompilerMETAL/*
- Code/Tools/RC/*
- Code/Tools/ShaderCacheGen/*
- Tools/CrySCompileServer/*
- Tools/PakGenFromRCList/*
This commit is contained in:
bosnichd
2021-04-30 11:22:33 -06:00
committed by GitHub
parent 24fb48ea8c
commit a1319983df
1365 changed files with 0 additions and 505641 deletions
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d1b50f7075e2ee47c8d184e38862a83b880a90fd2ff00da7bb1eb61e24e00f5d
size 4467904
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:414a8b92182fb5b76b41fd4dcd1af43cb804c30dde1d88a2ef9494d5c09f2ef3
size 1910976
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e93ab77a2ea0259eb0eaba86c6f849d2d52cec6d1e77d58eb9581b01eadabf1a
size 1925312
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:af73d401254446d9e5b80b69ad5d042981314832a9bb76f3f179081cb3d40820
size 151232
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e994847e01a6f1e4cbdc5a864616ac262f67ee4f14db194984661a8d927ab7f4
size 4173928
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:54c224dac495c9914b20a51b6c5177e3be612d8567111ac61a4c693292f40cce
size 348672
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e994847e01a6f1e4cbdc5a864616ac262f67ee4f14db194984661a8d927ab7f4
size 4173928
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3a623de58e3d6578f9a2b8104b0d9fdfac75a2b4336bf7633fb11296b2f8faa3
size 160768
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:1997ae4f0b59060a5c0065cea72069e5f9e0b9d25e69cd203fb842713b2a8484
size 203776
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3c11921fca6f631201865d7717a2d3ad8479c6f3df058a7932e7c4daa691bf36
size 460800
@@ -1,420 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import os.path
import sys
import optparse
import xml.etree.ElementTree as ET
from fnmatch import fnmatch
def filter_files(old_list):
new_list = set()
for item in old_list:
# split rc output
if '=' in item:
item_split = item.split('=')
if not len(item_split) == 2:
continue
item = item_split[1]
if not 'GameSDK' in item:
continue
item_split = item.split('GameSDK')
if not len(item_split) == 2:
continue
item = item_split[1][1:]
item = item.replace('\\', '/')
# get rid of double slashes
item = item.replace('//', '/')
# transform any dds into wildcard
if '.dds' in item:
item_split = item.split('.dds')
item = item_split[0] + '.dds\n'
if '.$dds' in item:
item_split = item.split('.$dds')
item = item_split[0] + '.dds\n'
# convert to lower case
item = item.lower()
new_list.add(item)
if item.split('.')[-1].strip() in ['cgf', 'cga', 'skin']:
newFile = item.strip() + 'm\n'
new_list.add(newFile)
return new_list
def read_file_mapping_rule(includeRules, excludeRules, node, levelTag):
for ruleNode in node:
rule = { 'levelTag' : levelTag }
rule['path'] = ruleNode.attrib['Path']
if ruleNode.tag.lower() == 'include':
includeRules.append(rule)
elif ruleNode.tag.lower() == 'exclude':
excludeRules.append(rule)
def read_file_mapping_rules(file_mapping_xml, masterSet, levelList, resultList):
fileMappingTree = ET.parse(file_mapping_xml)
fileMappingRoot = fileMappingTree.getroot()
levelIncludeRules = []
levelExcludeRules = []
globalIncludeRules = []
globalExcludeRules = []
launchIncludeRules = []
for node in fileMappingRoot:
if node.tag.lower() == 'level':
levelName = node.attrib['Name'].lower()
levelTag = levelList[levelName]
read_file_mapping_rule(levelIncludeRules, levelExcludeRules, node, levelTag)
for node in fileMappingRoot:
if node.tag.lower() == 'global':
read_file_mapping_rule(globalIncludeRules, globalExcludeRules, node, 'global')
for node in fileMappingRoot:
if node.tag.lower() == 'launch':
read_file_mapping_rule(launchIncludeRules, [], node, 'launch')
return (levelIncludeRules, levelExcludeRules, globalIncludeRules, globalExcludeRules, launchIncludeRules)
def apply_file_mapping(masterSet, levels, resultList, fileMappingRules):
levelIncludeRules = fileMappingRules[0]
levelExcludeRules = fileMappingRules[1]
globalIncludeRules = fileMappingRules[2]
globalExcludeRules = fileMappingRules[3]
launchIncludeRules = fileMappingRules[4]
allLevelSet = ''
for levelTag in levels:
allLevelSet += levelTag
masterSetList = list(masterSet)
for resource in masterSetList:
levelSet = ''
bLaunchInclude = False
bGlobalInclude = False
bGlobalExclude = False
bLevelInclude = False
bLevelExclude = False
resourceStripped = resource.strip()
for includeRule in launchIncludeRules:
if fnmatch(resourceStripped, includeRule['path']):
bLaunchInclude = True
levelSet = 'Launch'
break
if not bLaunchInclude:
for levelTag in levels:
bResourceInAutoList = resource in levels[levelTag]['resources']
bForceInclude = False
for includeRule in levelIncludeRules:
if (includeRule['levelTag'] == levelTag) and fnmatch(resourceStripped, includeRule['path']):
bLevelInclude = True
bForceInclude = True
break
if not bLevelInclude:
for excludeRule in levelExcludeRules:
if (excludeRule['levelTag'] == levelTag) and fnmatch(resourceStripped, excludeRule['path']):
bLevelExclude = True
break
if bForceInclude or (bResourceInAutoList and not bLevelExclude):
levelSet += levelTag
if not bLevelInclude:
for includeRule in globalIncludeRules:
if fnmatch(resourceStripped, includeRule['path']):
bGlobalInclude = True
levelSet = allLevelSet
break
if not (bLevelInclude or bGlobalInclude):
for excludeRule in globalExcludeRules:
if fnmatch(resourceStripped, excludeRule['path']):
bGlobalExclude = True
break
if levelSet != '' and not bGlobalExclude:
if not levelSet in resultList:
resultList[levelSet] = []
resultList[levelSet].append(resource)
masterSet.remove(resource)
elif bLevelExclude or bGlobalExclude:
masterSet.remove(resource)
def merge_paks(target_game, levels, resultList, pakSizeThreshold, numFilesThreshold):
allLevelList = ''
for levelIndicator in levels:
allLevelList += levelIndicator
smallList = []
bigList = []
for result in resultList.keys():
if result == 'Launch' or result == 'EXTRA':
continue
fileList = resultList[result]
if len(fileList) <= numFilesThreshold:
totalSize = 0
for fileName in fileList:
fullPath = target_game + '\\' + fileName.replace('/', '\\').strip()
if fullPath[-4:] == '.dds':
fullPath = fullPath[:-4] + '.$dds'
fileSize = os.path.getsize(fullPath) if os.path.exists(fullPath) else 0
totalSize += fileSize
if totalSize <= pakSizeThreshold:
smallList.append(result)
else:
bigList.append(result)
else:
bigList.append(result)
for small in smallList:
bestMatch = allLevelList
for big in bigList:
bigSet = set(big)
if all((c in bigSet) for c in small) and len(big) < len(bestMatch):
bestMatch = big
print ' Merged %s.pak with %s.pak' % (small, bestMatch)
fileList = resultList[small]
resultList[bestMatch].extend(fileList)
del resultList[small]
def merge_paks_fixed(levels, resultList, availableList):
allLevelList = ''
for levelIndicator in levels:
allLevelList += levelIndicator
invalidList = []
for result in resultList.keys():
if result == 'Launch' or result == 'EXTRA':
continue
if not result in availableList:
invalidList.append(result)
for invalid in invalidList:
bestMatch = allLevelList
for available in availableList:
availableSet = set(available)
if all((c in availableSet) for c in invalid) and len(available) < len(bestMatch):
bestMatch = available
print ' Merged %s.pak with %s.pak' % (invalid, bestMatch)
fileList = resultList[invalid]
if not bestMatch in resultList:
print '%s NOT FOUND IN RESULT LIST - ADDING IT NOW' % bestMatch
resultList[bestMatch] = fileList
else:
resultList[bestMatch].extend(fileList)
del resultList[invalid]
def filter_file_lists(level, levels, levelIndicator, rcFiles):
for levelIndicator in levels:
level = levels[levelIndicator]
print ' Filtering %s' % level['name']
levels[levelIndicator]['resources'] = filter_files(levels[levelIndicator]['resources'])
print ' Filtering RC log'
rcFiles = filter_files(rcFiles)
return rcFiles
def load_level_files(level, levels, levelIndicator):
for levelIndicator in levels:
level = levels[levelIndicator]
print ' Loading %s (%s)' % (level['name'], level['filename'])
if os.path.exists(level['filename']):
with open(level['filename']) as f:
levels[levelIndicator]['resources'] = set(f.readlines())
else:
print 'LEVEL %s NOT FOUND' % level['filename']
def create_extra_list(masterSet, resultList):
resultList['EXTRA'] = []
for resource in masterSet:
resultList['EXTRA'].append(resource)
def write_result_lists(path_to, resultList):
totalPath = os.path.join(path_to, 'files_TOTAL.txt')
with open(totalPath, 'w') as totalFileHandle:
for result in resultList:
resultPath = os.path.join(path_to, 'files_%s_GFL.txt' % result)
print ' %s -> %s' % (result, resultPath)
with open(resultPath, 'w') as f:
f.writelines(resultList[result])
for resource in resultList[result]:
totalFileHandle.write('%s - %s' % (result, resource))
def read_chunk_xmls(input):
chunks = []
try:
tree = ET.parse(input)
package = tree.getroot()
for chunk in package:
for k, v in chunk.attrib.items():
if k == 'CryLevels':
chunks.append(v)
except Exception as e:
print "Couldn't load chunk XML"
print e
return chunks
def main():
parser = optparse.OptionParser('usage: %prog source_game target_game path_to path_rc_log level_mapping_xml file_mapping_xml install_chunks_source_xml')
(_, args) = parser.parse_args()
if len(args) < 6:
parser.error("incorrect number of arguments")
return 1
print 'Generate fileName lists from level requirements'
# Get arguments
source_game = args[0]
target_game = args[1]
path_to = args[2]
path_rc_log = args[3]
level_mapping_xml = args[4]
file_mapping_xml = args[5]
install_chunks_source_xml = args[6]
patching = False
if not os.path.exists(source_game):
print 'Input path not valid'
return 1
if not os.path.exists(path_rc_log):
print 'RC log not found'
return 1
if not os.path.exists(path_to):
print 'Output path doesn\'t exist, creating it now'
os.makedirs(path_to)
print 'Cleaning output folder'
for fileName in os.listdir(path_to):
os.remove(path_to + '\\' + fileName)
testResultPath = source_game + '\Levels'
levelMappingTree = ET.parse(level_mapping_xml)
levelsRoot = levelMappingTree.getroot()
levelList = {}
for level in levelsRoot:
levelList[level.attrib['Name']] = level.attrib['tag']
if not os.path.exists(testResultPath):
print 'No results in build, exiting'
return 1
if len(levelList) == 0:
print 'Level list, exiting'
return 1
print 'Levels containing resources list:'
levels = {}
for level in levelList:
levelIndicator = levelList[level]
if levelIndicator in levels:
print 'Duplicate found, no support for this. Exiting'
return 1
levels[levelIndicator] = {'name': level, 'filename': os.path.join(testResultPath, level, 'auto_resourcelist_total.txt'), 'resources': set()}
print ' [%s] -> %s' % (levelIndicator, level)
print ''
print 'Loading files in memory'
load_level_files(level, levels, levelIndicator)
print ''
print 'Loading RC log in memory'
with open(path_rc_log, 'r') as f:
rcFiles = f.readlines()
print ''
print 'Filtering files'
rcFiles = filter_file_lists(level, levels, levelIndicator, rcFiles)
print ''
print 'Creating master set'
masterSet = set()
for levelIndicator in levels:
level = levels[levelIndicator]
masterSet |= level['resources']
masterSet |= set(rcFiles)
print 'Done, %s items' % len(masterSet)
resultList = {}
print ''
print 'Reading static rules...'
fileMappingRules = read_file_mapping_rules(file_mapping_xml, masterSet, levelList, resultList)
print 'Going through list to find matches per level...'
apply_file_mapping(masterSet, levels, resultList, fileMappingRules)
print 'Creating extra list...'
create_extra_list(masterSet, resultList)
print 'Done, %s combinations' % len(resultList)
pakSizeThreshold = 20000000
numFilesThreshold = 500
print ''
print 'Merging PAKs under threshold size of %d files and %d bytes' % (numFilesThreshold, pakSizeThreshold)
if patching:
print ' DISABLED'
else:
merge_paks(target_game, levels, resultList, pakSizeThreshold, numFilesThreshold)
if patching:
print ''
print 'Merging PAKs using fixed set of chunks'
availableList = read_chunk_xmls(install_chunks_source_xml)
merge_paks_fixed(levels, resultList, availableList)
print ''
print 'Dumping results in %s' % path_to
write_result_lists(path_to, resultList)
print ''
print 'Done, %d PAKs' % len(resultList)
return 0
if __name__ == '__main__':
sys.exit(main())
@@ -1,218 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import os
import os.path
import glob
import sys
import optparse
import subprocess
import xml.etree.ElementTree as ET
import xml.dom.minidom
import glob
def RunCmd(cmd, useShell=False, workingDir=None, silent=False, printOutput=True, stripTrailing=True):
if not silent:
print ''
print 'Running: %s' % ' '.join(cmd)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=useShell, cwd=workingDir)
while True:
output = process.stdout.readline()
if output:
if printOutput:
if stripTrailing:
print output[0:-2] # Print output stripping trailing \n
else:
print output
else:
break
process.stdout.close()
rc = process.wait()
assert rc != None
if not silent:
print 'Program finished with return code: %d \n' % rc
return rc
def createJob(rootFolder, outputPak, jobPath):
jobString = '''
<RCJobs>
<DefaultProperties
rootFolder="%s"
outputPak="%s"
gamedata="Difficulty\*.*;Entities\*.*;Fonts\*.*;Libs\*.*;Materials\*.*;Prefabs\*.*;Levels\*.*xml"
scripts="Scripts\*.*"
geomcache_list="GameRyse\Animations\cinematics\GeomCachesCine1.txt"
/>
<PakJob>
<Job sourceroot="${rootFolder}" input="*.*" zip="${outputPak}" exclude="${gamedata};${scripts};Videos\*.*;Sounds\*.*;Music\*.*;*dds*;*.cax;Libs\UI\*.*;Localization\*.*;_levelpak\*.*;levels\*.*;Animations\Mannequin\*.*" zip_sizesplit="1" />
<Job sourceroot="${rootFolder}" input="*.dds" zip="${outputPak}" listformat="{1}.dds;{1}.$dds;{1}.dds.1;{1}.dds.1a;{1}.dds.2;{1}.dds.2a;{1}.dds.3;{1}.dds.3a;{1}.dds.4;{1}.dds.4a;{1}.dds.5;{1}.dds.5a;{1}.dds.6;{1}.dds.6a;{1}.dds.7;{1}.dds.7a;{1}.dds.8;{1}.dds.8a;{1}.dds.9;{1}.dds.9a" exclude="Libs\UI\*.*;Localization\*.*" zip_sizesplit="1" />
<Job sourceroot="${rootFolder}" input="*.dds" zip="${outputPak}" listformat="{1}.dds.0;{1}.dds.0a" exclude="Libs\UI\*.*;Localization\*.*" sourceminsize="4001" zip_sizesplit="1" />
<Job sourceroot="${rootFolder}" input="*.cax" zip="${outputPak}" zip_compression="0" exclude_listfile="${rootFolder}\..\..\Build\${geomcache_list}" zip_sizesplit="1" />
<Job sourceroot="${rootFolder}" input="Music\*.*" zip="${outputPak}" zip_compression="0" zip_sizesplit="1" />
<Job sourceroot="${rootFolder}" input="Sounds\*.*" zip="${outputPak}" zip_compression="0" zip_sizesplit="1" />
<Job sourceroot="${rootFolder}" input="Videos\*.*" exclude="Videos\Cinematics\Unlockable\*.*" zip="${outputPak}" zip_compression="0" zip_sizesplit="1" />
<!-- Part of the RC Build proccess
<Job sourceroot="${rootFolder}" input="Libs\UI\*.*" zip="${outputPak}" zip_compression="0" exclude="*.dat;Libs\UI\Menus\NewMenu\Screens\CollectiblesComics\*.*;Libs\UI\Menus\NewMenu\Screens\CollectiblesDogtags\*.*;Libs\UI\Menus\NewMenu\Screens\CollectiblesVistas\*.*" zip_sizesplit="1" />
-->
</PakJob>
<Run Job="PakJob"/>
</RCJobs>
''' % (rootFolder, outputPak)
f = open(jobPath, 'w')
f.write(jobString)
f.close()
def generate_chunk_xmls(pakList, input, outputInstall, outputMakePkg):
try:
tree = ET.parse(input)
package = tree.getroot()
launch_chunk_index = 100
chunk_index = 10000
for result in pakList:
pattern, pak = result
chunk = ET.Element('Chunk')
if pattern != 'Launch':
chunk.attrib['Id'] = str(chunk_index)
chunk_index += 1
chunk.attrib['CryMarker'] = 'Level data'
chunk.attrib['CryLevels'] = pattern
else:
chunk.attrib['Id'] = str(launch_chunk_index)
launch_chunk_index += 1
chunk.attrib['CryMarker'] = 'Launch data'
file_group = ET.Element('FileGroup')
file_group.attrib['DestinationPath'] = '\\GameSDK'
file_group.attrib['SourcePath'] = 'GameSDK'
file_group.attrib['Include'] = pak
chunk.append(file_group)
if pattern != 'Launch':
package.append(chunk)
else:
package.insert(0, chunk)
# Insert update alignment chunk
chunk = ET.Element('Chunk')
chunk.attrib['Id'] = "1073741823"
file_group = ET.Element('FileGroup')
file_group.attrib['DestinationPath'] = '\\'
file_group.attrib['SourcePath'] = '.\\'
file_group.attrib['Include'] = "Update.AlignmentChunk"
chunk.append(file_group)
package.append(chunk)
with open(outputInstall,'w+b') as out_handle:
out_handle.write(xml.dom.minidom.parseString(ET.tostring(package)).toprettyxml())
for chunk in package:
for k, v in chunk.attrib.items():
if k[:3] == 'Cry':
del chunk.attrib[k]
with open(outputMakePkg,'w+b') as out_handle:
out_handle.write(xml.dom.minidom.parseString(ET.tostring(package)).toprettyxml())
except Exception as e:
print "Couldn't generate chunk XML"
print e
def main():
parser = optparse.OptionParser('usage: %prog filelist_path game_path rc_path archive_path install_chunks_source_xml install_chunks_target_xml makepkg_chunks_target_xml')
(options, args) = parser.parse_args()
if len(args) < 7:
parser.error("incorrect number of arguments")
return 1
print 'Generate PAK files from file lists and game folder into archive path'
# Get arguments
path_from = args[0]
path_game = args[1]
path_rc = args[2]
path_to = args[3]
install_chunks_source_xml = args[4]
install_chunks_target_xml = args[5]
makepkg_chunks_target_xml = args[6]
rcExecutable = os.path.join(args[2], 'rc.exe')
if not os.path.exists(path_from):
print 'Input path not valid'
return 1
if not os.path.exists(path_from):
print 'Input path not valid'
return 1
if not os.path.exists(rcExecutable):
print 'RC path not valid'
return 1
if not os.path.exists(path_to):
print 'Output path doesn\'t exist, creating it now'
os.makedirs(path_to)
fileList = glob.glob(os.path.join(path_from, '*GFL.txt'))
print ''
print 'Total number of files: %s' % len(fileList)
print ''
print 'Creating PAK files and storing them in %s' % path_to
pakList = []
for file in fileList:
pattern = file[len(path_from) + 7: -8]
pak_path = os.path.join(path_to, '%s.pak' % pattern)
pak_glob_path = os.path.join(path_to, '%s-part*.pak' % pattern)
print ' %s -> %s' % (pattern, pak_path)
createJob(path_game, pak_path, '%s\job_temp.xml' % path_from)
rcCmd = [rcExecutable, '/threads=8', '/logprefix=..\\%s_' % pattern, '/quiet', '/job=job_temp.xml', '/listfile=%s' % file]
RunCmd(rcCmd, workingDir=path_from, silent=False, printOutput=True)
if pattern.lower() != "extra":
patternPakList = glob.glob(pak_glob_path)
for patternPak in patternPakList:
pakList.append((pattern, os.path.basename(patternPak)))
if os.path.exists(pak_path):
pakList.append((pattern, pattern + '.pak'))
patching = False
print ''
print 'Writing chunk XMLs %s and %s' % (install_chunks_target_xml, makepkg_chunks_target_xml)
if patching:
print ' DISABLED'
else:
generate_chunk_xmls(pakList, install_chunks_source_xml, install_chunks_target_xml, makepkg_chunks_target_xml)
print ''
print 'Done!'
if __name__ == '__main__':
sys.exit(main())