birkeh
/
o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
development
monroegm-disable-blank-issue-2
main
2111.2
2111.1
2107.1
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '1dd1f85d2b'
${ noResults }
o3de/Code/Tools/RC/ResourceCompiler/PakManager.cpp

935 lines
39 KiB
C++
Raw Blame History

/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "ResourceCompiler_precompiled.h"
#include "PakManager.h"
#if defined(AZ_PLATFORM_WINDOWS)
#include <Shlwapi.h> // PathRelativePathTo(), PathCanonicalize()
#pragma message("Note: including Shlwapi.lib")
#pragma comment(lib, "Shlwapi.lib")
#endif
#include "IRCLog.h"
#include "FileUtil.h"
#include <ResourceCompiler.h> // for functions like GetSourceRootsReversed
#include "CryCrc32.h"
#include "ZipEncryptor.h"
#include "ThreadUtils.h"
#include <AzCore/std/algorithm.h>
#include <AzCore/IO/SystemFile.h>
//////////////////////////////////////////////////////////////////////////
PakManager::PakManager(IProgress* pProgress)
: m_pProgress(pProgress)
{
}
PakManager::~PakManager()
{
}
void PakManager::RegisterKeys(IResourceCompiler* pRC)
{
pRC->RegisterKey("split_listfile_to_zips", "split a list file into multiple zip files");
pRC->RegisterKey("zip", "Compress source files into the zip file specified with this parameter");
pRC->RegisterKey("zip_encrypt", "Encrypts headers of zip files. Disabled by default.");
pRC->RegisterKey("zip_encrypt_key", "Specifies a 128-bit key in hexadecimal format: 32-character string. Low endian format.");
pRC->RegisterKey("zip_encrypt_content", "Encrypts files inside of zip. Works only when zip_encrypt enabled. Disabled by default.");
pRC->RegisterKey("zip_compression", "Specify compression level for zipped files. [0-9] 0=no compression, 9=max compression. Default is 6.");
pRC->RegisterKey("zip_sort", "Define sorting type when adding files to the pak, currently supported:\n"
"nosort, size, streaming, suffix, alphabetically. Alphabetically is default.");
pRC->RegisterKey("zip_split", "Define split type for distributing files into different paks automatically, currently supported:\n"
"original, basedir, streaming, suffix. 'original' is default, except for streaming for which it is streaming.");
pRC->RegisterKey("zip_maxsize", "Maximum compressed size of the zip in KBs");
pRC->RegisterKey("zip_sizesplit", "Split zip files automatically when the maximum compressed size (configured or supported) has been reached");
pRC->RegisterKey("zip_alignment", "Alignment of files inside zip. Default is 1 byte.");
pRC->RegisterKey("zip_new", "Forces creation of new zip file overwriting existing one");
pRC->RegisterKey("FolderInZip", "Put source files into this specified folder inside of zip file (see 'zip' command)");
pRC->RegisterKey("sourceminsize", "only copy or zip a source file if its size is greater or equal than the size specified. used with 'copyonly' and 'zip' commands.");
pRC->RegisterKey("sourcemaxsize", "only copy or zip a source file if its size is less or equal than the size specified. used with 'copyonly' and 'zip' commands.");
pRC->RegisterKey("unzip", "Decompress source file into the specified folder with this parameter");
}
//////////////////////////////////////////////////////////////////////////
IPakSystem* PakManager::GetPakSystem()
{
return &m_pPakSystem;
}
//////////////////////////////////////////////////////////////////////////
unsigned PakManager::GetMaxThreads() const
{
return AZStd::GetMax<unsigned>(1, AZStd::thread::hardware_concurrency() / 2);
}
//////////////////////////////////////////////////////////////////////////
bool PakManager::HasPakFiles() const
{
return m_zipFiles.size() > 0;
}
//////////////////////////////////////////////////////////////////////////
PakManager::ECallResult PakManager::CompileFilesIntoPaks(
const IConfig* config,
const std::vector<RcFile>& m_allFiles)
{
{
const string pakFilePath = config->GetAsString("split_listfile_to_zips", "", "");
if (!pakFilePath.empty())
{
std::vector<string> sourceRootsReversed;
ResourceCompiler::GetSourceRootsReversed(config, sourceRootsReversed);
return SplitListFileToPaks(config, sourceRootsReversed, m_allFiles, pakFilePath);
}
}
{
const string pakFilename = config->GetAsString("zip", "", "");
if (!pakFilename.empty())
{
const string folderInPak = config->GetAsString("FolderInZip", "", "");
const bool bUpdate = !config->GetAsBool("zip_new", false, true); // forces to recreate pak-file
return CreatePakFile(config, m_allFiles, folderInPak, pakFilename, bUpdate);
}
}
{
const string unzipFolder = config->GetAsString("unzip", "", "");
if (!unzipFolder.empty())
{
return UnzipPakFile(config, m_allFiles, unzipFolder);
}
}
return eCallResult_Skipped;
}
//////////////////////////////////////////////////////////////////////////
PakManager::ECallResult PakManager::SplitListFileToPaks(
const IConfig* config,
const std::vector<string>& sourceRootsReversed,
const std::vector<RcFile>& files,
const string& pakFilePath)
{
typedef std::map<string, std::vector<RcFile> > TSplitListMap;
TSplitListMap splitListMap;
for (size_t i = 0; i < files.size(); ++i)
{
const string line(files[i].m_sourceInnerPathAndName);
const size_t splitter = line.find_first_of("|;,");
if (splitter == string::npos)
{
continue;
}
string groupName = line.substr(0, splitter);
string fileName = line.substr(splitter + 1);
groupName.Trim();
fileName.Trim();
const string sourceLeftPath = ResourceCompiler::FindSuitableSourceRoot(sourceRootsReversed, fileName);
splitListMap[groupName].push_back(RcFile(sourceLeftPath, fileName, ""));
}
for (TSplitListMap::iterator it = splitListMap.begin(); it != splitListMap.end(); ++it)
{
const string& groupName = it->first;
std::vector<RcFile>& fileList = it->second;
const string pakFilename = pakFilePath + groupName + ".pak";
CreatePakFile(config, fileList, "", pakFilename, true);
}
return eCallResult_Succeeded;
}
//////////////////////////////////////////////////////////////////////////
PakManager::ECallResult PakManager::DeleteFilesFromPaks(
const IConfig* config,
const std::vector<string>& deletedTargetFiles)
{
if (HasPakFiles() && deletedTargetFiles.size())
{
RCLog("Deleting files from zip archives");
return SynchronizePaks(config, deletedTargetFiles);
}
return eCallResult_Skipped;
}
PakManager::ECallResult PakManager::SynchronizePaks(
const IConfig* config,
const std::vector<string>& deletedTargetFiles)
{
const int nTotalToScan = m_zipFiles.size() * deletedTargetFiles.size();
const int zipFileAlignment = config->GetAsInt("zip_alignment", 1, 1);
int nDeletedInZip = 0;
int nScannedFiles = 0;
// If we created some zip file, check if files need to be deleted from them.
for (size_t nzip = 0; (nzip < m_zipFiles.size()) && (nTotalToScan > 0); ++nzip)
{
string zipFilename = m_zipFiles[nzip];
PakSystemArchive* pPakFile = GetPakSystem()->OpenArchive(zipFilename.c_str(), zipFileAlignment);
if (pPakFile)
{
const string progress = string("Deleting files from ") + zipFilename;
char szRelative[MAX_PATH];
string zipFileDir = PathHelpers::GetDirectory(zipFilename);
for (size_t i = 0; i < deletedTargetFiles.size(); ++i)
{
string filename = deletedTargetFiles[i];
#if defined(AZ_PLATFORM_WINDOWS)
// Detect path offset of the zip location to source folder
if (TRUE == PathRelativePathTo(szRelative, zipFileDir.c_str(), FILE_ATTRIBUTE_DIRECTORY, filename.c_str(), FILE_ATTRIBUTE_NORMAL))
{
char szRelative2[MAX_PATH];
PathCanonicalize(szRelative2, szRelative);
filename = szRelative2;
}
#else
//TODO: Needs implementation for other platforms via localFileIo. Adding an assert for now in case this code gets run.
assert(0);
#endif
++nScannedFiles;
if (GetPakSystem()->DeleteFromArchive(pPakFile, filename.c_str()))
{
++nDeletedInZip;
RCLog("Remove file from zip: [%s] %s", zipFilename.c_str(), filename.c_str());
}
m_pProgress->ShowProgress(progress.c_str(), nScannedFiles, nTotalToScan);
}
GetPakSystem()->CloseArchive(pPakFile);
}
}
return eCallResult_Succeeded;
}
//////////////////////////////////////////////////////////////////////////
PakManager::ECallResult PakManager::CreatePakFile(
const IConfig* config,
const std::vector<RcFile>& sourceFiles,
const string& folderInPak,
const string& requestedPakFilename,
bool bUpdate)
{
const int iVerbose = config->GetAsInt("verbose", 0, 1);
const bool bSkipMissing = config->GetAsBool("skipmissing", false, true);
if (iVerbose > 0)
{
RCLog("CreatingPakFile %s ...", requestedPakFilename.c_str());
}
PakHelpers::ESortType eSortType = PakHelpers::eSortType_Alphabetically;
const string sortType = config->GetAsString("zip_sort", "", "");
if (!sortType.empty())
{
if (StringHelpers::EqualsIgnoreCase(sortType, "nosort"))
{
eSortType = PakHelpers::eSortType_NoSort;
}
else if (StringHelpers::EqualsIgnoreCase(sortType, "size"))
{
eSortType = PakHelpers::eSortType_Size;
}
else if (StringHelpers::EqualsIgnoreCase(sortType, "streaming"))
{
eSortType = PakHelpers::eSortType_Streaming;
}
else if (StringHelpers::EqualsIgnoreCase(sortType, "suffix"))
{
eSortType = PakHelpers::eSortType_Suffix;
}
else if (StringHelpers::EqualsIgnoreCase(sortType, "alphabetically"))
{
eSortType = PakHelpers::eSortType_Alphabetically;
}
else
{
RCLogError("Invalid zip_sort argument: '%s'. Creating of pak failed.", sortType.c_str());
return eCallResult_BadArgs;
}
}
PakHelpers::ESplitType eSplitType = PakHelpers::eSplitType_Original;
if (eSortType == PakHelpers::eSortType_Streaming)
{
eSplitType = PakHelpers::eSplitType_ExtensionMipmap;
}
const string splitType = config->GetAsString("zip_split", "", "");
if (!splitType.empty())
{
if (StringHelpers::EqualsIgnoreCase(splitType, "original"))
{
eSplitType = PakHelpers::eSplitType_Original;
}
else if (StringHelpers::EqualsIgnoreCase(splitType, "basedir"))
{
eSplitType = PakHelpers::eSplitType_Basedir;
}
else if (StringHelpers::EqualsIgnoreCase(splitType, "streaming"))
{
eSplitType = PakHelpers::eSplitType_ExtensionMipmap;
}
else if (StringHelpers::EqualsIgnoreCase(splitType, "suffix"))
{
eSplitType = PakHelpers::eSplitType_Suffix;
}
else
{
RCLogError("Invalid zip_split argument: '%s'. Creating of pak failed.", splitType.c_str());
return eCallResult_BadArgs;
}
}
string platformPakFilename = PathHelpers::ToPlatformPath(requestedPakFilename);
if (!FileUtil::EnsureDirectoryExists(PathHelpers::GetDirectory(platformPakFilename).c_str()))
{
RCLogError("Failed creating directory for %s", platformPakFilename.c_str());
return eCallResult_Failed;
}
std::map<string, std::vector<PakHelpers::PakEntry> > fileMap;
{
const size_t nCount = PakHelpers::CreatePakEntryList(sourceFiles, fileMap, eSortType, eSplitType, platformPakFilename);
if (nCount == 0)
{
return eCallResult_Failed;
}
RCLog("Requested %u files to be packed. Found %u valid files to add.", sourceFiles.size(), nCount);
}
std::set<unsigned int> crc32set;
const bool name_as_crc32 = config->GetAsBool("name_as_crc32", false, true);
const int nMinZipSize = sizeof(ZipFile::CDREnd); // size of an empty CDR
const int nMaxZipSize = config->GetAsInt("zip_maxsize", 0, 0) * 1024;
const bool bSplitOnSizeOverflow = config->GetAsBool("zip_sizesplit", false, true);
const int nMaxSrcSize = config->GetAsInt("sourcemaxsize", -1, -1);
const int nMinSrcSize = config->GetAsInt("sourceminsize", 0, 0);
const int zipCompressionLevel = config->GetAsInt("zip_compression", 6, 6);
const bool useFastestDecompressionCodec = config->GetAsBool("use_fastest", false, false);
ECallResult bResult = eCallResult_Succeeded;
for (std::map<string, std::vector<PakHelpers::PakEntry> >::iterator it = fileMap.begin(); it != fileMap.end(); ++it)
{
const string& pakFilename = it->first;
std::vector<PakHelpers::PakEntry>& files = it->second;
RCLog("Found %u valid files to add to zip file %s", files.size(), pakFilename.c_str());
AZ::IO::LocalFileIO localFileIO;
if (!bUpdate)
{
#if defined(AZ_PLATFORM_WINDOWS)
// Delete old pak file.
::SetFileAttributes(pakFilename.c_str(), FILE_ATTRIBUTE_ARCHIVE);
#endif
localFileIO.Remove(pakFilename.c_str());
}
if (!FileUtil::EnsureDirectoryExists(PathHelpers::GetDirectory(pakFilename).c_str()))
{
RCLogError("Failed creating directory for %s", pakFilename.c_str());
return eCallResult_Failed;
}
const int zipFileAlignment = config->GetAsInt("zip_alignment", 1, 1);
const bool zipEncrypt = config->GetAsBool("zip_encrypt", false, true);
const bool zipEncryptContent = config->GetAsBool("zip_encrypt_content", false, true);
const string zipEncryptKey = config->GetAsString("zip_encrypt_key", "", "");
uint32 encryptionKey[4];
if (!zipEncryptKey.empty())
{
if (!ZipEncryptor::ParseKey(encryptionKey, zipEncryptKey.c_str()))
{
RCLogError("Misformed zip_encrypt_key: expected 128-bit integer in hexadecimal format (32 character)");
return eCallResult_Failed;
}
}
// check if pak is multi-part and redirect it before opening the pak
bool bMultiPartPak = false;
string pakFilenameToWrite = pakFilename;
if (bSplitOnSizeOverflow)
{
string pakFilenameMultiPart = pakFilename;
pakFilenameMultiPart.replace(".pak", "-part0.pak");
if (FileUtil::FileExists(pakFilenameMultiPart))
{
RCLog("Found explicit multi-part zip, writing to zip file %s instead", pakFilenameMultiPart.c_str());
bMultiPartPak = true;
pakFilenameToWrite = pakFilenameMultiPart;
}
}
const size_t numFiles = files.size();
size_t numFilesAdded = 0;
size_t numFilesUpToDate = 0;
size_t numFilesSkipped = 0;
size_t numFilesMissing = 0;
size_t numFilesFailed = 0;
const string sProgressOp = "Adding files to zip file " + pakFilenameToWrite;
m_pProgress->StartProgress();
std::vector<string> realFilenames;
std::vector<string> filenamesInZip;
realFilenames.reserve(numFiles);
filenamesInZip.reserve(numFiles);
// create list of filenames
for (size_t i = 0; i < numFiles; ++i)
{
string sFileNameInZip = PathHelpers::RemoveDuplicateSeparators(PathHelpers::ToPlatformPath(PathHelpers::Join(folderInPak, files[i].m_rcFile.m_sourceInnerPathAndName)));
const string sRealFilename = PathHelpers::Join(files[i].m_rcFile.m_sourceLeftPath, files[i].m_rcFile.m_sourceInnerPathAndName);
// Skip files with extensions starting with "$" or "pak".
{
const string ext = PathHelpers::FindExtension(sRealFilename);
if (!ext.empty() && (ext[0] == '$' || _stricmp(ext, "pak") == 0))
{
++numFilesSkipped;
continue;
}
}
if (name_as_crc32)
{
const unsigned int crc32 = CCrc32::ComputeLowercase(sFileNameInZip.c_str());
if (crc32set.find(crc32) != crc32set.end())
{
RCLogError("Duplicate CRC32 code %X for file %s when creating Pak File: %s", crc32, sFileNameInZip.c_str(), pakFilenameToWrite.c_str());
++numFilesFailed;
bResult = eCallResult_Erroneous;
break;
}
crc32set.insert(crc32);
sFileNameInZip.Format("%X", crc32);
}
filenamesInZip.push_back(sFileNameInZip);
realFilenames.push_back(sRealFilename);
}
std::vector<const char*> realFilenamePtrs;
std::vector<const char*> filenameInZipPtrs;
std::vector<const char*> filenameInZipPtrsForDelete;
size_t filenameCount = realFilenames.size();
assert(filenameCount == filenamesInZip.size());
realFilenamePtrs.resize(filenameCount);
filenameInZipPtrs.resize(filenameCount);
for (size_t i = 0; i < filenameCount; ++i)
{
realFilenamePtrs[i] = realFilenames[i].c_str();
filenameInZipPtrs[i] = filenamesInZip[i].c_str();
}
size_t currentPakPart = 0;
bool bKeepTrying = false;
do
{
// Add them to pak file.
PakSystemArchive* pPakFile = GetPakSystem()->OpenArchive(pakFilenameToWrite.c_str(), zipFileAlignment, zipEncrypt, zipEncryptKey.empty() ? 0 : encryptionKey);
if (!pPakFile)
{
// Problem to accss the file? it could have get corrupted so try to delete the file and recreate it
AZ::IO::SystemFile::Delete(pakFilenameToWrite.c_str());
pPakFile = GetPakSystem()->OpenArchive(pakFilenameToWrite.c_str(), zipFileAlignment, zipEncrypt, zipEncryptKey.empty() ? 0 : encryptionKey);
if (!pPakFile)
{
RCLogError("Error: Failed to create zip file %s", pakFilenameToWrite.c_str());
return eCallResult_Failed;
}
}
// submit files for packing
{
struct ZipErrorReporter
: public ZipDir::IReporter
{
IProgress& m_progress;
const char* m_zipFilename;
bool m_bVerbose;
size_t m_fileCount;
size_t& m_numFilesAdded;
size_t& m_numFilesUpToDate;
size_t& m_numFilesSkipped;
size_t& m_numFilesMissing;
size_t& m_numFilesFailed;
ZipErrorReporter(IProgress& progress, const char* zipFilename, int numFiles, bool verbose,
size_t& numFilesAdded, size_t& numFilesUpToDate, size_t& numFilesSkipped, size_t& numFilesMissing, size_t& numFilesFailed)
: m_progress(progress)
, m_zipFilename(zipFilename)
, m_bVerbose(verbose)
, m_fileCount(numFiles)
, m_numFilesAdded(numFilesAdded)
, m_numFilesUpToDate(numFilesUpToDate)
, m_numFilesSkipped(numFilesSkipped)
, m_numFilesMissing(numFilesMissing)
, m_numFilesFailed(numFilesFailed)
{
}
virtual void ReportAdded(const char* filename)
{
if (m_bVerbose)
{
RCLog("Zip [%s]: added file %s", m_zipFilename, filename);
}
++m_numFilesAdded;
ShowProgress();
}
virtual void ReportMissing(const char* filename)
{
RCLogWarning("Zip [%s]: missing file %s", m_zipFilename, filename);
++m_numFilesMissing;
ShowProgress();
}
virtual void ReportUpToDate(const char* filename)
{
if (m_bVerbose)
{
RCLog("Zip [%s]: up to date %s", m_zipFilename, filename);
}
++m_numFilesUpToDate;
ShowProgress();
}
virtual void ReportSkipped(const char* filename)
{
RCLog("Zip [%s]: skipped %s", m_zipFilename, filename);
++m_numFilesSkipped;
ShowProgress();
}
virtual void ReportFailed(const char* filename, const char* reason)
{
RCLog("Zip [%s]: failed to add %s. %s", m_zipFilename, filename, reason);
++m_numFilesFailed;
ShowProgress();
}
virtual void ReportSpeed(double speed)
{
RCLog("Zip [%s] compression speed: %.2f MB/sec", m_zipFilename, speed / 1024.0 / 1024.0);
ShowProgress();
}
void ShowProgress()
{
size_t processedFiles = m_numFilesAdded + m_numFilesUpToDate + m_numFilesFailed + m_numFilesSkipped + m_numFilesMissing;
m_progress.ShowProgress("Adding files into pak", processedFiles, m_fileCount);
}
};
struct ZipSizeSplitter
: public ZipDir::ISplitter
{
int m_fileLast;
int m_fileCount;
size_t m_fileSizeLimit;
size_t m_fileSizeThreshold;
ZipSizeSplitter(int filenameCount, size_t filesizeLimit)
: m_fileLast(filenameCount - 1)
, m_fileCount(filenameCount - 1)
, m_fileSizeLimit(filesizeLimit)
, m_fileSizeThreshold(0)
{
}
virtual bool CheckWriteLimit(size_t total, size_t add, size_t sub) const
{
return ((total - sub) > (m_fileSizeLimit - add));
}
virtual void SetLastFile([[maybe_unused]] size_t total, size_t add, [[maybe_unused]] size_t sub, int last)
{
m_fileLast = last;
m_fileSizeThreshold = m_fileSizeLimit - add;
}
bool HasReachedWriteLimit() const
{
return m_fileLast < m_fileCount;
}
};
ZipErrorReporter errorReporter(*m_pProgress, pakFilenameToWrite.c_str(), numFiles, iVerbose>1,
numFilesAdded, numFilesUpToDate, numFilesSkipped, numFilesMissing, numFilesFailed);
ZipSizeSplitter sizeSplitter(filenameCount, nMaxZipSize ? min(nMaxZipSize, INT_MAX) : INT_MAX);
RCLog("Adding files into %s...", pakFilenameToWrite.c_str());
pPakFile->zip->UpdateMultipleFiles(&realFilenamePtrs[0], &filenameInZipPtrs[0], filenameCount,
zipCompressionLevel, zipEncrypt && zipEncryptContent, nMaxZipSize, nMinSrcSize, nMaxSrcSize,
GetMaxThreads(), &errorReporter, bSplitOnSizeOverflow ? &sizeSplitter : nullptr, useFastestDecompressionCodec);
// divide files in case it has overflown the maximum allowed file-size
if (bSplitOnSizeOverflow)
{
char cPart[16];
char nPart[16];
azsnprintf(cPart, sizeof(cPart), "-part%zu.pak", currentPakPart + 0);
azsnprintf(nPart, sizeof(nPart), "-part%zu.pak", currentPakPart + 1);
const size_t pos = pakFilenameToWrite.find(cPart);
if (!bKeepTrying)
{
// delete previously consumed files from archive
const size_t filenameCountForDelete = filenameInZipPtrsForDelete.size();
for (size_t i = 0; i < filenameCountForDelete; ++i)
{
pPakFile->zip->RemoveFile(filenameInZipPtrsForDelete[i]);
}
}
// move consumed filenames into deletable list
const size_t filenameCountForDelete = filenameInZipPtrsForDelete.size();
const size_t filenameCountConsumed = sizeSplitter.m_fileLast + 1;
filenameInZipPtrsForDelete.resize(filenameCountForDelete + filenameCountConsumed);
memcpy(&filenameInZipPtrsForDelete[filenameCountForDelete],
&filenameInZipPtrs[0], sizeof(filenameInZipPtrs[0]) * filenameCountConsumed);
filenameCount -= filenameCountConsumed;
if (filenameCount > 0)
{
assert(realFilenamePtrs.size() > filenameCountConsumed);
memmove(&realFilenamePtrs[0], &realFilenamePtrs[filenameCountConsumed],
sizeof(realFilenamePtrs[0]) * filenameCount);
assert(filenameInZipPtrs.size() > filenameCountConsumed);
memmove(&filenameInZipPtrs[0], &filenameInZipPtrs[filenameCountConsumed],
sizeof(filenameInZipPtrs[0]) * filenameCount);
}
realFilenamePtrs.resize(filenameCount);
filenameInZipPtrs.resize(filenameCount);
if (!bKeepTrying)
{
// delete skipped over files from archive
for (size_t i = 0; i < filenameCount; ++i)
{
pPakFile->zip->RemoveFile(filenameInZipPtrs[i]);
}
}
if (sizeSplitter.HasReachedWriteLimit())
{
RCLog("Hitting limit of %d bytes on %s, trying reconsolidation...", sizeSplitter.m_fileSizeLimit, pakFilenameToWrite.c_str());
// close archive
GetPakSystem()->CloseArchive(pPakFile);
// check if the reconsolidation of the pak gave more space free than we needed previously
// in case it did, keep adding to the same file, instead of adding it to the next part
const int64 fileSize = FileUtil::GetFileSize(pakFilenameToWrite);
if (fileSize < sizeSplitter.m_fileSizeThreshold)
{
// if we tried keepTrying without effect, don't try again
if ((filenameCountConsumed != 0) || !bKeepTrying)
{
if (pos == string::npos)
{
RCLog("Reconsolidation on %s dropped at least %d bytes below %d, keep adding...", pakFilenameToWrite.c_str(), sizeSplitter.m_fileSizeLimit - sizeSplitter.m_fileSizeThreshold, sizeSplitter.m_fileSizeLimit);
}
else
{
RCLog("Reconsolidation on %s dropped at least %d bytes below %d, keep adding to part %d...", pakFilenameToWrite.c_str(), sizeSplitter.m_fileSizeLimit - sizeSplitter.m_fileSizeThreshold, sizeSplitter.m_fileSizeLimit, currentPakPart);
}
bKeepTrying = true;
continue;
}
}
// rename archive if it's the first time becoming multi-part
if (pos == string::npos)
{
RCLog("Start splitting %s, writing to part %d...", pakFilenameToWrite.c_str(), currentPakPart + 1);
string pakFilenameToRename = pakFilenameToWrite;
pakFilenameToRename.replace(".pak", cPart);
localFileIO.Rename(pakFilenameToWrite.c_str(), pakFilenameToRename.c_str());
bMultiPartPak = true;
pakFilenameToWrite = pakFilenameToRename;
}
else
{
RCLog("Continue splitting %s, writing to part %d...", pakFilenameToWrite.c_str(), currentPakPart + 1);
}
// continue adding to the next part
pakFilenameToWrite.replace(cPart, nPart);
currentPakPart++;
bKeepTrying = false;
continue;
}
assert(filenameCount == 0);
assert(realFilenamePtrs.size() == 0);
assert(filenameInZipPtrs.size() == 0);
}
else
{
filenameCount = 0;
realFilenamePtrs.clear();
filenameInZipPtrs.clear();
}
}
GetPakSystem()->CloseArchive(pPakFile);
const int64 fileSize = FileUtil::GetFileSize(pakFilenameToWrite);
if (fileSize > INT_MAX)
{
RCLogError("PAK File size exceeds 2GB limit. This will not be loaded by Engine: %s", pakFilenameToWrite.c_str());
}
else if (bSplitOnSizeOverflow && bMultiPartPak)
{
// delete all 0 size pak-parts and close holes in the numbering of the parts
bool trailingUntouched = false;
int trailingPakPart = currentPakPart;
for (;; )
{
char cPart[16];
char nPart[16];
azsnprintf(cPart, sizeof(cPart), "-part%d.pak", trailingPakPart + 0);
azsnprintf(nPart, sizeof(nPart), "-part%d.pak", trailingPakPart + 1);
string pakFilenameToDelete = pakFilename;
pakFilenameToDelete.replace(".pak", cPart);
if (!FileUtil::FileExists(pakFilenameToDelete))
{
break;
}
// delete skipped over files from archives not touched
if (trailingUntouched)
{
PakSystemArchive* const pPakFile2 = GetPakSystem()->OpenArchive(pakFilenameToDelete.c_str(), zipFileAlignment, zipEncrypt, zipEncryptKey.empty() ? 0 : encryptionKey);
if (pPakFile2)
{
const size_t filenameCountForDelete = filenameInZipPtrsForDelete.size();
for (size_t i = 0; i < filenameCountForDelete; ++i)
{
pPakFile2->zip->RemoveFile(filenameInZipPtrsForDelete[i]);
}
GetPakSystem()->CloseArchive(pPakFile2);
}
}
const int64 fileSize2 = FileUtil::GetFileSize(pakFilenameToDelete);
if (fileSize2 <= nMinZipSize)
{
// eliminate paks without content (may occur by filtering or reordering)
localFileIO.Remove(pakFilenameToDelete);
// shift successive part-names into the hole left by the deleted pak
for (int q = trailingPakPart;; q++)
{
char cPart2[16];
char nPart2[16];
azsnprintf(cPart2, sizeof(cPart2), "-part%d.pak", q + 0);
azsnprintf(nPart2, sizeof(nPart2), "-part%d.pak", q + 1);
string pakFilenameToReplace = pakFilename;
string pakFilenameToRename = pakFilename;
pakFilenameToReplace.replace(".pak", cPart2);
pakFilenameToRename.replace(".pak", nPart2);
if (!FileUtil::FileExists(pakFilenameToRename))
{
break;
}
localFileIO.Rename(pakFilenameToRename.c_str(), pakFilenameToReplace.c_str());
}
}
else
{
++trailingPakPart;
}
trailingUntouched = true;
}
{
string pakFilenameFirstPart = pakFilename;
string pakFilenameNextPart = pakFilename;
pakFilenameFirstPart.replace(".pak", "-part0.pak");
pakFilenameNextPart.replace(".pak", "-part1.pak");
// remove part-suffix if just one part exists after cleanup
if (FileUtil::FileExists(pakFilenameFirstPart) &&
!FileUtil::FileExists(pakFilenameNextPart))
{
localFileIO.Rename(pakFilenameFirstPart.c_str(), pakFilename.c_str());
m_zipFiles.push_back(pakFilename);
}
// register all parts of the pak for sucessive operations
else
{
for (int q = 0; q < trailingPakPart; q++)
{
char cPart[16];
azsnprintf(cPart, sizeof(cPart), "-part%d.pak", q + 0);
string pakFilenamePart = pakFilename;
pakFilenamePart.replace(".pak", cPart);
m_zipFiles.push_back(pakFilenamePart);
}
}
}
}
else if (fileSize <= nMinZipSize)
{
// eliminate paks without content (may occur by filtering)
localFileIO.Remove(pakFilenameToWrite);
}
else
{
// Add this zip to the array.
m_zipFiles.push_back(pakFilenameToWrite);
}
bKeepTrying = false;
} while (filenameCount);
RCLog("Finished adding %d files to zip file %s:",
numFiles, pakFilename.c_str());
RCLog(" %d added, %d up-to-date, %d skipped, %d missing, %d failed",
numFilesAdded, numFilesUpToDate, numFilesSkipped, numFilesMissing, numFilesFailed);
}
return bResult;
}
struct UnpakParameters
{
typedef AZStd::function<void(bool, const string&)> OnFinishCallback;
ZipDir::CachePtr m_cache;
string m_srcFile;
string m_destFolder;
OnFinishCallback& m_onFinishCb;
};
PakManager::ECallResult PakManager::UnzipPakFile(const IConfig* config, const std::vector<RcFile>& sourceFiles, const string& unzipFolder)
{
const string zipDecryptKey = config->GetAsString("zip_encrypt_key", "", "");
const int decryptKeySizeInUint32 = 4;
uint32 decryptionKey[decryptKeySizeInUint32];
if (!zipDecryptKey.empty())
{
if (!ZipEncryptor::ParseKey(decryptionKey, zipDecryptKey.c_str()))
{
RCLogError("Misformed zip_encrypt_key: expected 128-bit integer in hexadecimal format (32 character)");
return eCallResult_Failed;
}
}
ThreadUtils::SimpleThreadPool pool(false);
std::vector<UnpakParameters> params;
params.reserve(sourceFiles.size());
// Don't capture parameters in the lambda so it can be converted to a function pointer. Instead pack everything in a struct that is passed as an argument.
auto unpakFunc = [](UnpakParameters* params)
{
params->m_onFinishCb(params->m_cache->UnpakToDisk(params->m_destFolder), params->m_srcFile);
};
AZStd::mutex progressMutex;
size_t currentProgress = 0;
UnpakParameters::OnFinishCallback onFinishCb = [&](bool result, const string& srcFile)
{
AZStd::lock_guard<AZStd::mutex> lock(progressMutex);
string msg = (result ? "Finished unpacking file " : "Failed to unpack file ") + srcFile;
m_pProgress->ShowProgress(msg.c_str(), ++currentProgress, sourceFiles.size());
};
m_pProgress->StartProgress();
for (auto it = sourceFiles.begin(); it != sourceFiles.end(); ++it)
{
const RcFile& pakFile = *it;
#if defined(AZ_PLATFORM_WINDOWS)
const string pakFilePath = PathHelpers::Join(PathHelpers::ToDosPath(pakFile.m_sourceLeftPath), PathHelpers::ToDosPath(pakFile.m_sourceInnerPathAndName));
#elif AZ_TRAIT_OS_PLATFORM_APPLE || defined(AZ_PLATFORM_LINUX)
const string pakFilePath = PathHelpers::Join(PathHelpers::ToUnixPath(pakFile.m_sourceLeftPath), PathHelpers::ToUnixPath(pakFile.m_sourceInnerPathAndName));
#endif
ZipDir::CacheFactory factory(ZipDir::ZD_INIT_FAST, 0);
ZipDir::CachePtr cache = factory.New(pakFilePath.c_str(), zipDecryptKey.empty() ? nullptr : decryptionKey);
size_t index = pakFile.m_sourceInnerPathAndName.find(PathHelpers::GetFilename(pakFile.m_sourceInnerPathAndName));
if (index == string::npos)
{
continue;
}
#if defined(AZ_PLATFORM_WINDOWS)
const string destFolder = PathHelpers::Join(PathHelpers::ToDosPath(unzipFolder), PathHelpers::ToDosPath(pakFile.m_sourceInnerPathAndName.substr(0, index)));
#elif AZ_TRAIT_OS_PLATFORM_APPLE || defined(AZ_PLATFORM_LINUX)
const string destFolder = PathHelpers::Join(PathHelpers::ToUnixPath(unzipFolder), PathHelpers::ToUnixPath(pakFile.m_sourceInnerPathAndName.substr(0, index)));
#endif
params.emplace_back(UnpakParameters{ cache, pakFilePath, destFolder, onFinishCb });
pool.Submit<UnpakParameters>(unpakFunc, &params.back());
}
pool.Start(GetMaxThreads());
pool.WaitAllJobs();
m_pProgress->FinishProgress();
return eCallResult_Succeeded;
}
Reference in New Issue View Git Blame Copy Permalink