From 447832dd81c74eab4927d1f595b029593ea59a0e Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Thu, 16 Sep 2021 11:01:00 -0500 Subject: [PATCH] Updated the GameApplication to mount the engine.pak (#4128) * Updated the GameApplication to mount the engine.pak This allows loading the autoexec.cfg and bootstrap.game...setreg from the engine.pak files The engine.pak is searched for in the following order: /engine.pak, followed by /engine.pak Removed a lot of unused APIs from the AZ::IO::Archive feature suite Updated many of the AZ::IO::Archive classes to use AZ::IO::Path internally. The logic to search for files within an Archive has been updated to use AZ::IO::Path and to remove case-insensitve string comparisons Somehow removed the CryFile dependency on anything Cry Updated the Settings Registry to support reading from the FileIOBase and therefore Archive files in the GameLauncher via the `SetUseFileIO` function Removed AzFramework Dependency on md5 3rdParty library Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Linux build fix Added an include of before the include as it usesnprintf. Added `static` to the constexpr constants in ExtractFileDescription in SettingsRegistryImpl.cpp to fix clang compile issue Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Updated the case used to mount the Engine PAK file in the GameApplication to be Engine.pak to match the other locations where it is mounted Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Updated the proper FFont call to FileIOBase::Size to supply the correct integer type of AZ::u64 instead of size_t This fixes building on platforms where size_t is type defined to be unsigned long Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Fixed segmentation fault in Archive::Unregister when outputing the filename of the Archive file being closed Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Fix calls to OpenPack in the Legacy LevelSystem The LevelSystem was calling the incorrect overload of OpenPack that accepts BindRoot for the mounted level.pak instead of the overload that that passes a memory block object. This was causing the level pak files to be mounted using an invalid directory, causing file accesses inside the level pak to fail. Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Updated the error messages in the ZipDir CacheFactory class to use AZ_Warning directly Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Updated the ArchiveFileIO m_trackedFiles container to store mapped type as an AZ::IO::Path Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> --- Code/Editor/CryEditDoc.cpp | 8 +- Code/Editor/GameExporter.cpp | 19 - .../PerforcePlugin/PerforceSourceControl.cpp | 1 + Code/Editor/PythonEditorEventsBus.h | 3 +- Code/Editor/PythonEditorFuncs.cpp | 9 +- Code/Editor/PythonEditorFuncs.h | 2 +- Code/Editor/Settings.cpp | 3 +- Code/Editor/Settings.h | 2 - Code/Editor/Util/FileUtil.cpp | 29 +- Code/Editor/Util/XmlArchive.cpp | 2 +- .../AzCore/AzCore/Settings/SettingsRegistry.h | 5 + .../AzCore/Settings/SettingsRegistryImpl.cpp | 272 ++++-- .../AzCore/Settings/SettingsRegistryImpl.h | 10 +- .../UnitTest/Mocks/MockSettingsRegistry.h | 1 + Code/Framework/AzCore/AzCore/XML/rapidxml.h | 1 + .../AzFramework/Application/Application.cpp | 65 -- .../AzFramework/Archive/Archive.cpp | 895 ++++-------------- .../AzFramework/AzFramework/Archive/Archive.h | 102 +- .../AzFramework/Archive/ArchiveFileIO.cpp | 42 +- .../AzFramework/Archive/ArchiveFileIO.h | 3 +- .../AzFramework/Archive/ArchiveFindData.cpp | 141 ++- .../AzFramework/Archive/ArchiveFindData.h | 56 +- .../AzFramework/Archive/IArchive.h | 178 +--- .../AzFramework/Archive/INestedArchive.h | 26 +- .../AzFramework/Archive/NestedArchive.cpp | 30 +- .../AzFramework/Archive/NestedArchive.h | 10 +- .../AzFramework/Archive/ZipDirCache.cpp | 70 +- .../AzFramework/Archive/ZipDirCache.h | 12 +- .../Archive/ZipDirCacheFactory.cpp | 95 +- .../AzFramework/Archive/ZipDirCacheFactory.h | 9 +- .../AzFramework/Archive/ZipDirFind.cpp | 83 +- .../AzFramework/Archive/ZipDirFind.h | 15 +- .../AzFramework/Archive/ZipDirList.cpp | 4 +- .../AzFramework/Archive/ZipDirStructures.cpp | 25 +- .../AzFramework/Archive/ZipDirStructures.h | 4 - .../AzFramework/Archive/ZipDirTree.cpp | 57 +- .../AzFramework/Archive/ZipDirTree.h | 29 +- .../AzFramework/IO/LocalFileIO.cpp | 2 +- Code/Framework/AzFramework/CMakeLists.txt | 1 - .../Tests/ArchiveCompressionTests.cpp | 2 +- .../AzFramework/Tests/ArchiveTests.cpp | 256 +---- .../Application/GameApplication.cpp | 32 +- Code/Legacy/CryCommon/CryFile.h | 124 +-- Code/Legacy/CryCommon/CryPath.h | 1 + Code/Legacy/CryCommon/Mocks/ICryPakMock.h | 49 +- Code/Legacy/CrySystem/ConsoleBatchFile.cpp | 6 +- .../CrySystem/LevelSystem/LevelSystem.cpp | 16 +- Code/Legacy/CrySystem/SystemCFG.cpp | 18 +- Code/Legacy/CrySystem/SystemInit.cpp | 26 +- .../CrySystem/WindowsErrorReporting.cpp | 9 +- Code/Legacy/CrySystem/XConsoleVariable.h | 4 +- Code/Legacy/CrySystem/XML/XmlUtils.cpp | 2 +- Code/Legacy/CrySystem/XML/xml.cpp | 25 +- .../SettingsRegistryBuilder.cpp | 43 +- .../AtomLyIntegration/AtomFont/FFont.h | 2 +- .../AtomFont/Code/Source/FFont.cpp | 43 +- .../CommonFeatures/Code/CMakeLists.txt | 1 + 57 files changed, 987 insertions(+), 1993 deletions(-) diff --git a/Code/Editor/CryEditDoc.cpp b/Code/Editor/CryEditDoc.cpp index a61429fc87..07d946c609 100644 --- a/Code/Editor/CryEditDoc.cpp +++ b/Code/Editor/CryEditDoc.cpp @@ -1135,7 +1135,6 @@ bool CCryEditDoc::SaveLevel(const QString& filename) { // if we're saving to a new folder, we need to copy the old folder tree. auto pIPak = GetIEditor()->GetSystem()->GetIPak(); - pIPak->Lock(); const QString oldLevelPattern = QDir(oldLevelFolder).absoluteFilePath("*.*"); const QString oldLevelName = Path::GetFile(GetLevelPathName()); @@ -1199,7 +1198,6 @@ bool CCryEditDoc::SaveLevel(const QString& filename) QFile(filePath).setPermissions(QFile::ReadOther | QFile::WriteOther); }); - pIPak->Unlock(); } // Save level to XML archive. @@ -1813,8 +1811,8 @@ bool CCryEditDoc::BackupBeforeSave(bool force) QString subFolder = theTime.toString("yyyy-MM-dd [HH.mm.ss]"); QString levelName = GetIEditor()->GetGameEngine()->GetLevelName(); - QString backupPath = saveBackupPath + "/" + subFolder + "/"; - gEnv->pCryPak->MakeDir(backupPath.toUtf8().data()); + QString backupPath = saveBackupPath + "/" + subFolder; + AZ::IO::FileIOBase::GetDirectInstance()->CreatePath(backupPath.toUtf8().data()); QString sourcePath = QString::fromUtf8(resolvedLevelPath) + "/"; @@ -2028,7 +2026,7 @@ const char* CCryEditDoc::GetTemporaryLevelName() const void CCryEditDoc::DeleteTemporaryLevel() { QString tempLevelPath = (Path::GetEditingGameDataFolder() + "/Levels/" + GetTemporaryLevelName()).c_str(); - GetIEditor()->GetSystem()->GetIPak()->ClosePacks(tempLevelPath.toUtf8().data(), AZ::IO::IArchive::EPathResolutionRules::FLAGS_ADD_TRAILING_SLASH); + GetIEditor()->GetSystem()->GetIPak()->ClosePacks(tempLevelPath.toUtf8().data()); CFileUtil::Deltree(tempLevelPath.toUtf8().data(), true); } diff --git a/Code/Editor/GameExporter.cpp b/Code/Editor/GameExporter.cpp index 00dc3d8de1..9315505e08 100644 --- a/Code/Editor/GameExporter.cpp +++ b/Code/Editor/GameExporter.cpp @@ -432,25 +432,6 @@ void CGameExporter::ExportFileList(const QString& path, const QString& levelName newFileNode->setAttr("src", handle.m_filename.data()); newFileNode->setAttr("dest", handle.m_filename.data()); newFileNode->setAttr("size", handle.m_fileDesc.nSize); - - unsigned char md5[16]; - AZStd::string filenameToHash = GetIEditor()->GetGameEngine()->GetLevelPath().toUtf8().data(); - filenameToHash += "/"; - filenameToHash += AZStd::string{ handle.m_filename.data(), handle.m_filename.size() }; - if (gEnv->pCryPak->ComputeMD5(filenameToHash.data(), md5)) - { - char md5string[33]; - sprintf_s(md5string, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", - md5[0], md5[1], md5[2], md5[3], - md5[4], md5[5], md5[6], md5[7], - md5[8], md5[9], md5[10], md5[11], - md5[12], md5[13], md5[14], md5[15]); - newFileNode->setAttr("md5", md5string); - } - else - { - newFileNode->setAttr("md5", ""); - } } } } while (handle = gEnv->pCryPak->FindNext(handle)); diff --git a/Code/Editor/Plugins/PerforcePlugin/PerforceSourceControl.cpp b/Code/Editor/Plugins/PerforcePlugin/PerforceSourceControl.cpp index 10c43c3d1b..dd9ccaa832 100644 --- a/Code/Editor/Plugins/PerforcePlugin/PerforceSourceControl.cpp +++ b/Code/Editor/Plugins/PerforcePlugin/PerforceSourceControl.cpp @@ -6,6 +6,7 @@ * */ +#include #include "CryFile.h" #include "PerforceSourceControl.h" #include "PasswordDlg.h" diff --git a/Code/Editor/PythonEditorEventsBus.h b/Code/Editor/PythonEditorEventsBus.h index 5107a1c9cc..ffc357e64d 100644 --- a/Code/Editor/PythonEditorEventsBus.h +++ b/Code/Editor/PythonEditorEventsBus.h @@ -8,6 +8,7 @@ */ #pragma once +#include #include namespace AzToolsFramework @@ -138,7 +139,7 @@ namespace AzToolsFramework /* * Finds a pak file name for a given file. */ - virtual const char* GetPakFromFile(const char* filename) = 0; + virtual AZ::IO::Path GetPakFromFile(const char* filename) = 0; /* * Prints the message to the editor console window. diff --git a/Code/Editor/PythonEditorFuncs.cpp b/Code/Editor/PythonEditorFuncs.cpp index 200fd28f87..455cdfa17d 100644 --- a/Code/Editor/PythonEditorFuncs.cpp +++ b/Code/Editor/PythonEditorFuncs.cpp @@ -625,7 +625,7 @@ namespace } ////////////////////////////////////////////////////////////////////////// - const char* PyGetPakFromFile(const char* filename) + AZ::IO::Path PyGetPakFromFile(const char* filename) { auto pIPak = GetIEditor()->GetSystem()->GetIPak(); AZ::IO::HandleType fileHandle = pIPak->FOpen(filename, "rb"); @@ -633,8 +633,9 @@ namespace { throw std::logic_error("Invalid file name."); } - const char* pArchPath = pIPak->GetFileArchivePath(fileHandle); + AZ::IO::Path pArchPath = pIPak->GetFileArchivePath(fileHandle); pIPak->FClose(fileHandle); + return pArchPath; } @@ -1040,7 +1041,7 @@ namespace AzToolsFramework return PySetAxisConstraint(pConstrain); } - const char* PythonEditorComponent::GetPakFromFile(const char* filename) + AZ::IO::Path PythonEditorComponent::GetPakFromFile(const char* filename) { return PyGetPakFromFile(filename); } @@ -1114,7 +1115,7 @@ namespace AzToolsFramework addLegacyGeneral(behaviorContext->Method("get_axis_constraint", PyGetAxisConstraint, nullptr, "Gets axis.")); addLegacyGeneral(behaviorContext->Method("set_axis_constraint", PySetAxisConstraint, nullptr, "Sets axis.")); - addLegacyGeneral(behaviorContext->Method("get_pak_from_file", PyGetPakFromFile, nullptr, "Finds a pak file name for a given file.")); + addLegacyGeneral(behaviorContext->Method("get_pak_from_file", [](const char* filename) -> AZStd::string { return PyGetPakFromFile(filename).Native(); }, nullptr, "Finds a pak file name for a given file.")); addLegacyGeneral(behaviorContext->Method("log", PyLog, nullptr, "Prints the message to the editor console window.")); diff --git a/Code/Editor/PythonEditorFuncs.h b/Code/Editor/PythonEditorFuncs.h index 97ad8829ba..ef0c1327fa 100644 --- a/Code/Editor/PythonEditorFuncs.h +++ b/Code/Editor/PythonEditorFuncs.h @@ -91,7 +91,7 @@ namespace AzToolsFramework void SetAxisConstraint(AZStd::string_view pConstrain) override; - const char* GetPakFromFile(const char* filename) override; + AZ::IO::Path GetPakFromFile(const char* filename) override; void Log(const char* pMessage) override; diff --git a/Code/Editor/Settings.cpp b/Code/Editor/Settings.cpp index 80d248e1bf..05a6960695 100644 --- a/Code/Editor/Settings.cpp +++ b/Code/Editor/Settings.cpp @@ -171,7 +171,6 @@ SEditorSettings::SEditorSettings() bBackupOnSave = true; backupOnSaveMaxCount = 3; bApplyConfigSpecInEditor = true; - useLowercasePaths = 0; showErrorDialogOnLoad = 1; consoleBackgroundColorTheme = AzToolsFramework::ConsoleColorTheme::Dark; @@ -887,6 +886,7 @@ void SEditorSettings::Load() ////////////////////////////////////////////////////////////////////////// AZ_CVAR(bool, ed_previewGameInFullscreen_once, false, nullptr, AZ::ConsoleFunctorFlags::IsInvisible, "Preview the game (Ctrl+G, \"Play Game\", etc.) in fullscreen once"); +AZ_CVAR(bool, ed_lowercasepaths, false, nullptr, AZ::ConsoleFunctorFlags::Null, "Convert CCryFile paths to lowercase on Open"); void SEditorSettings::PostInitApply() { @@ -898,7 +898,6 @@ void SEditorSettings::PostInitApply() // Create CVars. REGISTER_CVAR2("ed_highlightGeometry", &viewports.bHighlightMouseOverGeometry, viewports.bHighlightMouseOverGeometry, 0, "Highlight geometry when mouse over it"); REGISTER_CVAR2("ed_showFrozenHelpers", &viewports.nShowFrozenHelpers, viewports.nShowFrozenHelpers, 0, "Show helpers of frozen objects"); - REGISTER_CVAR2("ed_lowercasepaths", &useLowercasePaths, useLowercasePaths, 0, "generate paths in lowercase"); gEnv->pConsole->RegisterInt("fe_fbx_savetempfile", 0, 0, "When importing an FBX file into Facial Editor, this will save out a conversion FSQ to the Animations/temp folder for trouble shooting"); REGISTER_CVAR2_CB("ed_toolbarIconSize", &gui.nToolbarIconSize, gui.nToolbarIconSize, VF_NULL, "Override size of the toolbar icons 0-default, 16,32,...", ToolbarIconSizeChanged); diff --git a/Code/Editor/Settings.h b/Code/Editor/Settings.h index a408822129..8bf22b43e5 100644 --- a/Code/Editor/Settings.h +++ b/Code/Editor/Settings.h @@ -340,8 +340,6 @@ AZ_POP_DISABLE_DLL_EXPORT_BASECLASS_WARNING //! how many save backups to keep int backupOnSaveMaxCount; - int useLowercasePaths; - ////////////////////////////////////////////////////////////////////////// // Autobackup. ////////////////////////////////////////////////////////////////////////// diff --git a/Code/Editor/Util/FileUtil.cpp b/Code/Editor/Util/FileUtil.cpp index 610a9c6e16..eeb6912acf 100644 --- a/Code/Editor/Util/FileUtil.cpp +++ b/Code/Editor/Util/FileUtil.cpp @@ -149,12 +149,13 @@ bool CFileUtil::ExtractFile(QString& file, bool bMsgBoxAskForExtraction, const c // Check if in pack. if (cryfile.IsInPak()) { - const char* sPakName = cryfile.GetPakPath(); - if (bMsgBoxAskForExtraction) { + AZ::IO::FixedMaxPath sPakName{ cryfile.GetPakPath() }; // Cannot edit file in pack, suggest to extract it for editing. - if (QMessageBox::critical(QApplication::activeWindow(), QString(), QObject::tr("File %1 is inside a PAK file %2\r\nDo you want it to be extracted for editing ?").arg(file, sPakName), QMessageBox::Yes | QMessageBox::No) == QMessageBox::No) + if (QMessageBox::critical(QApplication::activeWindow(), QString(), + QObject::tr("File %1 is inside a PAK file %2\r\nDo you want it to be extracted for editing ?").arg(file, sPakName.c_str()), + QMessageBox::Yes | QMessageBox::No) == QMessageBox::No) { return false; } @@ -173,10 +174,9 @@ bool CFileUtil::ExtractFile(QString& file, bool bMsgBoxAskForExtraction, const c if (diskFile.open(QFile::WriteOnly)) { // Copy data from packed file to disk file. - char* data = new char[cryfile.GetLength()]; - cryfile.ReadRaw(data, cryfile.GetLength()); - diskFile.write(data, cryfile.GetLength()); - delete []data; + auto data = AZStd::make_unique(cryfile.GetLength()); + cryfile.ReadRaw(data.get(), cryfile.GetLength()); + diskFile.write(data.get(), cryfile.GetLength()); } else { @@ -185,7 +185,14 @@ bool CFileUtil::ExtractFile(QString& file, bool bMsgBoxAskForExtraction, const c } else { - file = cryfile.GetAdjustedFilename(); + + if (auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); fileIoBase != nullptr) + { + if (AZ::IO::FixedMaxPath resolvedFilePath; fileIoBase->ResolvePath(resolvedFilePath, cryfile.GetFilename())) + { + file = QString::fromUtf8(resolvedFilePath.c_str(), static_cast(resolvedFilePath.Native().size())); + } + } } return true; @@ -2157,13 +2164,13 @@ uint32 CFileUtil::GetAttributes(const char* filename, bool bUseSourceControl /*= return SCC_FILE_ATTRIBUTE_READONLY | SCC_FILE_ATTRIBUTE_INPAK; } - const char* adjustedFile = file.GetAdjustedFilename(); - if (!AZ::IO::SystemFile::Exists(adjustedFile)) + auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); + if (!fileIoBase->Exists(file.GetFilename())) { return SCC_FILE_ATTRIBUTE_INVALID; } - if (!AZ::IO::SystemFile::IsWritable(adjustedFile)) + if (fileIoBase->IsReadOnly(file.GetFilename())) { return SCC_FILE_ATTRIBUTE_NORMAL | SCC_FILE_ATTRIBUTE_READONLY; } diff --git a/Code/Editor/Util/XmlArchive.cpp b/Code/Editor/Util/XmlArchive.cpp index e6bc93fdf4..18c3fc8e63 100644 --- a/Code/Editor/Util/XmlArchive.cpp +++ b/Code/Editor/Util/XmlArchive.cpp @@ -124,7 +124,7 @@ bool CXmlArchive::SaveToPak([[maybe_unused]] const QString& levelPath, CPakFile& if (pakFile.GetArchive()) { - CLogFile::FormatLine("Saving pak file %s", (const char*)pakFile.GetArchive()->GetFullPath()); + CLogFile::FormatLine("Saving pak file %.*s", AZ_STRING_ARG(pakFile.GetArchive()->GetFullPath().Native())); } pNamedData->Save(pakFile); diff --git a/Code/Framework/AzCore/AzCore/Settings/SettingsRegistry.h b/Code/Framework/AzCore/AzCore/Settings/SettingsRegistry.h index 106e232904..3dc1be87c5 100644 --- a/Code/Framework/AzCore/AzCore/Settings/SettingsRegistry.h +++ b/Code/Framework/AzCore/AzCore/Settings/SettingsRegistry.h @@ -370,6 +370,11 @@ namespace AZ //! @param applyPatchSettings The ApplyPatchSettings which are using during JSON Merging virtual void SetApplyPatchSettings(const AZ::JsonApplyPatchSettings& applyPatchSettings) = 0; virtual void GetApplyPatchSettings(AZ::JsonApplyPatchSettings& applyPatchSettings) = 0; + + //! Stores option to indicate whether the FileIOBase instance should be used for file operations + //! @param useFileIo If true the FileIOBase instance will attempted to be used for FileIOBase + //! operations before falling back to use SystemFile + virtual void SetUseFileIO(bool useFileIo) = 0; }; inline SettingsRegistryInterface::Visitor::~Visitor() = default; diff --git a/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.cpp b/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.cpp index 6864dcd1c8..7ef1fa661d 100644 --- a/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.cpp +++ b/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.cpp @@ -9,11 +9,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include @@ -131,6 +134,12 @@ namespace AZ pointer.Create(m_settings, m_settings.GetAllocator()).SetArray(); } + SettingsRegistryImpl::SettingsRegistryImpl(bool useFileIo) + : SettingsRegistryImpl() + { + m_useFileIo = useFileIo; + } + void SettingsRegistryImpl::SetContext(SerializeContext* context) { AZStd::scoped_lock lock(m_settingMutex); @@ -723,15 +732,10 @@ namespace AZ RegistryFileList fileList; scratchBuffer->clear(); - AZ::IO::FixedMaxPathString folderPath{ path }; - constexpr AZStd::string_view pathSeparators{ AZ_CORRECT_AND_WRONG_DATABASE_SEPARATOR }; - if (pathSeparators.find_first_of(folderPath.back()) == AZStd::string_view::npos) - { - folderPath.push_back(AZ_CORRECT_DATABASE_SEPARATOR); - } + AZ::IO::FixedMaxPath folderPath{ path }; - const size_t platformKeyOffset = folderPath.size(); - folderPath.push_back('*'); + const size_t platformKeyOffset = folderPath.Native().size(); + folderPath /= '*'; Value specialzationArray(kArrayType); size_t specializationCount = specializations.GetCount(); @@ -741,47 +745,13 @@ namespace AZ specialzationArray.PushBack(Value(name.data(), aznumeric_caster(name.length()), m_settings.GetAllocator()), m_settings.GetAllocator()); } pointer.Create(m_settings, m_settings.GetAllocator()).SetObject() - .AddMember(StringRef("Folder"), Value(folderPath.c_str(), aznumeric_caster(folderPath.size()), m_settings.GetAllocator()), m_settings.GetAllocator()) + .AddMember(StringRef("Folder"), Value(folderPath.c_str(), aznumeric_caster(folderPath.Native().size()), m_settings.GetAllocator()), m_settings.GetAllocator()) .AddMember(StringRef("Specializations"), AZStd::move(specialzationArray), m_settings.GetAllocator()); - auto callback = [this, &fileList, &specializations, &pointer, &folderPath](const char* filename, bool isFile) -> bool - { - if (isFile) - { - if (fileList.size() >= MaxRegistryFolderEntries) - { - AZ_Error("Settings Registry", false, "Too many files in registry folder."); - AZStd::scoped_lock lock(m_settingMutex); - pointer.Create(m_settings, m_settings.GetAllocator()).SetObject() - .AddMember(StringRef("Error"), StringRef("Too many files in registry folder."), m_settings.GetAllocator()) - .AddMember(StringRef("Path"), Value(folderPath.c_str(), aznumeric_caster(folderPath.size()), m_settings.GetAllocator()), m_settings.GetAllocator()) - .AddMember(StringRef("File"), Value(filename, m_settings.GetAllocator()), m_settings.GetAllocator()); - return false; - } - - fileList.push_back(); - RegistryFile& registryFile = fileList.back(); - if (!ExtractFileDescription(registryFile, filename, specializations)) - { - fileList.pop_back(); - } - } - return true; - }; - SystemFile::FindFiles(folderPath.c_str(), callback); - - if (!platform.empty()) + auto CreateSettingsFindCallback = [this, &fileList, &specializations, &pointer, &folderPath](bool isPlatformFile) { - // Move the folderPath prefix back to the supplied path before the wildcard - folderPath.erase(platformKeyOffset); - folderPath += PlatformFolder; - folderPath.push_back(AZ_CORRECT_DATABASE_SEPARATOR); - folderPath += platform; - folderPath.push_back(AZ_CORRECT_DATABASE_SEPARATOR); - folderPath.push_back('*'); - - auto platformCallback = [this, &fileList, &specializations, &pointer, &folderPath](const char* filename, bool isFile) -> bool + return [this, &fileList, &specializations, &pointer, &folderPath, isPlatformFile](AZStd::string_view filename, bool isFile) -> bool { if (isFile) { @@ -791,8 +761,8 @@ namespace AZ AZStd::scoped_lock lock(m_settingMutex); pointer.Create(m_settings, m_settings.GetAllocator()).SetObject() .AddMember(StringRef("Error"), StringRef("Too many files in registry folder."), m_settings.GetAllocator()) - .AddMember(StringRef("Path"), Value(folderPath.c_str(), aznumeric_caster(folderPath.size()), m_settings.GetAllocator()), m_settings.GetAllocator()) - .AddMember(StringRef("File"), Value(filename, m_settings.GetAllocator()), m_settings.GetAllocator()); + .AddMember(StringRef("Path"), Value(folderPath.c_str(), aznumeric_caster(folderPath.Native().size()), m_settings.GetAllocator()), m_settings.GetAllocator()) + .AddMember(StringRef("File"), Value(filename.data(), aznumeric_caster(filename.size()), m_settings.GetAllocator()), m_settings.GetAllocator()); return false; } @@ -800,7 +770,7 @@ namespace AZ RegistryFile& registryFile = fileList.back(); if (ExtractFileDescription(registryFile, filename, specializations)) { - registryFile.m_isPlatformFile = true; + registryFile.m_isPlatformFile = isPlatformFile; } else { @@ -809,7 +779,42 @@ namespace AZ } return true; }; - SystemFile::FindFiles(folderPath.c_str(), platformCallback); + }; + + struct FindFilesPayload + { + bool m_isPlatformFile{}; + AZStd::fixed_vector m_pathSegmentsToAppend; + }; + + AZStd::fixed_vector findFilesPayloads{ {false} }; + if (!platform.empty()) + { + findFilesPayloads.push_back(FindFilesPayload{ true, { PlatformFolder, platform } }); + } + + for (const FindFilesPayload& findFilesPayload : findFilesPayloads) + { + // Erase back to initial path + folderPath.Native().erase(platformKeyOffset); + for (AZStd::string_view pathSegmentToAppend : findFilesPayload.m_pathSegmentsToAppend) + { + folderPath /= pathSegmentToAppend; + } + + auto findFilesCallback = CreateSettingsFindCallback(findFilesPayload.m_isPlatformFile); + if (AZ::IO::FileIOBase* fileIo = m_useFileIo ? AZ::IO::FileIOBase::GetInstance() : nullptr; fileIo != nullptr) + { + auto FileIoToSystemFileFindFiles = [findFilesCallback = AZStd::move(findFilesCallback), fileIo](const char* filePath) -> bool + { + return findFilesCallback(AZ::IO::PathView(filePath).Filename().Native(), !fileIo->IsDirectory(filePath)); + }; + fileIo->FindFiles(folderPath.c_str(), "*", FileIoToSystemFileFindFiles); + } + else + { + SystemFile::FindFiles((folderPath / "*").c_str(), findFilesCallback); + } } if (!fileList.empty()) @@ -831,16 +836,14 @@ namespace AZ // Load the registry files in the sorted order. for (RegistryFile& registryFile : fileList) { - folderPath.erase(platformKeyOffset); // Erase all characters after the platformKeyOffset + folderPath.Native().erase(platformKeyOffset); // Erase all characters after the platformKeyOffset if (registryFile.m_isPlatformFile) { - folderPath += PlatformFolder; - folderPath.push_back(AZ_CORRECT_DATABASE_SEPARATOR); - folderPath += platform; - folderPath.push_back(AZ_CORRECT_DATABASE_SEPARATOR); + folderPath /= PlatformFolder; + folderPath /= platform; } - folderPath += registryFile.m_relativePath; + folderPath /= registryFile.m_relativePath; if (!registryFile.m_isPatch) { @@ -1027,39 +1030,44 @@ namespace AZ return false; } - bool SettingsRegistryImpl::ExtractFileDescription(RegistryFile& output, const char* filename, const Specializations& specializations) + bool SettingsRegistryImpl::ExtractFileDescription(RegistryFile& output, AZStd::string_view filename, const Specializations& specializations) { - if (!filename || filename[0] == 0) + static constexpr auto PatchExtensionWithDot = AZStd::fixed_string<32>(".") + PatchExtension; + static constexpr auto ExtensionWithDot = AZStd::fixed_string<32>(".") + Extension; + static constexpr AZ::IO::PathView PatchExtensionView(PatchExtensionWithDot); + static constexpr AZ::IO::PathView ExtensionView(ExtensionWithDot); + + if (filename.empty()) { AZ_Error("Settings Registry", false, "Settings file without name found"); return false; } - AZStd::string_view filePath{ filename }; - const size_t filePathSize = filePath.size(); + AZ::IO::PathView filePath{ filename }; + const size_t filePathSize = filePath.Native().size(); // The filePath.empty() check makes sure that the file extension after the final isn't added to the output.m_tags - AZStd::optional pathTag = AZ::StringFunc::TokenizeNext(filePath, '.'); - for (; pathTag && !filePath.empty(); pathTag = AZ::StringFunc::TokenizeNext(filePath, '.')) + auto AppendSpecTags = [&output](AZStd::string_view pathTag) { - output.m_tags.push_back(Specializations::Hash(*pathTag)); - } + output.m_tags.push_back(Specializations::Hash(pathTag)); + }; + AZ::StringFunc::TokenizeVisitor(filePath.Stem().Native(), AppendSpecTags, '.'); // If token is invalid, then the filename has no characters and therefore no extension - if (pathTag) + if (AZ::IO::PathView fileExtension = filePath.Extension(); !fileExtension.empty()) { - if (pathTag->size() >= AZStd::char_traits::length(PatchExtension) && azstrnicmp(pathTag->data(), PatchExtension, pathTag->size()) == 0) + if (fileExtension == PatchExtensionView) { output.m_isPatch = true; } - else if (pathTag->size() != AZStd::char_traits::length(Extension) || azstrnicmp(pathTag->data(), Extension, pathTag->size()) != 0) + else if (fileExtension != ExtensionView) { return false; } } else { - AZ_Error("Settings Registry", false, R"(Settings file without extension found: "%s")", filename); + AZ_Error("Settings Registry", false, R"(Settings file without extension found: "%.*s")", AZ_STRING_ARG(filename)); return false; } @@ -1074,7 +1082,7 @@ namespace AZ { if (*currentIt == *(currentIt - 1)) { - AZ_Error("Settings Registry", false, R"(One or more tags are duplicated in registry file "%s")", filename); + AZ_Error("Settings Registry", false, R"(One or more tags are duplicated in registry file "%.*s")", AZ_STRING_ARG(filename)); return false; } ++currentIt; @@ -1103,11 +1111,123 @@ namespace AZ } else { - AZ_Error("Settings Registry", false, R"(Found relative path to settings file "%s" is too long.)", filename); + AZ_Error("Settings Registry", false, R"(Found relative path to settings file "%.*s" is too long.)", AZ_STRING_ARG(filename)); return false; } } + //! Structure which encapsulates Commands to either the FileIOBase or SystemFile classes based on + //! the SettingsRegistry option to use FileIO + struct SettingsRegistryFileReader + { + using FileHandleType = AZStd::variant; + + SettingsRegistryFileReader() = default; + SettingsRegistryFileReader(bool useFileIo, const char* filePath) + { + Open(useFileIo, filePath); + } + + ~SettingsRegistryFileReader() + { + if (auto fileHandle = AZStd::get_if(&m_file); fileHandle != nullptr) + { + if (AZ::IO::FileIOBase* fileIo = AZ::IO::FileIOBase::GetInstance(); fileIo != nullptr) + { + fileIo->Close(*fileHandle); + } + } + } + + bool Open(bool useFileIo, const char* filePath) + { + Close(); + if (AZ::IO::FileIOBase* fileIo = useFileIo ? AZ::IO::FileIOBase::GetInstance() : nullptr; fileIo != nullptr) + { + AZ::IO::HandleType fileHandle; + if (fileIo->Open(filePath, IO::OpenMode::ModeRead, fileHandle)) + { + m_file = fileHandle; + return true; + } + } + else + { + AZ::IO::SystemFile file; + if (file.Open(filePath, IO::SystemFile::OpenMode::SF_OPEN_READ_ONLY)) + { + m_file = AZStd::move(file); + return true; + } + } + + return false; + } + + bool IsOpen() const + { + if (auto fileHandle = AZStd::get_if(&m_file); fileHandle != nullptr) + { + return *fileHandle != AZ::IO::InvalidHandle; + } + else if (auto systemFile = AZStd::get_if(&m_file); systemFile != nullptr) + { + return systemFile->IsOpen(); + } + + return false; + } + + void Close() + { + if (auto fileHandle = AZStd::get_if(&m_file); fileHandle != nullptr) + { + if (AZ::IO::FileIOBase* fileIo = AZ::IO::FileIOBase::GetInstance(); fileIo != nullptr) + { + fileIo->Close(*fileHandle); + } + } + + m_file = AZStd::monostate{}; + } + + u64 Length() const + { + if (auto fileHandle = AZStd::get_if(&m_file); fileHandle != nullptr) + { + if (u64 fileSize{}; AZ::IO::FileIOBase::GetInstance()->Size(*fileHandle, fileSize)) + { + return fileSize; + } + } + else if (auto systemFile = AZStd::get_if(&m_file); systemFile != nullptr) + { + return systemFile->Length(); + } + + return 0; + } + + AZ::IO::SizeType Read(AZ::IO::SizeType byteSize, void* buffer) + { + if (auto fileHandle = AZStd::get_if(&m_file); fileHandle != nullptr) + { + if (AZ::u64 bytesRead{}; AZ::IO::FileIOBase::GetInstance()->Read(*fileHandle, buffer, byteSize, false, &bytesRead)) + { + return bytesRead; + } + } + else if (auto systemFile = AZStd::get_if(&m_file); systemFile != nullptr) + { + return systemFile->Read(byteSize, buffer); + } + + return 0; + } + + FileHandleType m_file; + }; + bool SettingsRegistryImpl::MergeSettingsFileInternal(const char* path, Format format, AZStd::string_view rootKey, AZStd::vector& scratchBuffer) { @@ -1116,8 +1236,8 @@ namespace AZ Pointer pointer(AZ_SETTINGS_REGISTRY_HISTORY_KEY "/-"); - SystemFile file; - if (!file.Open(path, SystemFile::OpenMode::SF_OPEN_READ_ONLY)) + SettingsRegistryFileReader fileReader(m_useFileIo, path); + if (!fileReader.IsOpen()) { AZ_Error("Settings Registry", false, R"(Unable to open registry file "%s".)", path); pointer.Create(m_settings, m_settings.GetAllocator()).SetObject() @@ -1126,7 +1246,7 @@ namespace AZ return false; } - u64 fileSize = file.Length(); + u64 fileSize = fileReader.Length(); if (fileSize == 0) { AZ_Warning("Settings Registry", false, R"(Registry file "%s" is 0 bytes in length. There is no nothing to merge)", path); @@ -1136,9 +1256,10 @@ namespace AZ .AddMember(StringRef("Path"), Value(path, m_settings.GetAllocator()), m_settings.GetAllocator()); return false; } + scratchBuffer.clear(); scratchBuffer.resize_no_construct(fileSize + 1); - if (file.Read(fileSize, scratchBuffer.data()) != fileSize) + if (fileReader.Read(fileSize, scratchBuffer.data()) != fileSize) { AZ_Error("Settings Registry", false, R"(Unable to read registry file "%s".)", path); pointer.Create(m_settings, m_settings.GetAllocator()).SetObject() @@ -1268,4 +1389,9 @@ namespace AZ { applyPatchSettings = m_applyPatchSettings; } + + void SettingsRegistryImpl::SetUseFileIO(bool useFileIo) + { + m_useFileIo = useFileIo; + } } // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.h b/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.h index 036f5c6596..ac214711b8 100644 --- a/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.h +++ b/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.h @@ -35,6 +35,10 @@ namespace AZ static constexpr size_t MaxRegistryFolderEntries = 128; SettingsRegistryImpl(); + //! @param useFileIo - If true attempt to redirect + //! file read operations through the FileIOBase instance first before falling back to SystemFile + //! otherwise always use SystemFile + explicit SettingsRegistryImpl(bool useFileIo); AZ_DISABLE_COPY_MOVE(SettingsRegistryImpl); ~SettingsRegistryImpl() override = default; @@ -83,6 +87,8 @@ namespace AZ void SetApplyPatchSettings(const AZ::JsonApplyPatchSettings& applyPatchSettings) override; void GetApplyPatchSettings(AZ::JsonApplyPatchSettings& applyPatchSettings) override; + void SetUseFileIO(bool useFileIo) override; + private: using TagList = AZStd::fixed_vector; struct RegistryFile @@ -104,7 +110,7 @@ namespace AZ // Compares if lhs is less than rhs in terms of processing order. This can also detect and report conflicts. bool IsLessThan(bool& collisionFound, const RegistryFile& lhs, const RegistryFile& rhs, const Specializations& specializations, const rapidjson::Pointer& historyPointer, AZStd::string_view folderPath); - bool ExtractFileDescription(RegistryFile& output, const char* filename, const Specializations& specializations); + bool ExtractFileDescription(RegistryFile& output, AZStd::string_view filename, const Specializations& specializations); bool MergeSettingsFileInternal(const char* path, Format format, AZStd::string_view rootKey, AZStd::vector& scratchBuffer); void SignalNotifier(AZStd::string_view jsonPath, Type type); @@ -119,5 +125,7 @@ namespace AZ JsonSerializerSettings m_serializationSettings; JsonDeserializerSettings m_deserializationSettings; JsonApplyPatchSettings m_applyPatchSettings; + + bool m_useFileIo{}; }; } // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/UnitTest/Mocks/MockSettingsRegistry.h b/Code/Framework/AzCore/AzCore/UnitTest/Mocks/MockSettingsRegistry.h index 7e6bb3d7b4..91dc0924c8 100644 --- a/Code/Framework/AzCore/AzCore/UnitTest/Mocks/MockSettingsRegistry.h +++ b/Code/Framework/AzCore/AzCore/UnitTest/Mocks/MockSettingsRegistry.h @@ -57,6 +57,7 @@ namespace AZ MOCK_METHOD1(SetApplyPatchSettings, void(const JsonApplyPatchSettings&)); MOCK_METHOD1(GetApplyPatchSettings, void(JsonApplyPatchSettings&)); + MOCK_METHOD1(SetUseFileIO, void(bool)); }; } // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/XML/rapidxml.h b/Code/Framework/AzCore/AzCore/XML/rapidxml.h index 694e0a1bf6..9e6d648293 100644 --- a/Code/Framework/AzCore/AzCore/XML/rapidxml.h +++ b/Code/Framework/AzCore/AzCore/XML/rapidxml.h @@ -13,6 +13,7 @@ // the intention is that you only include the customized version of rapidXML through this header, so that // you can override behavior here. +#include #include #endif // AZCORE_RAPIDXML_RAPIDXML_H_INCLUDED diff --git a/Code/Framework/AzFramework/AzFramework/Application/Application.cpp b/Code/Framework/AzFramework/AzFramework/Application/Application.cpp index 958dba2cc9..12974d03cf 100644 --- a/Code/Framework/AzFramework/AzFramework/Application/Application.cpp +++ b/Code/Framework/AzFramework/AzFramework/Application/Application.cpp @@ -81,71 +81,6 @@ namespace AzFramework static constexpr const char s_prefabSystemKey[] = "/Amazon/Preferences/EnablePrefabSystem"; static constexpr const char s_prefabWipSystemKey[] = "/Amazon/Preferences/EnablePrefabSystemWipFeatures"; static constexpr const char s_legacySlicesAssertKey[] = "/Amazon/Preferences/ShouldAssertForLegacySlicesUsage"; - - // A Helper function that can load an app descriptor from file. - AZ::Outcome, AZStd::string> LoadDescriptorFromFilePath(const char* appDescriptorFilePath, AZ::SerializeContext& serializeContext) - { - AZStd::unique_ptr loadedDescriptor; - - AZ::IO::SystemFile appDescriptorFile; - if (!appDescriptorFile.Open(appDescriptorFilePath, AZ::IO::SystemFile::SF_OPEN_READ_ONLY)) - { - return AZ::Failure(AZStd::string::format("Failed to open file: %s", appDescriptorFilePath)); - } - - AZ::IO::SystemFileStream appDescriptorFileStream(&appDescriptorFile, true); - if (!appDescriptorFileStream.IsOpen()) - { - return AZ::Failure(AZStd::string::format("Failed to stream file: %s", appDescriptorFilePath)); - } - - // Callback function for allocating the root elements in the file. - AZ::ObjectStream::InplaceLoadRootInfoCB inplaceLoadCb = - [](void** rootAddress, const AZ::SerializeContext::ClassData**, const AZ::Uuid& classId, AZ::SerializeContext*) - { - if (rootAddress && classId == azrtti_typeid()) - { - // ComponentApplication::Descriptor is normally a singleton. - // Force a unique instance to be created. - *rootAddress = aznew AZ::ComponentApplication::Descriptor(); - } - }; - - // Callback function for saving the root elements in the file. - AZ::ObjectStream::ClassReadyCB classReadyCb = - [&loadedDescriptor](void* classPtr, const AZ::Uuid& classId, AZ::SerializeContext* context) - { - // Save descriptor, delete anything else loaded from file. - if (classId == azrtti_typeid()) - { - loadedDescriptor.reset(static_cast(classPtr)); - } - else if (const AZ::SerializeContext::ClassData* classData = context->FindClassData(classId)) - { - classData->m_factory->Destroy(classPtr); - } - else - { - AZ_Error("Application", false, "Unexpected type %s found in application descriptor file. This memory will leak.", - classId.ToString().c_str()); - } - }; - - // There's other stuff in the file we may not recognize (system components), but we're not interested in that stuff. - AZ::ObjectStream::FilterDescriptor loadFilter(&AZ::Data::AssetFilterNoAssetLoading, AZ::ObjectStream::FILTERFLAG_IGNORE_UNKNOWN_CLASSES); - - if (!AZ::ObjectStream::LoadBlocking(&appDescriptorFileStream, serializeContext, classReadyCb, loadFilter, inplaceLoadCb)) - { - return AZ::Failure(AZStd::string::format("Failed to load objects from file: %s", appDescriptorFilePath)); - } - - if (!loadedDescriptor) - { - return AZ::Failure(AZStd::string::format("Failed to find descriptor object in file: %s", appDescriptorFilePath)); - } - - return AZ::Success(AZStd::move(loadedDescriptor)); - } } Application::Application() diff --git a/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp b/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp index 1891c50d10..c94d588a90 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp @@ -40,8 +40,6 @@ #include -#include - namespace AZ::IO { AZ_CVAR(int, sys_PakPriority, aznumeric_cast(ArchiveVars{}.nPriority), nullptr, AZ::ConsoleFunctorFlags::Null, @@ -64,40 +62,6 @@ namespace AZ::IO::ArchiveInternal // to the actual index , this offset is added to get the valid handle static constexpr size_t PseudoFileIdxOffset = 1; - // Explanation of this function: it is like a 'find and replace' for paths - // if the source path starts with 'aliasToLookFor' it will replace it with 'aliasToReplaceWith' - // else it will leave it untouched. - // the only caveat here is that it will perform this replacement if the source path either begins - // with the literal alias to look for, or begins with the actual absolute path that the alias to - // look for represents. It is a way of redirecting all @devassets@ to @assets@ regardless of whether - // you input a string that literally starts with @devassets@ or one that starts with the absolute path to the - // folder that @devassets@ aliases. - AZStd::optional ConvertAbsolutePathToAliasedPath(AZStd::string_view sourcePath, - AZStd::string_view aliasToLookFor, AZStd::string_view aliasToReplaceWith) - { - if (auto fileIo = AZ::IO::FileIOBase::GetDirectInstance(); !aliasToLookFor.empty() && !aliasToReplaceWith.empty() && !sourcePath.empty() && fileIo) - { - auto convertedPath = fileIo->ConvertToAlias(sourcePath); - if (!convertedPath) - { - return AZStd::nullopt; - } - - if (convertedPath->Native().starts_with(aliasToLookFor)) - { - convertedPath->Native().replace(0, aliasToLookFor.size(), aliasToReplaceWith); - } - // lowercase path if it starts with either the @assets@ or @root@ alias - if (convertedPath->Native().starts_with("@assets@") || convertedPath->Native().starts_with("@root@") - || convertedPath->Native().starts_with("@projectplatformcache@")) - { - AZStd::to_lower(convertedPath->Native().begin(), convertedPath->Native().end()); - } - return convertedPath; - } - return AZStd::make_optional(sourcePath); - } - struct CCachedFileRawData { void* m_pCachedData; @@ -146,15 +110,12 @@ namespace AZ::IO::ArchiveInternal uint32_t GetFileSize() { return GetFile() ? GetFile()->GetFileEntry()->desc.lSizeUncompressed : 0; } int FSeek(uint64_t nOffset, int nMode); - size_t FRead(void* pDest, size_t nSize, size_t nCount, AZ::IO::HandleType fileHandle); - size_t FReadAll(void* pDest, size_t nFileSize, AZ::IO::HandleType fileHandle); + size_t FRead(void* pDest, size_t bytesToRead, AZ::IO::HandleType fileHandle); void* GetFileData(size_t& nFileSize, AZ::IO::HandleType fileHandle); int FEof(); - char* FGets(char* pBuf, int n); - int Getc(); uint64_t GetModificationTime() { return m_pFileData->GetFileEntry()->GetModificationTime(); } - const char* GetArchivePath() { return m_pFileData->GetZip()->GetFilePath(); } + AZ::IO::PathView GetArchivePath() { return m_pFileData->GetZip()->GetFilePath(); } protected: uint64_t m_nCurSeek; CCachedFileDataPtr m_pFileData; @@ -205,7 +166,7 @@ namespace AZ::IO::ArchiveInternal } ////////////////////////////////////////////////////////////////////////// - size_t ArchiveInternal::CZipPseudoFile::FRead(void* pDest, size_t nSize, size_t nCount, [[maybe_unused]] AZ::IO::HandleType fileHandle) + size_t ArchiveInternal::CZipPseudoFile::FRead(void* pDest, size_t bytesToRead, [[maybe_unused]] AZ::IO::HandleType fileHandle) { AZ_PROFILE_FUNCTION(AzCore); @@ -214,21 +175,13 @@ namespace AZ::IO::ArchiveInternal return 0; } - size_t nTotal = nSize * nCount; + size_t nTotal = bytesToRead; if (!nTotal || (uint32_t)m_nCurSeek >= GetFileSize()) { return 0; } - if (nTotal > GetFileSize() - m_nCurSeek) - { - nTotal = GetFileSize() - m_nCurSeek; - if (nTotal < nSize) - { - return 0; - } - nTotal -= nTotal % nSize; - } + nTotal = (AZStd::min)(nTotal, GetFileSize() - m_nCurSeek); int64_t nReadBytes = GetFile()->ReadData(pDest, m_nCurSeek, nTotal); if (nReadBytes == -1) @@ -242,32 +195,9 @@ namespace AZ::IO::ArchiveInternal nTotal = (size_t)nReadBytes; } m_nCurSeek += nTotal; - return nTotal / nSize; + return nTotal; } - ////////////////////////////////////////////////////////////////////////// - size_t ArchiveInternal::CZipPseudoFile::FReadAll(void* pDest, size_t nFileSize, [[maybe_unused]] AZ::IO::HandleType fileHandle) - { - if (!GetFile()) - { - return 0; - } - - if (nFileSize != GetFileSize()) - { - AZ_Assert(false, "File size parameter of nFileSize does not match the file size of the zip file"); // Bad call - return 0; - } - - if (!GetFile()->ReadData(pDest, 0, nFileSize)) - { - return 0; - } - - m_nCurSeek = nFileSize; - - return nFileSize; - } ////////////////////////////////////////////////////////////////////////// void* ArchiveInternal::CZipPseudoFile::GetFileData(size_t& nFileSize, [[maybe_unused]] AZ::IO::HandleType fileHandle) @@ -292,70 +222,6 @@ namespace AZ::IO::ArchiveInternal return (uint32_t)m_nCurSeek >= GetFileSize(); } - char* ArchiveInternal::CZipPseudoFile::FGets(char* pBuf, int n) - { - if (!GetFile()) - { - return nullptr; - } - - char* pData = (char*)GetFile()->GetData(); - if (!pData) - { - return nullptr; - } - int nn = 0; - int i; - for (i = 0; i < n; i++) - { - if (i + m_nCurSeek == GetFileSize()) - { - break; - } - char c = pData[i + m_nCurSeek]; - if (c == 0xa || c == 0) - { - pBuf[nn++] = c; - i++; - break; - } - else - if (c == 0xd) - { - continue; - } - pBuf[nn++] = c; - } - pBuf[nn] = 0; - m_nCurSeek += i; - - if (m_nCurSeek == GetFileSize()) - { - return nullptr; - } - return pBuf; - } - - int ArchiveInternal::CZipPseudoFile::Getc() - { - if (!GetFile()) - { - return EOF; - } - char* pData = (char*)GetFile()->GetData(); - if (!pData) - { - return EOF; - } - int c = EOF; - if (m_nCurSeek == GetFileSize()) - { - return c; - } - c = pData[m_nCurSeek]; - m_nCurSeek += 1; - return c; - } } namespace AZ::IO @@ -379,16 +245,17 @@ namespace AZ::IO void Add(AZStd::string_view sResourceFile) override { - auto filename = ArchiveInternal::ConvertAbsolutePathToAliasedPath(sResourceFile); - if (!filename) + if (sResourceFile.empty()) + { + return; + } + AZ::IO::FixedMaxPath convertedFilename; + if (!AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(convertedFilename, sResourceFile)) { - AZ_Error("Archive", false, "Path %s cannot be converted to @alias@ form. It is longer than MaxPathLength %zu", aznumeric_cast(sResourceFile.size()), - sResourceFile.data(), AZ::IO::MaxPathLength); + AZ_Error("Archive", false, "Path %.*s cannot be resolved. It is longer than MaxPathLength %zu", + AZ_STRING_ARG(sResourceFile), AZ::IO::MaxPathLength); return; } - AZ::IO::FixedMaxPathString& convertedFilename = filename->Native(); - AZStd::replace(convertedFilename.begin(), convertedFilename.end(), AZ_WRONG_DATABASE_SEPARATOR, AZ_CORRECT_DATABASE_SEPARATOR); - AZStd::to_lower(convertedFilename.begin(), convertedFilename.end()); AZStd::scoped_lock lock(m_lock); m_set.emplace(convertedFilename); @@ -397,23 +264,20 @@ namespace AZ::IO { AZStd::scoped_lock lock(m_lock); m_set.clear(); - m_iter = m_set.begin(); + m_iter = m_set.end(); } bool IsExist(AZStd::string_view sResourceFile) override { - auto filename = ArchiveInternal::ConvertAbsolutePathToAliasedPath(sResourceFile); - if (!filename) + AZ::IO::FixedMaxPath convertedFilename; + if (!AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(convertedFilename, sResourceFile)) { - AZ_Error("Archive", false, "Path %.*s cannot be converted to @alias@ form. It is longer than MaxPathLength %zu", aznumeric_cast(sResourceFile.size()), - sResourceFile.data(), AZ::IO::MaxPathLength); + AZ_Error("Archive", false, "Path %.*s cannot be resolved. It is longer than MaxPathLength %zu", + AZ_STRING_ARG(sResourceFile), AZ::IO::MaxPathLength); return false; } - AZ::IO::FixedMaxPathString& convertedFilename = filename->Native(); - AZStd::replace(convertedFilename.begin(), convertedFilename.end(), AZ_WRONG_DATABASE_SEPARATOR, AZ_CORRECT_DATABASE_SEPARATOR); - AZStd::to_lower(convertedFilename.begin(), convertedFilename.end()); AZStd::scoped_lock lock(m_lock); - return m_set.contains(AZStd::string_view{ convertedFilename }); + return m_set.contains(AZ::IO::PathView{ convertedFilename }); } bool Load(AZStd::string_view sResourceListFilename) override { @@ -425,9 +289,8 @@ namespace AZ::IO AZ::IO::SizeType nLen = file.Length(); AZStd::string pMemBlock; - pMemBlock.resize_no_construct(nLen); - char* buf = pMemBlock.data(); - file.Read(nLen, buf); + pMemBlock.resize_no_construct(nLen);; + file.Read(pMemBlock.size(), pMemBlock.data()); // Parse file, every line in a file represents a resource filename. AZ::StringFunc::TokenizeVisitor(pMemBlock, @@ -464,7 +327,7 @@ namespace AZ::IO } private: - using ResourceSet = AZStd::set; + using ResourceSet = AZStd::set>; AZStd::recursive_mutex m_lock; ResourceSet m_set; ResourceSet::iterator m_iter; @@ -499,12 +362,13 @@ namespace AZ::IO , m_pNextLevelResourceList{ new CResourceList{} } , m_mainThreadId{ AZStd::this_thread::get_id() } { + CompressionBus::Handler::BusConnect(); } ////////////////////////////////////////////////////////////////////////// Archive::~Archive() { - Release(); + CompressionBus::Handler::BusDisconnect(); m_arrZips = {}; @@ -530,51 +394,13 @@ namespace AZ::IO AZ_Assert(m_cachedFileRawDataSet.empty(), "All Archive file cached raw data instances not closed"); } - bool Archive::CheckFileAccessDisabled([[maybe_unused]] AZStd::string_view name, [[maybe_unused]] const char* mode) - { - return false; - } - void Archive::LogFileAccessCallStack([[maybe_unused]] AZStd::string_view name, [[maybe_unused]] AZStd::string_view nameFull, [[maybe_unused]] const char* mode) { // Print call stack for each find. - AZ_TracePrintf("Archive", "LogFileAccessCallStack() - name=%.*s; nameFull=%.*s; mode=%s\n", aznumeric_cast(name.size()), name.data(), aznumeric_cast(nameFull.size()), nameFull.data(), mode); + AZ_TracePrintf("Archive", "LogFileAccessCallStack() - name=%.*s; nameFull=%.*s; mode=%s\n", AZ_STRING_ARG(name), AZ_STRING_ARG(nameFull), mode); AZ::Debug::Trace::PrintCallstack("Archive", 32); } - ////////////////////////////////////////////////////////////////////////// - - bool Archive::IsInstalledToHDD(AZStd::string_view) const - { - return true; - } - - ////////////////////////////////////////////////////////////////////////// - void Archive::ParseAliases(AZStd::string_view szCommandLine) - { - // this is a list of pairs separated by commas, i.e. Folder1,FolderNew,Textures,TestBuildTextures etc. - AZStd::optional aliasKey = AZ::StringFunc::TokenizeNext(szCommandLine, ','); - AZStd::optional aliasPath = AZ::StringFunc::TokenizeNext(szCommandLine, ','); - for ( ;aliasKey && aliasPath; aliasKey = AZ::StringFunc::TokenizeNext(szCommandLine,','), AZ::StringFunc::TokenizeNext(szCommandLine,',')) - { - // inform the Archive system - SetAlias(*aliasKey, *aliasPath, true); - AZ_TracePrintf("Archive", "Archive ALIAS:%.*s = %.*s\n", aznumeric_cast(aliasKey->size()), aliasKey->data(), - aznumeric_cast(aliasPath->size()), aliasPath->data()); - - } - } - - ////////////////////////////////////////////////////////////////////////// - //! if bReturnSame==true, it will return the input name if an alias doesn't exist. Otherwise returns nullptr - const char* Archive::GetAlias(AZStd::string_view szName, bool bReturnSame) - { - constexpr size_t MaxAliasLength = 32; - AZStd::fixed_string aliasKey{ szName }; - const char* dest = AZ::IO::FileIOBase::GetDirectInstance()->GetAlias(aliasKey.c_str()); - return (bReturnSame && !dest) ? szName.data() : dest; - } - ////////////////////////////////////////////////////////////////////////// void Archive::SetLocalizationFolder(AZStd::string_view sLocalizationFolder) { @@ -591,28 +417,6 @@ namespace AZ::IO m_sLocalizationFolder += AZ_CORRECT_FILESYSTEM_SEPARATOR_STRING; } - ////////////////////////////////////////////////////////////////////////// - void Archive::SetAlias(AZStd::string_view szName, AZStd::string_view szAlias, bool bAdd) - { - constexpr size_t MaxAliasLength = 32; - AZStd::fixed_string aliasKey{ szName }; - if (bAdd) - { - AZ::IO::PathString aliasPath{ szAlias }; - AZ::IO::FileIOBase::GetDirectInstance()->SetAlias(aliasKey.c_str(), aliasPath.c_str()); - } - else - { - AZ::IO::FileIOBase::GetDirectInstance()->ClearAlias(aliasKey.c_str()); - } - } - - - const char* Archive::AdjustFileName(AZStd::string_view src, char* dst, size_t dstSize, uint32_t, bool) - { - AZ::IO::FixedMaxPathString srcPath{ src }; - return AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(srcPath.c_str(), dst, dstSize) ? dst : nullptr; - } ////////////////////////////////////////////////////////////////////////// bool Archive::IsFileExist(AZStd::string_view sFilename, EFileSearchLocation fileLocation) @@ -679,52 +483,50 @@ namespace AZ::IO } ////////////////////////////////////////////////////////////////////////// - AZ::IO::HandleType Archive::FOpen(AZStd::string_view pName, const char* szMode, uint32_t nInputFlags) + AZ::IO::HandleType Archive::FOpen(AZStd::string_view pName, const char* szMode) { AZ_PROFILE_FUNCTION(AzCore); const size_t pathLen = pName.size(); - if (pathLen == 0 || pathLen >= MaxPath) + if (pathLen == 0 || pathLen >= AZ::IO::MaxPathLength) { return AZ::IO::InvalidHandle; } SAutoCollectFileAccessTime accessTime(this); - AZ::IO::HandleType fileHandle = AZ::IO::InvalidHandle; - - const bool bFileCanBeOnDisk = 0 != (nInputFlags & FOPEN_ONDISK); - // get the priority into local variable to avoid it changing in the course of // this function execution (?) const ArchiveLocationPriority nVarPakPriority = GetPakPriority(); AZ::IO::OpenMode nOSFlags = AZ::IO::GetOpenModeFromStringMode(szMode); - auto szFullPath = AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(pName); - if (!szFullPath) + AZ::IO::FixedMaxPath szFullPath; + if (!AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(szFullPath, pName)) { - AZ_Assert(szFullPath, "Unable to resolve path for filepath %.*s", aznumeric_cast(pName.size()), pName.data()); + AZ_Assert(false, "Unable to resolve path for filepath %.*s", aznumeric_cast(pName.size()), pName.data()); return false; } const bool fileWritable = (nOSFlags & (AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeAppend | AZ::IO::OpenMode::ModeUpdate)) != AZ::IO::OpenMode::Invalid; - AZ_PROFILE_SCOPE(Game, "File: %s Archive: %p", szFullPath->c_str(), this); + AZ_PROFILE_SCOPE(Game, "File: %s Archive: %p", szFullPath.c_str(), this); if (fileWritable) { // we need to open the file for writing, but we failed to do so. // the only reason that can be is that there are no directories for that file. // now create those dirs - if (!MakeDir(szFullPath->ParentPath().Native())) + if (AZ::IO::FixedMaxPath parentPath = szFullPath.ParentPath(); + !AZ::IO::FileIOBase::GetDirectInstance()->CreatePath(parentPath.c_str())) { return AZ::IO::InvalidHandle; } - if (AZ::IO::FileIOBase::GetDirectInstance()->Open(szFullPath->c_str(), nOSFlags, fileHandle)) + if (AZ::IO::HandleType fileHandle = AZ::IO::InvalidHandle; + AZ::IO::FileIOBase::GetDirectInstance()->Open(szFullPath.c_str(), nOSFlags, fileHandle)) { if (az_archive_verbosity) { - AZ_TracePrintf("Archive", " Archive::FOpen() has directly opened requested file %s for writing", szFullPath->c_str()); + AZ_TracePrintf("Archive", " Archive::FOpen() has directly opened requested file %s for writing", szFullPath.c_str()); } return fileHandle; } @@ -732,35 +534,41 @@ namespace AZ::IO return AZ::IO::InvalidHandle; } - if (nVarPakPriority == ArchiveLocationPriority::ePakPriorityFileFirst) // if the file system files have priority now.. + auto OpenFromFileSystem = [this, &szFullPath, pName, nOSFlags]() -> HandleType { - if (AZ::IO::FileIOBase::GetDirectInstance()->Open(szFullPath->c_str(), nOSFlags, fileHandle)) + if (AZ::IO::HandleType fileHandle = AZ::IO::InvalidHandle; + AZ::IO::FileIOBase::GetDirectInstance()->Open(szFullPath.c_str(), nOSFlags, fileHandle)) { if (az_archive_verbosity) { - AZ_TracePrintf("Archive", " Archive::FOpen() has directly opened requested file %s with FileFirst priority", szFullPath->c_str()); + AZ_TracePrintf("Archive", " Archive::FOpen() has directly opened requested file %s on for reading", szFullPath.c_str()); } RecordFile(fileHandle, pName); return fileHandle; } - } - uint32_t archiveFlags = 0; - CCachedFileDataPtr pFileData = GetFileData(szFullPath->Native(), archiveFlags); - if (pFileData) + return AZ::IO::InvalidHandle; + }; + auto OpenFromArchive = [this, &szFullPath, pName]() -> HandleType { + uint32_t archiveFlags = 0; + CCachedFileDataPtr pFileData = GetFileData(szFullPath.Native(), archiveFlags); + if (pFileData == nullptr) + { + return AZ::IO::InvalidHandle; + } + bool logged = false; - ZipDir::Cache* pZip = pFileData->GetZip(); - if (pZip) + if (ZipDir::Cache* pZip = pFileData->GetZip(); pZip != nullptr) { - const char* pZipFilePath = pZip->GetFilePath(); - if (pZipFilePath && pZipFilePath[0]) + AZ::IO::PathView pZipFilePath = pZip->GetFilePath(); + if (!pZipFilePath.empty()) { if (az_archive_verbosity) { - AZ_TracePrintf("Archive", " Archive::FOpen() has opened requested file %s from archive %s, disk offset %u", - szFullPath->c_str(), pZipFilePath, pFileData->GetFileEntry()->nFileDataOffset); + AZ_TracePrintf("Archive", " Archive::FOpen() has opened requested file %s from archive %.*s, disk offset %u", + szFullPath.c_str(), AZ_STRING_ARG(pZipFilePath.Native()), pFileData->GetFileEntry()->nFileDataOffset); logged = true; } } @@ -771,57 +579,54 @@ namespace AZ::IO if (az_archive_verbosity) { AZ_TracePrintf("Archive", " Archive::FOpen() has opened requested file %s from an archive file who's path isn't known", - szFullPath->c_str()); + szFullPath.c_str()); } } - } - else - { - if (nVarPakPriority != ArchiveLocationPriority::ePakPriorityPakOnly || bFileCanBeOnDisk) // if the archive files had more priority, we didn't attempt fopen before- try it now + + size_t nFile; + // find the empty slot and open the file there; return the handle { - if (AZ::IO::FileIOBase::GetDirectInstance()->Open(szFullPath->c_str(), nOSFlags, fileHandle)) + // try to open the pseudofile from one of the zips, make sure there is no user alias + AZStd::unique_lock lock(m_csOpenFiles); + for (nFile = 0; nFile < m_arrOpenFiles.size() && m_arrOpenFiles[nFile]->GetFile(); ++nFile) { - if (az_archive_verbosity) - { - AZ_TracePrintf("Archive", " Archive::FOpen() has directly opened requested file %s after failing to open from archives", - szFullPath->c_str()); - } - - RecordFile(fileHandle, pName); - return fileHandle; + continue; + } + if (nFile == m_arrOpenFiles.size()) + { + m_arrOpenFiles.emplace_back(AZStd::make_unique()); } + AZStd::unique_ptr& rZipFile = m_arrOpenFiles[nFile]; + rZipFile->Construct(pFileData.get()); } - return AZ::IO::InvalidHandle; // we can't find such file in the pack files - } - // try to open the pseudofile from one of the zips, make sure there is no user alias - AZStd::unique_lock lock(m_csOpenFiles); + AZ::IO::HandleType handle = (AZ::IO::HandleType)(nFile + ArchiveInternal::PseudoFileIdxOffset); - size_t nFile; - // find the empty slot and open the file there; return the handle - { - for (nFile = 0; nFile < m_arrOpenFiles.size() && m_arrOpenFiles[nFile]->GetFile(); ++nFile) - { - continue; - } - if (nFile == m_arrOpenFiles.size()) - { - m_arrOpenFiles.emplace_back(AZStd::make_unique()); - } - AZStd::unique_ptr& rZipFile = m_arrOpenFiles[nFile]; - rZipFile->Construct(pFileData.get()); - } + RecordFile(handle, pName); - AZ::IO::HandleType ret = (AZ::IO::HandleType)(nFile + ArchiveInternal::PseudoFileIdxOffset); + return handle; // the handle to the file + }; - if (az_archive_verbosity) + switch (nVarPakPriority) { - AZ_TracePrintf("Archive", " Archive::FOpen() has opened psuedo zip file %.*s", aznumeric_cast(pName.size()), pName.data()); + case ArchiveLocationPriority::ePakPriorityFileFirst: + { + AZ::IO::HandleType fileHandle = OpenFromFileSystem(); + return fileHandle != AZ::IO::InvalidHandle ? fileHandle : OpenFromArchive(); + } + case ArchiveLocationPriority::ePakPriorityPakFirst: + { + AZ::IO::HandleType fileHandle = OpenFromArchive(); + return fileHandle != AZ::IO::InvalidHandle ? fileHandle : OpenFromFileSystem(); + } + case ArchiveLocationPriority::ePakPriorityPakOnly: + { + return OpenFromArchive(); + } + default: + return AZ::IO::InvalidHandle; } - RecordFile(ret, pName); - - return ret; // the handle to the file } ////////////////////////////////////////////////////////////////////////// @@ -872,19 +677,14 @@ namespace AZ::IO ////////////////////////////////////////////////////////////////////////// // tests if the given file path refers to an existing file inside registered (opened) packs // the path must be absolute normalized lower-case with forward-slashes - ZipDir::FileEntry* Archive::FindPakFileEntry(AZStd::string_view szPath, uint32_t& nArchiveFlags, ZipDir::CachePtr* pZip, bool bSkipInMemoryArchives) const + ZipDir::FileEntry* Archive::FindPakFileEntry(AZStd::string_view szPath, uint32_t& nArchiveFlags, ZipDir::CachePtr* pZip) const { - AZ::IO::FixedMaxPath unaliasedPath; + AZ::IO::FixedMaxPath resolvedPath; + if (!AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(resolvedPath, szPath)) { - auto convertedPath = ArchiveInternal::ConvertAbsolutePathToAliasedPath(szPath); - - if (!convertedPath) - { - AZ_Error("Archive", false, "Path %s cannot be converted to @alias@ form. It is longer than MaxPathLength %zu", aznumeric_cast(szPath.size()), - szPath.data(), AZ::IO::MaxPathLength); - return nullptr; - } - unaliasedPath = AZStd::move(*convertedPath); + AZ_Error("Archive", false, "Path %s cannot be converted to @alias@ form. It is longer than MaxPathLength %zu", aznumeric_cast(szPath.size()), + szPath.data(), AZ::IO::MaxPathLength); + return nullptr; } @@ -892,28 +692,17 @@ namespace AZ::IO // scan through registered archive files and try to find this file for (auto itZip = m_arrZips.rbegin(); itZip != m_arrZips.rend(); ++itZip) { - if (bSkipInMemoryArchives && itZip->pArchive->GetFlags() & INestedArchive::FLAGS_IN_MEMORY_MASK) - { - continue; - } - if (itZip->pArchive->GetFlags() & INestedArchive::FLAGS_DISABLE_PAK) { continue; } - auto [bindRootIter, unaliasedIter] = AZStd::mismatch(itZip->m_pathBindRoot.begin(), itZip->m_pathBindRoot.end(), - unaliasedPath.begin(), unaliasedPath.end()); // If the bindRootIter is at the end then it is a prefix of the source path - if (bindRootIter == itZip->m_pathBindRoot.end()) + if (resolvedPath.IsRelativeTo(itZip->m_pathBindRoot)) { // unaliasedIter is past the bind root, so append the rest of it to a new relative path object - AZ::IO::FixedMaxPath relativePathInZip; - for (; unaliasedIter != unaliasedPath.end(); ++unaliasedIter) - { - relativePathInZip /= *unaliasedIter; - } + AZ::IO::FixedMaxPath relativePathInZip = resolvedPath.LexicallyRelative(itZip->m_pathBindRoot); ZipDir::FileEntry* pFileEntry = itZip->pZip->FindFile(relativePathInZip.Native()); if (pFileEntry) @@ -955,7 +744,7 @@ namespace AZ::IO } // returns the path to the archive in which the file was opened - const char* Archive::GetFileArchivePath(AZ::IO::HandleType fileHandle) + AZ::IO::PathView Archive::GetFileArchivePath(AZ::IO::HandleType fileHandle) { ArchiveInternal::CZipPseudoFile* pseudoFile = GetPseudoFile(fileHandle); if (pseudoFile) @@ -964,7 +753,7 @@ namespace AZ::IO } else { - return nullptr; + return {}; } } @@ -1066,7 +855,7 @@ namespace AZ::IO return 1; } - size_t Archive::FWrite(const void* data, size_t length, size_t elems, AZ::IO::HandleType fileHandle) + size_t Archive::FWrite(const void* data, size_t bytesToWrite, AZ::IO::HandleType fileHandle) { SAutoCollectFileAccessTime accessTime(this); @@ -1077,47 +866,28 @@ namespace AZ::IO } AZ_Assert(fileHandle != AZ::IO::InvalidHandle, "Invalid file has been passed to FWrite"); - if (AZ::IO::FileIOBase::GetDirectInstance()->Write(fileHandle, data, length * elems)) + if (AZ::u64 bytesWritten{}; AZ::IO::FileIOBase::GetDirectInstance()->Write(fileHandle, data, bytesToWrite, &bytesWritten)) { - return elems; + return bytesWritten; } return 0; } ////////////////////////////////////////////////////////////////////////// - size_t Archive::FReadRaw(void* pData, size_t nSize, size_t nCount, AZ::IO::HandleType fileHandle) + size_t Archive::FRead(void* pData, size_t bytesToRead, AZ::IO::HandleType fileHandle) { AZ_PROFILE_FUNCTION(AzCore); - AZ_PROFILE_SCOPE(Game, "Size: %d Archive: %p", nSize, this); SAutoCollectFileAccessTime accessTime(this); ArchiveInternal::CZipPseudoFile* pseudoFile = GetPseudoFile(fileHandle); if (pseudoFile) { - return pseudoFile->FRead(pData, nSize, nCount, fileHandle); - } - - AZ::u64 bytesRead = 0; - AZ::IO::FileIOBase::GetDirectInstance()->Read(fileHandle, pData, nSize * nCount, false, &bytesRead); - return static_cast(bytesRead / nSize); - } - - ////////////////////////////////////////////////////////////////////////// - size_t Archive::FReadRawAll(void* pData, size_t nFileSize, AZ::IO::HandleType fileHandle) - { - AZ_PROFILE_FUNCTION(AzCore); - - SAutoCollectFileAccessTime accessTime(this); - ArchiveInternal::CZipPseudoFile* pseudoFile = GetPseudoFile(fileHandle); - if (pseudoFile) - { - return pseudoFile->FReadAll(pData, nFileSize, fileHandle); + return pseudoFile->FRead(pData, bytesToRead, fileHandle); } - AZ::IO::FileIOBase::GetDirectInstance()->Seek(fileHandle, 0, AZ::IO::SeekType::SeekFromStart); AZ::u64 bytesRead = 0; - AZ::IO::FileIOBase::GetDirectInstance()->Read(fileHandle, pData, nFileSize, false, &bytesRead); - return static_cast(bytesRead); + AZ::IO::FileIOBase::GetDirectInstance()->Read(fileHandle, pData, bytesToRead, false, &bytesRead); + return bytesRead; } ////////////////////////////////////////////////////////////////////////// @@ -1234,48 +1004,6 @@ namespace AZ::IO } - int Archive::FPrintf(AZ::IO::HandleType fileHandle, const char* szFormat, ...) - { - SAutoCollectFileAccessTime accessTime(this); - ArchiveInternal::CZipPseudoFile* pseudoFile = GetPseudoFile(fileHandle); - if (pseudoFile) - { - return 0; // we don't support it now - } - - va_list arglist; - int rv; - va_start(arglist, szFormat); - rv = static_cast(AZ::IO::PrintV(fileHandle, szFormat, arglist)); - va_end(arglist); - return rv; - } - - char* Archive::FGets(char* str, int n, AZ::IO::HandleType fileHandle) - { - SAutoCollectFileAccessTime accessTime(this); - ArchiveInternal::CZipPseudoFile* pseudoFile = GetPseudoFile(fileHandle); - if (pseudoFile) - { - return pseudoFile->FGets(str, n); - } - - return AZ::IO::FGetS(str, n, fileHandle); - } - - int Archive::Getc(AZ::IO::HandleType fileHandle) - { - SAutoCollectFileAccessTime accessTime(this); - ArchiveInternal::CZipPseudoFile* pseudoFile = GetPseudoFile(fileHandle); - if (pseudoFile) - { - return pseudoFile->Getc(); - } - - return AZ::IO::GetC(fileHandle); - } - - ////////////////////////////////////////////////////////////////////////// AZ::IO::ArchiveFileIterator Archive::FindFirst(AZStd::string_view pDir, EFileSearchType searchType) { @@ -1304,7 +1032,7 @@ namespace AZ::IO break; } - AZStd::intrusive_ptr pFindData = new AZ::IO::FindData(); + AZStd::intrusive_ptr pFindData = aznew AZ::IO::FindData(); pFindData->Scan(this, szFullPath->Native(), bAllowUseFileSystem, bScanZips); return pFindData->Fetch(); @@ -1322,18 +1050,6 @@ namespace AZ::IO return true; } - ////////////////////////////////////////////////////////////////////////// - bool Archive::LoadPakToMemory([[maybe_unused]] AZStd::string_view pName, [[maybe_unused]] IArchive::EInMemoryArchiveLocation nLoadPakToMemory, - [[maybe_unused]] AZStd::intrusive_ptr pMemoryBlock) - { - return true; - } - - ////////////////////////////////////////////////////////////////////////// - void Archive::LoadPaksToMemory([[maybe_unused]] int nMaxArchiveSize, [[maybe_unused]] bool bLoadToMemory) - { - } - auto Archive::GetLevelPackOpenEvent() -> LevelPackOpenEvent* { return &m_levelOpenEvent; @@ -1344,7 +1060,7 @@ namespace AZ::IO return &m_levelCloseEvent; } //====================================================================== - bool Archive::OpenPack(AZStd::string_view szBindRootIn, AZStd::string_view szPath, uint32_t nFlags, + bool Archive::OpenPack(AZStd::string_view szBindRootIn, AZStd::string_view szPath, AZStd::intrusive_ptr pData, AZ::IO::FixedMaxPathString* pFullPath, bool addLevels) { AZ_Assert(!szBindRootIn.empty(), "Bind Root should not be empty"); @@ -1363,7 +1079,7 @@ namespace AZ::IO return false; } - bool result = OpenPackCommon(szBindRoot->Native(), szFullPath->Native(), nFlags, pData, addLevels); + bool result = OpenPackCommon(szBindRoot->Native(), szFullPath->Native(), pData, addLevels); if (pFullPath) { @@ -1373,7 +1089,7 @@ namespace AZ::IO return result; } - bool Archive::OpenPack(AZStd::string_view szPath, uint32_t nFlags, AZStd::intrusive_ptr pData, + bool Archive::OpenPack(AZStd::string_view szPath, AZStd::intrusive_ptr pData, AZ::IO::FixedMaxPathString* pFullPath, bool addLevels) { auto szFullPath = AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(szPath); @@ -1385,7 +1101,7 @@ namespace AZ::IO AZStd::string_view bindRoot = szFullPath->ParentPath().Native(); - bool result = OpenPackCommon(bindRoot, szFullPath->Native(), nFlags, pData, addLevels); + bool result = OpenPackCommon(bindRoot, szFullPath->Native(), pData, addLevels); if (pFullPath) { @@ -1396,34 +1112,22 @@ namespace AZ::IO } - bool Archive::OpenPackCommon(AZStd::string_view szBindRoot, AZStd::string_view szFullPath, uint32_t nArchiveFlags, + bool Archive::OpenPackCommon(AZStd::string_view szBindRoot, AZStd::string_view szFullPath, AZStd::intrusive_ptr pData, bool addLevels) { - // Note this will replace @devassets@ with @assets@ to provide a proper bind root for the archives - auto conversionResult = ArchiveInternal::ConvertAbsolutePathToAliasedPath(szBindRoot); - if (!conversionResult) - { - AZ_Error("Archive", false, "Path %.*s cannot be converted to @alias@ form. It is longer than MaxPathLength %zu", - aznumeric_cast(szBindRoot.size()), szBindRoot.data(), AZ::IO::MaxPathLength); - return false; - } - // setup PackDesc before the duplicate test PackDesc desc; - desc.strFileName = szFullPath; + desc.m_strFileName = szFullPath; - if (!conversionResult || conversionResult->empty()) + if (AZ::IO::FixedMaxPath pathBindRoot; !AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(pathBindRoot, szBindRoot)) { - desc.m_pathBindRoot = "@assets@"; + AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(pathBindRoot, "@assets@"); + desc.m_pathBindRoot = pathBindRoot.LexicallyNormal().String(); } else { - // Create a bind root without any trailing slashes - desc.m_pathBindRoot = AZStd::move(*conversionResult); - if (desc.m_pathBindRoot.HasRelativePath() && !desc.m_pathBindRoot.HasFilename()) - { - desc.m_pathBindRoot = desc.m_pathBindRoot.ParentPath(); - } + // Create a bind root + desc.m_pathBindRoot = pathBindRoot.LexicallyNormal().String(); } // hold the lock from the point we query the zip array, @@ -1433,56 +1137,23 @@ namespace AZ::IO // try to find this - maybe the pack has already been opened for (auto it = m_arrZips.begin(); it != m_arrZips.end(); ++it) { - const char* pFilePath = it->pZip->GetFilePath(); - if (pFilePath == desc.strFileName && it->m_pathBindRoot == desc.m_pathBindRoot) + if (AZ::IO::PathView archiveFilePath = it->pZip->GetFilePath(); + archiveFilePath == desc.m_strFileName && it->m_pathBindRoot == desc.m_pathBindRoot) { return true; // already opened } } } - int flags = INestedArchive::FLAGS_OPTIMIZED_READ_ONLY | INestedArchive::FLAGS_ABSOLUTE_PATHS; - if ((nArchiveFlags & FLAGS_PAK_IN_MEMORY) != 0) - { - flags |= INestedArchive::FLAGS_IN_MEMORY; - } - if ((nArchiveFlags & FLAGS_PAK_IN_MEMORY_CPU) != 0) - { - flags |= INestedArchive::FLAGS_IN_MEMORY_CPU; - } - if ((nArchiveFlags & FLAGS_FILENAMES_AS_CRC32) != 0) - { - flags |= INestedArchive::FLAGS_FILENAMES_AS_CRC32; - } - if ((nArchiveFlags & FLAGS_REDIRECT_TO_DISC) != 0) - { - flags |= FLAGS_REDIRECT_TO_DISC; - } - if ((nArchiveFlags & INestedArchive::FLAGS_OVERRIDE_PAK) != 0) - { - flags |= INestedArchive::FLAGS_OVERRIDE_PAK; - } - if ((nArchiveFlags & FLAGS_LEVEL_PAK_INSIDE_PAK) != 0) - { - flags |= INestedArchive::FLAGS_INSIDE_PAK; - } + const int flags = INestedArchive::FLAGS_OPTIMIZED_READ_ONLY | INestedArchive::FLAGS_ABSOLUTE_PATHS; desc.pArchive = OpenArchive(szFullPath, szBindRoot, flags, pData); if (!desc.pArchive) { return false; // couldn't open the archive } - if (m_filesCachedOnHDD.size()) - { - uint32_t crc = AZ::Crc32(szFullPath); - if (m_filesCachedOnHDD.find(crc) != m_filesCachedOnHDD.end()) - { - uint32_t eFlags = desc.pArchive->GetFlags(); - desc.pArchive->SetFlags(eFlags | INestedArchive::FLAGS_ON_HDD); - } - } - AZ_TracePrintf("Archive", "Opening archive file %.*s\n", aznumeric_cast(szFullPath.size()), szFullPath.data()); + AZ_TracePrintf("Archive", "Opening archive file %.*s\n", AZ_STRING_ARG(szFullPath)); desc.pZip = static_cast(desc.pArchive.get())->GetCache(); AZStd::unique_lock lock(m_csZips); @@ -1493,20 +1164,14 @@ namespace AZ::IO // All we have to do is name the archive appropriately to make // sure later archives added to the current set of archives sort higher // and therefore get used instead of lower sorted archives - AZStd::string_view nextBundle; + AZ::IO::PathView nextBundle; ZipArray::reverse_iterator revItZip = m_arrZips.rbegin(); - if ((nArchiveFlags & INestedArchive::FLAGS_OVERRIDE_PAK) == 0) + for (; revItZip != m_arrZips.rend(); ++revItZip) { - for (; revItZip != m_arrZips.rend(); ++revItZip) + nextBundle = revItZip->GetFullPath(); + if (desc.GetFullPath() > revItZip->GetFullPath()) { - if ((revItZip->pArchive->GetFlags() & INestedArchive::FLAGS_OVERRIDE_PAK) == 0) - { - nextBundle = revItZip->GetFullPath(); - if (azstricmp(desc.GetFullPath(), revItZip->GetFullPath()) > 0) - { - break; - } - } + break; } } @@ -1555,17 +1220,17 @@ namespace AZ::IO } AZ::IO::ArchiveNotificationBus::Broadcast([](AZ::IO::ArchiveNotifications* archiveNotifications, const char* bundleName, - AZStd::shared_ptr bundleManifest, const char* nextBundle, AZStd::shared_ptr bundleCatalog) + AZStd::shared_ptr bundleManifest, const AZ::IO::FixedMaxPath& nextBundle, AZStd::shared_ptr bundleCatalog) { - archiveNotifications->BundleOpened(bundleName, bundleManifest, nextBundle, bundleCatalog); - }, desc.strFileName.c_str(), bundleManifest, nextBundle.data(), bundleCatalog); + archiveNotifications->BundleOpened(bundleName, bundleManifest, nextBundle.c_str(), bundleCatalog); + }, desc.m_strFileName.c_str(), bundleManifest, nextBundle, bundleCatalog); return true; } // after this call, the file will be unlocked and closed, and its contents won't be used to search for files - bool Archive::ClosePack(AZStd::string_view pName, [[maybe_unused]] uint32_t nFlags) + bool Archive::ClosePack(AZStd::string_view pName) { auto szZipPath = AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(pName); if (!szZipPath) @@ -1581,16 +1246,16 @@ namespace AZ::IO AZStd::unique_lock lock(m_csZips); for (auto it = m_arrZips.begin(); it != m_arrZips.end();) { - if (azstricmp(szZipPath->c_str(), it->GetFullPath()) == 0) + if (szZipPath == it->GetFullPath()) { // this is the pack with the given name - remove it, and if possible it will be deleted // the zip is referenced from the archive and *it; the archive is referenced only from *it // // the pZip (cache) can be referenced from stream engine and pseudo-files. // the archive can be referenced from outside - AZ::IO::ArchiveNotificationBus::Broadcast([](AZ::IO::ArchiveNotifications* archiveNotifications, const char* bundleName) + AZ::IO::ArchiveNotificationBus::Broadcast([](AZ::IO::ArchiveNotifications* archiveNotifications, const AZ::IO::FixedMaxPath& bundleName) { - archiveNotifications->BundleClosed(bundleName); + archiveNotifications->BundleClosed(bundleName.c_str()); }, it->GetFullPath()); if (usePrefabSystemForLevels) @@ -1643,7 +1308,7 @@ namespace AZ::IO return foundMatchingPackFile; } - bool Archive::OpenPacks(AZStd::string_view pWildcardIn, uint32_t nFlags, AZStd::vector* pFullPaths) + bool Archive::OpenPacks(AZStd::string_view pWildcardIn, AZStd::vector* pFullPaths) { auto strBindRoot{ AZ::IO::PathView(pWildcardIn).ParentPath() }; AZ::IO::FixedMaxPath bindRoot; @@ -1651,10 +1316,10 @@ namespace AZ::IO { bindRoot = strBindRoot; } - return OpenPacksCommon(bindRoot.Native(), pWildcardIn, nFlags, pFullPaths); + return OpenPacksCommon(bindRoot.Native(), pWildcardIn, pFullPaths); } - bool Archive::OpenPacks(AZStd::string_view szBindRoot, AZStd::string_view pWildcardIn, uint32_t nFlags, AZStd::vector* pFullPaths) + bool Archive::OpenPacks(AZStd::string_view szBindRoot, AZStd::string_view pWildcardIn, AZStd::vector* pFullPaths) { auto bindRoot = AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(szBindRoot); if (!bindRoot) @@ -1662,16 +1327,16 @@ namespace AZ::IO AZ_Assert(false, "Unable to resolve path for filepath %.*s", aznumeric_cast(szBindRoot.size()), szBindRoot.data()); return false; } - return OpenPacksCommon(bindRoot->Native(), pWildcardIn, nFlags, pFullPaths); + return OpenPacksCommon(bindRoot->Native(), pWildcardIn, pFullPaths); } - bool Archive::OpenPacksCommon(AZStd::string_view szDir, AZStd::string_view pWildcardIn, uint32_t nArchiveFlags, AZStd::vector* pFullPaths, bool addLevels) + bool Archive::OpenPacksCommon(AZStd::string_view szDir, AZStd::string_view pWildcardIn, AZStd::vector* pFullPaths, bool addLevels) { constexpr AZStd::string_view wildcards{ "*?" }; if (wildcards.find_first_of(pWildcardIn) == AZStd::string_view::npos) { // No wildcards, just open pack - if (OpenPackCommon(szDir, pWildcardIn, nArchiveFlags, nullptr, addLevels)) + if (OpenPackCommon(szDir, pWildcardIn, nullptr, addLevels)) { if (pFullPaths) { @@ -1683,25 +1348,24 @@ namespace AZ::IO if (AZ::IO::ArchiveFileIterator fileIterator = FindFirst(pWildcardIn, IArchive::eFileSearchType_AllowOnDiskOnly); fileIterator) { - AZStd::vector files; + AZStd::vector files; do { - AZStd::string foundFilename{ fileIterator.m_filename }; - AZStd::to_lower(foundFilename.begin(), foundFilename.end()); - files.emplace_back(AZStd::move(foundFilename)); + auto& foundFilename = files.emplace_back(fileIterator.m_filename); + AZStd::to_lower(foundFilename.Native().begin(), foundFilename.Native().end()); } while (fileIterator = FindNext(fileIterator)); - // Open files in alphabet order. + // Open files in alphabetical order. AZStd::sort(files.begin(), files.end()); bool bAllOk = true; - for (const AZStd::string& file : files) + for (const AZ::IO::FixedMaxPath& file : files) { - bAllOk = OpenPackCommon(szDir, file, nArchiveFlags, nullptr, addLevels) && bAllOk; + bAllOk = OpenPackCommon(szDir, file.Native(), nullptr, addLevels) && bAllOk; if (pFullPaths) { - pFullPaths->emplace_back(file.begin(), file.end()); + pFullPaths->emplace_back(AZStd::move(file.Native())); } } @@ -1713,7 +1377,7 @@ namespace AZ::IO } - bool Archive::ClosePacks(AZStd::string_view pWildcardIn, uint32_t nFlags) + bool Archive::ClosePacks(AZStd::string_view pWildcardIn) { auto path = AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(pWildcardIn); if (!path) @@ -1723,26 +1387,13 @@ namespace AZ::IO } return AZ::IO::FileIOBase::GetDirectInstance()->FindFiles(AZ::IO::FixedMaxPath(path->ParentPath()).c_str(), - AZ::IO::FixedMaxPath(path->Filename()).c_str(), [&](const char* filePath) -> bool + AZ::IO::FixedMaxPath(path->Filename()).c_str(), [this](const char* filePath) -> bool { - ClosePack(filePath, nFlags); + ClosePack(filePath); return true; }); } - - ///////////////////////////////////////////////////// - bool Archive::Init([[maybe_unused]] AZStd::string_view szBasePath) - { - BusConnect(); - return true; - } - - void Archive::Release() - { - BusDisconnect(); - } - ////////////////////////////////////////////////////////////////////////// ArchiveInternal::CZipPseudoFile* Archive::GetPseudoFile(AZ::IO::HandleType fileHandle) const { @@ -1912,36 +1563,6 @@ namespace AZ::IO return m_pFileEntry->nFileDataOffset; } - bool Archive::MakeDir(AZStd::string_view szPathIn) - { - AZ::IO::StackString pathStr{ szPathIn }; - // Determine if there is a period ('.') after the last slash to determine if the path contains a file. - // This used to be a strchr on the whole path which could contain a period in a path, such as network domain paths (domain.user). - size_t findDotFromPos = pathStr.rfind(AZ_CORRECT_FILESYSTEM_SEPARATOR); - if (findDotFromPos == AZ::IO::StackString::npos) - { - findDotFromPos = pathStr.rfind(AZ_WRONG_FILESYSTEM_SEPARATOR); - if (findDotFromPos == AZ::IO::StackString::npos) - { - findDotFromPos = 0; - } - } - size_t dotPos = pathStr.find('.', findDotFromPos); - if (dotPos != AZ::IO::StackString::npos) - { - AZStd::string fullPath; - AZ::StringFunc::Path::GetFullPath(pathStr.c_str(), fullPath); - pathStr = fullPath; - } - - if (pathStr.empty()) - { - return true; - } - - return AZ::IO::FileIOBase::GetDirectInstance()->CreatePath(pathStr.c_str()); - } - ////////////////////////////////////////////////////////////////////////// // open the physical archive file - creates if it doesn't exist // returns nullptr if it's invalid or can't open the file @@ -1962,15 +1583,6 @@ namespace AZ::IO uint32_t nFactoryFlags = 0; - if (nFlags & INestedArchive::FLAGS_IN_MEMORY) - { - nFactoryFlags |= ZipDir::CacheFactory::FLAGS_IN_MEMORY; - } - - if (nFlags & INestedArchive::FLAGS_IN_MEMORY_CPU) - { - nFactoryFlags |= ZipDir::CacheFactory::FLAGS_IN_MEMORY_CPU; - } if (nFlags & INestedArchive::FLAGS_DONT_COMPACT) { @@ -1982,10 +1594,6 @@ namespace AZ::IO nFactoryFlags |= ZipDir::CacheFactory::FLAGS_READ_ONLY; } - if (nFlags & INestedArchive::FLAGS_INSIDE_PAK) - { - nFactoryFlags |= ZipDir::CacheFactory::FLAGS_READ_INSIDE_PAK; - } INestedArchive* pArchive = FindArchive(szFullPath->Native()); if (pArchive) @@ -2016,7 +1624,10 @@ namespace AZ::IO if (!pakOnDisk && (nFactoryFlags & ZipDir::CacheFactory::FLAGS_READ_ONLY)) { // Archive file not found. - AZ_TracePrintf("Archive", "Archive file %s does not exist\n", szFullPath->c_str()); + if (az_archive_verbosity) + { + AZ_TracePrintf("Archive", "Archive file %s does not exist\n", szFullPath->c_str()); + } return nullptr; } @@ -2031,148 +1642,6 @@ namespace AZ::IO return nullptr; } - uint32_t Archive::ComputeCRC(AZStd::string_view szPath, [[maybe_unused]] uint32_t nFileOpenFlags) - { - AZ_Assert(!szPath.empty(), "Path to compute Crc cannot be empty"); - - AZ::Crc32 dwCRC = 0; - - // generate crc32 - { - // avoid heap allocation by working in 8k chunks - const uint32_t dwChunkSize = 1024 * 8; - - // note that the actual CRC algorithm can work on various sized words but operates on individual words - // so there's little difference between feeding it 8k and 8mb, except you might save yourself some io calls. - - uint8_t pMem[dwChunkSize]; - - - AZ::IO::FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance(); - if (!fileIO) - { - return ZipDir::ZD_ERROR_INVALID_CALL; - } - - AZ::IO::HandleType fileHandle = AZ::IO::InvalidHandle; - - if (AZ::IO::PathString filepath{ szPath }; !fileIO->Open(filepath.c_str(), AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeBinary, fileHandle)) - { - return ZipDir::ZD_ERROR_INVALID_PATH; - } - // load whole file in chunks and compute CRC - while (true) - { - AZ::u64 bytesRead = 0; - fileIO->Read(fileHandle, pMem, dwChunkSize, false, &bytesRead); // read up to ChunkSize bytes and put the actual number of bytes read into bytesRead. - - if (bytesRead) - { - dwCRC.Add(pMem, aznumeric_caster(bytesRead)); - } - else - { - break; - } - } - - FClose(fileHandle); - } - - return dwCRC; - } - - bool Archive::ComputeMD5(AZStd::string_view szPath, uint8_t* md5, uint32_t nFileOpenFlags, bool useDirectFileAccess) - { - if (szPath.empty() || !md5) - { - return false; - } - - MD5Context context; - MD5Init(&context); - - // generate checksum - { - const AZ::u64 dwChunkSize = 1024 * 1024; // 1MB chunks - AZStd::unique_ptr pMem{ reinterpret_cast(AZ::AllocatorInstance::Get().Allocate(dwChunkSize, alignof(uint8_t))), - [](uint8_t* ptr) { AZ::AllocatorInstance::Get().DeAllocate(ptr); } - }; - - if (!pMem) - { - return false; - } - - AZ::u64 dwSize = 0; - - AZ::IO::PathString filepath{ szPath }; - if (useDirectFileAccess) - { - - AZ::IO::FileIOBase::GetDirectInstance()->Size(filepath.c_str(), dwSize); - } - else - { - AZ::IO::HandleType fileHandle = FOpen(filepath, "rb", nFileOpenFlags); - - if (fileHandle != AZ::IO::InvalidHandle) - { - dwSize = FGetSize(fileHandle); - FClose(fileHandle); - } - } - - // rbx open flags, x is a hint to not cache whole file in memory. - AZ::IO::HandleType fileHandle = AZ::IO::InvalidHandle; - if (useDirectFileAccess) - { - AZ::IO::FileIOBase::GetDirectInstance()->Open(filepath.c_str(), AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeBinary, fileHandle); - } - else - { - fileHandle = FOpen(filepath, "rbx", nFileOpenFlags); - } - - if (fileHandle == AZ::IO::InvalidHandle) - { - return false; - } - - // load whole file in chunks and compute Md5 - while (dwSize > 0) - { - uint64_t dwLocalSize = AZStd::min(dwSize, dwChunkSize); - - AZ::u64 read{ 0 }; - if (useDirectFileAccess) - { - AZ::IO::FileIOBase::GetDirectInstance()->Read(fileHandle, pMem.get(), dwLocalSize, false, &read); - } - else - { - read = FReadRaw(pMem.get(), 1, dwLocalSize, fileHandle); - } - AZ_Assert(read == dwLocalSize, "Failed to read dwLocalSize %" PRIu32 " bytes from file", dwLocalSize); - - MD5Update(&context, pMem.get(), aznumeric_cast(dwLocalSize)); - dwSize -= dwLocalSize; - } - - if (useDirectFileAccess) - { - AZ::IO::FileIOBase::GetDirectInstance()->Close(fileHandle); - } - else - { - FClose(fileHandle); - } - } - - MD5Final(md5, &context); - return true; - } - void Archive::Register(INestedArchive* pArchive) { AZStd::unique_lock lock(m_archiveMutex); @@ -2185,7 +1654,7 @@ namespace AZ::IO AZStd::unique_lock lock(m_archiveMutex); if (pArchive) { - AZ_TracePrintf("Archive", "Closing Archive file: %s\n", pArchive->GetFullPath()); + AZ_TracePrintf("Archive", "Closing Archive file: %.*s\n", AZ_STRING_ARG(pArchive->GetFullPath().Native())); } ArchiveArray::iterator it; if (m_arrArchives.size() < 16) @@ -2212,7 +1681,7 @@ namespace AZ::IO { AZStd::shared_lock lock(m_archiveMutex); auto it = AZStd::lower_bound(m_arrArchives.begin(), m_arrArchives.end(), szFullPath, NestedArchiveSortByName()); - if (it != m_arrArchives.end() && !azstrnicmp(szFullPath.data(), (*it)->GetFullPath(), szFullPath.size())) + if (it != m_arrArchives.end() && szFullPath == (*it)->GetFullPath()) { return *it; } @@ -2280,7 +1749,7 @@ namespace AZ::IO case RFOM_Disabled: default: - AZ_Assert(false, "File record option %d", aznumeric_cast(eList));; + AZ_Assert(false, "File record option %d", aznumeric_cast(eList)); } return nullptr; } @@ -2325,11 +1794,14 @@ namespace AZ::IO if (m_eRecordFileOpenList != IArchive::RFOM_Disabled) { // we only want to record ASSET access - // assets are identified as things which start with no alias, or with the @assets@ alias - auto assetPath = AZ::IO::FileIOBase::GetInstance()->ConvertToAlias(szFilename); - if (assetPath && (assetPath->Native().starts_with("@assets@") - || assetPath->Native().starts_with("@root@") - || assetPath->Native().starts_with("@projectplatformcache@"))) + // assets are identified as files that are relative to the resolved @assets@ alias path + auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); + const char* aliasValue = fileIoBase->GetAlias("@assets@"); + + if (AZ::IO::FixedMaxPath resolvedFilePath; + fileIoBase->ResolvePath(resolvedFilePath, szFilename) + && aliasValue != nullptr + && resolvedFilePath.IsRelativeTo(aliasValue)) { IResourceList* pList = GetResourceList(m_eRecordFileOpenList); @@ -2360,13 +1832,8 @@ namespace AZ::IO bool prev = false; if (threadId == m_mainThreadId) { - prev = m_disableRuntimeFileAccess[0]; - m_disableRuntimeFileAccess[0] = status; - } - else if (threadId == m_renderThreadId) - { - prev = m_disableRuntimeFileAccess[1]; - m_disableRuntimeFileAccess[1] = status; + prev = m_disableRuntimeFileAccess; + m_disableRuntimeFileAccess = status; } return prev; } @@ -2440,16 +1907,6 @@ namespace AZ::IO return AZ::AllocatorInstance::Get().DeAllocate(p); } - void Archive::Lock() - { - m_csMain.lock(); - } - - void Archive::Unlock() - { - m_csMain.unlock(); - } - // gets the current archive priority ArchiveLocationPriority Archive::GetPakPriority() const { @@ -2519,7 +1976,7 @@ namespace AZ::IO { found = true; - info.m_archiveFilename.InitFromRelativePath(archive->GetFilePath()); + info.m_archiveFilename.InitFromRelativePath(archive->GetFilePath().Native()); info.m_offset = pFileData->GetFileDataOffset(); info.m_compressedSize = entry->desc.lSizeCompressed; info.m_uncompressedSize = entry->desc.lSizeUncompressed; @@ -2561,7 +2018,7 @@ namespace AZ::IO ZipDir::CachePtr pZip; uint32_t nArchiveFlags; - ZipDir::FileEntry* pFileEntry = FindPakFileEntry(szFullPath->Native(), nArchiveFlags, &pZip, false); + ZipDir::FileEntry* pFileEntry = FindPakFileEntry(szFullPath->Native(), nArchiveFlags, &pZip); if (!pFileEntry) { return 0; @@ -2589,7 +2046,7 @@ namespace AZ::IO return static_cast(StreamMediaType::TypeHDD); } - bool Archive::SetPacksAccessible(bool bAccessible, AZStd::string_view pWildcard, uint32_t nFlags) + bool Archive::SetPacksAccessible(bool bAccessible, AZStd::string_view pWildcard) { auto filePath = AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(pWildcard); if (!filePath) @@ -2601,24 +2058,24 @@ namespace AZ::IO return AZ::IO::FileIOBase::GetDirectInstance()->FindFiles(AZ::IO::FixedMaxPath(filePath->ParentPath()).c_str(), AZ::IO::FixedMaxPath(filePath->Filename()).c_str(), [&](const char* filePath) -> bool { - SetPackAccessible(bAccessible, filePath, nFlags); + SetPackAccessible(bAccessible, filePath); return true; }); } - bool Archive::SetPackAccessible(bool bAccessible, AZStd::string_view pName, [[maybe_unused]] uint32_t nFlags) + bool Archive::SetPackAccessible(bool bAccessible, AZStd::string_view pName) { auto szZipPath = AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(pName); if (!szZipPath) { - AZ_Assert(false, "Unable to resolve path for filepath %.*s", aznumeric_cast(pName.size()), pName.data()); + AZ_Assert(false, "Unable to resolve path for filepath %.*s", AZ_STRING_ARG(pName)); return false; } AZStd::unique_lock lock(m_csZips); for (auto it = m_arrZips.begin(); it != m_arrZips.end(); ++it) { - if (!azstricmp(szZipPath->c_str(), it->GetFullPath())) + if (szZipPath == it->GetFullPath()) { return it->pArchive->SetPackAccessible(bAccessible); } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/Archive.h b/Code/Framework/AzFramework/AzFramework/Archive/Archive.h index ec964f7fa3..f08d90a66e 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/Archive.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/Archive.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -26,7 +27,6 @@ #include #include #include -#include #include #include @@ -115,12 +115,12 @@ namespace AZ::IO struct PackDesc { AZ::IO::Path m_pathBindRoot; // the zip binding root - AZStd::string strFileName; // the zip file name (with path) - very useful for debugging so please don't remove + AZ::IO::Path m_strFileName; // the zip file name (with path) - very useful for debugging so please don't remove // [LYN-2376] Remove once legacy slice support is removed bool m_containsLevelPak = false; // indicates whether this archive has level.pak inside it or not - const char* GetFullPath() const { return pZip->GetFilePath(); } + AZ::IO::PathView GetFullPath() const { return pZip->GetFilePath(); } AZStd::intrusive_ptr pArchive; ZipDir::CachePtr pZip; @@ -129,10 +129,7 @@ namespace AZ::IO // ArchiveFindDataSet entire purpose is to keep a reference to the intrusive_ptr of ArchiveFindData // so that it doesn't go out of scope - using ArchiveFindDataSet = AZStd::set, AZ::OSStdAllocator>; - - // given the source relative path, constructs the full path to the file according to the flags - const char* AdjustFileName(AZStd::string_view src, char* dst, size_t dstSize, uint32_t nFlags, bool skipMods = false) override; + using ArchiveFindDataSet = AZStd::set>; /** @@ -154,29 +151,17 @@ namespace AZ::IO //! CompressionBus Handler implementation. void FindCompressionInfo(bool& found, AZ::IO::CompressionInfo& info, const AZStd::string_view filename) override; - //! Processes an alias command line containing multiple aliases. - void ParseAliases(AZStd::string_view szCommandLine) override; - //! adds or removes an alias from the list - if bAdd set to false will remove it - void SetAlias(AZStd::string_view szName, AZStd::string_view szAlias, bool bAdd) override; - //! gets an alias from the list, if any exist. - //! if bReturnSame==true, it will return the input name if an alias doesn't exist. Otherwise returns nullptr - const char* GetAlias(AZStd::string_view szName, bool bReturnSame = true) override; - // Set the localization folder void SetLocalizationFolder(AZStd::string_view sLocalizationFolder) override; const char* GetLocalizationFolder() const override { return m_sLocalizationFolder.c_str(); } const char* GetLocalizationRoot() const override { return m_sLocalizationRoot.c_str(); } - // lock all the operations - void Lock() override; - void Unlock() override; - // open the physical archive file - creates if it doesn't exist // returns nullptr if it's invalid or can't open the file - AZStd::intrusive_ptr OpenArchive(AZStd::string_view szPath, AZStd::string_view bindRoot = {}, uint32_t nFlags = 0, AZStd::intrusive_ptr pData = nullptr) override; + AZStd::intrusive_ptr OpenArchive(AZStd::string_view szPath, AZStd::string_view bindRoot = {}, uint32_t nArchiveFlags = 0, AZStd::intrusive_ptr pData = nullptr) override; // returns the path to the archive in which the file was opened - const char* GetFileArchivePath(AZ::IO::HandleType fileHandle) override; + AZ::IO::PathView GetFileArchivePath(AZ::IO::HandleType fileHandle) override; ////////////////////////////////////////////////////////////////////////// @@ -192,40 +177,31 @@ namespace AZ::IO void RegisterFileAccessSink(IArchiveFileAccessSink* pSink) override; void UnregisterFileAccessSink(IArchiveFileAccessSink* pSink) override; - bool Init(AZStd::string_view szBasePath) override; - void Release() override; - - bool IsInstalledToHDD(AZStd::string_view acFilePath = 0) const override; - // [LYN-2376] Remove 'addLevels' parameter once legacy slice support is removed - bool OpenPack(AZStd::string_view pName, uint32_t nFlags = 0, AZStd::intrusive_ptr pData = nullptr, AZ::IO::FixedMaxPathString* pFullPath = nullptr, bool addLevels = true) override; - bool OpenPack(AZStd::string_view szBindRoot, AZStd::string_view pName, uint32_t nFlags = 0, AZStd::intrusive_ptr pData = nullptr, AZ::IO::FixedMaxPathString* pFullPath = nullptr, bool addLevels = true) override; + bool OpenPack(AZStd::string_view pName, AZStd::intrusive_ptr pData = nullptr, AZ::IO::FixedMaxPathString* pFullPath = nullptr, bool addLevels = true) override; + bool OpenPack(AZStd::string_view szBindRoot, AZStd::string_view pName, AZStd::intrusive_ptr pData = nullptr, AZ::IO::FixedMaxPathString* pFullPath = nullptr, bool addLevels = true) override; // after this call, the file will be unlocked and closed, and its contents won't be used to search for files - bool ClosePack(AZStd::string_view pName, uint32_t nFlags = 0) override; - bool OpenPacks(AZStd::string_view pWildcard, uint32_t nFlags = 0, AZStd::vector* pFullPaths = nullptr) override; - bool OpenPacks(AZStd::string_view szBindRoot, AZStd::string_view pWildcard, uint32_t nFlags = 0, AZStd::vector* pFullPaths = nullptr) override; + bool ClosePack(AZStd::string_view pName) override; + bool OpenPacks(AZStd::string_view pWildcard, AZStd::vector* pFullPaths = nullptr) override; + bool OpenPacks(AZStd::string_view szBindRoot, AZStd::string_view pWildcard, AZStd::vector* pFullPaths = nullptr) override; // closes pack files by the path and wildcard - bool ClosePacks(AZStd::string_view pWildcard, uint32_t nFlags = 0) override; + bool ClosePacks(AZStd::string_view pWildcard) override; //returns if a archive exists matching the wildcard bool FindPacks(AZStd::string_view pWildcardIn) override; // prevent access to specific archive files - bool SetPacksAccessible(bool bAccessible, AZStd::string_view pWildcard, uint32_t nFlags = 0) override; - bool SetPackAccessible(bool bAccessible, AZStd::string_view pName, uint32_t nFlags = 0) override; + bool SetPacksAccessible(bool bAccessible, AZStd::string_view pWildcard) override; + bool SetPackAccessible(bool bAccessible, AZStd::string_view pName) override; // returns the file modification time uint64_t GetModificationTime(AZ::IO::HandleType fileHandle) override; - bool LoadPakToMemory(AZStd::string_view pName, EInMemoryArchiveLocation nLoadArchiveToMemory, AZStd::intrusive_ptr pMemoryBlock = nullptr) override; - void LoadPaksToMemory(int nMaxArchiveSize, bool bLoadToMemory) override; - - AZ::IO::HandleType FOpen(AZStd::string_view pName, const char* mode, uint32_t nPathFlags = 0) override; - size_t FReadRaw(void* data, size_t length, size_t elems, AZ::IO::HandleType handle) override; - size_t FReadRawAll(void* data, size_t nFileSize, AZ::IO::HandleType handle) override; + AZ::IO::HandleType FOpen(AZStd::string_view pName, const char* mode) override; + size_t FRead(void* data, size_t bytesToRead, AZ::IO::HandleType handle) override; void* FGetCachedFileData(AZ::IO::HandleType handle, size_t& nFileSize) override; - size_t FWrite(const void* data, size_t length, size_t elems, AZ::IO::HandleType handle) override; + size_t FWrite(const void* data, size_t bytesToWrite, AZ::IO::HandleType handle) override; size_t FSeek(AZ::IO::HandleType handle, uint64_t seek, int mode) override; uint64_t FTell(AZ::IO::HandleType handle) override; int FFlush(AZ::IO::HandleType handle) override; @@ -234,9 +210,7 @@ namespace AZ::IO AZ::IO::ArchiveFileIterator FindNext(AZ::IO::ArchiveFileIterator fileIterator) override; bool FindClose(AZ::IO::ArchiveFileIterator fileIterator) override; int FEof(AZ::IO::HandleType handle) override; - char* FGets(char*, int, AZ::IO::HandleType) override; - int Getc(AZ::IO::HandleType) override; - int FPrintf(AZ::IO::HandleType handle, const char* format, ...) override; + size_t FGetSize(AZ::IO::HandleType fileHandle) override; size_t FGetSize(AZStd::string_view sFilename, bool bAllowUseFileSystem = false) override; bool IsInPak(AZ::IO::HandleType handle) override; @@ -248,9 +222,6 @@ namespace AZ::IO bool IsFolder(AZStd::string_view sPath) override; IArchive::SignedFileSize GetFileSizeOnDisk(AZStd::string_view filename) override; - // creates a directory - bool MakeDir(AZStd::string_view szPath) override; - // compresses the raw data into raw data. The buffer for compressed data itself with the heap passed. Uses method 8 (deflate) // returns one of the Z_* errors (Z_OK upon success) // MT-safe @@ -275,22 +246,12 @@ namespace AZ::IO IResourceList* GetResourceList(ERecordFileOpenList eList) override; void SetResourceList(ERecordFileOpenList eList, IResourceList* pResourceList) override; - uint32_t ComputeCRC(AZStd::string_view szPath, uint32_t nFileOpenFlags = 0) override; - bool ComputeMD5(AZStd::string_view szPath, uint8_t* md5, uint32_t nFileOpenFlags = 0, bool useDirectFileAccess = false) override; - void DisableRuntimeFileAccess(bool status) override { - m_disableRuntimeFileAccess[0] = status; - m_disableRuntimeFileAccess[1] = status; + m_disableRuntimeFileAccess = status; } bool DisableRuntimeFileAccess(bool status, AZStd::thread_id threadId) override; - bool CheckFileAccessDisabled(AZStd::string_view name, const char* mode) override; - - void SetRenderThreadId(AZStd::thread_id renderThreadId) override - { - m_renderThreadId = renderThreadId; - } // gets the current archive priority ArchiveLocationPriority GetPakPriority() const override; @@ -307,11 +268,11 @@ namespace AZ::IO // Return cached file data for entries inside archive file. CCachedFileDataPtr GetOpenedFileDataInZip(AZ::IO::HandleType file); ZipDir::FileEntry* FindPakFileEntry(AZStd::string_view szPath, uint32_t& nArchiveFlags, - ZipDir::CachePtr* pZip = {}, bool bSkipInMemoryArchives = {}) const; + ZipDir::CachePtr* pZip = {}) const; private: - bool OpenPackCommon(AZStd::string_view szBindRoot, AZStd::string_view pName, uint32_t nArchiveFlags, AZStd::intrusive_ptr pData = nullptr, bool addLevels = true); - bool OpenPacksCommon(AZStd::string_view szDir, AZStd::string_view pWildcardIn, uint32_t nArchiveFlags, AZStd::vector* pFullPaths = nullptr, bool addLevels = true); + bool OpenPackCommon(AZStd::string_view szBindRoot, AZStd::string_view pName, AZStd::intrusive_ptr pData = nullptr, bool addLevels = true); + bool OpenPacksCommon(AZStd::string_view szDir, AZStd::string_view pWildcardIn, AZStd::vector* pFullPaths = nullptr, bool addLevels = true); ZipDir::FileEntry* FindPakFileEntry(AZStd::string_view szPath) const; @@ -346,9 +307,6 @@ namespace AZ::IO AZStd::mutex m_cachedFileRawDataMutex; // For m_pCachedFileRawDataSet using RawDataCacheLockGuard = AZStd::scoped_lock; - // The F* emulation functions critical section: protects all F* functions - // that don't have a chance to be called recursively (to avoid deadlocks) - AZStd::mutex m_csMain; mutable AZStd::shared_mutex m_archiveMutex; ArchiveArray m_arrArchives; @@ -360,8 +318,6 @@ namespace AZ::IO ////////////////////////////////////////////////////////////////////////// IArchive::ERecordFileOpenList m_eRecordFileOpenList = RFOM_Disabled; - using RecordedFilesSet = AZStd::set; - RecordedFilesSet m_recordedFilesSet; AZStd::intrusive_ptr m_pEngineStartupResourceList; @@ -372,28 +328,16 @@ namespace AZ::IO float m_fFileAccessTime{}; // Time used to perform file operations AZStd::vector m_FileAccessSinks; // useful for gathering file access statistics - bool m_disableRuntimeFileAccess[2]{}; + bool m_disableRuntimeFileAccess{}; //threads which we don't want to access files from during the game AZStd::thread_id m_mainThreadId{}; - AZStd::thread_id m_renderThreadId{}; AZStd::fixed_string<128> m_sLocalizationFolder; AZStd::fixed_string<128> m_sLocalizationRoot; - AZStd::set, AZ::OSStdAllocator> m_filesCachedOnHDD; - // [LYN-2376] Remove once legacy slice support is removed LevelPackOpenEvent m_levelOpenEvent; LevelPackCloseEvent m_levelCloseEvent; }; } - -namespace AZ::IO::ArchiveInternal -{ - // Utility function to de-alias archive file opening and file-within-archive opening - // if the file specified was an absolute path but it points at one of the aliases, de-alias it and replace it with that alias. - // this works around problems where the level editor is in control but still mounts asset packs (ie, level.pak mounted as @assets@) - AZStd::optional ConvertAbsolutePathToAliasedPath(AZStd::string_view sourcePath, - AZStd::string_view aliasToLookFor = "@devassets@", AZStd::string_view aliasToReplaceWith = "@assets@"); -} diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFileIO.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFileIO.cpp index 55f7640785..85ce0b6f9a 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFileIO.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFileIO.cpp @@ -5,10 +5,9 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT * */ -#include +#include #include #include // for function<> in the find files callback. -#include #include #include @@ -188,7 +187,7 @@ namespace AZ::IO return IO::ResultCode::Error; } - size_t result = m_archive->FReadRaw(buffer, 1, size, fileHandle); + size_t result = m_archive->FRead(buffer, size, fileHandle); if (bytesRead) { *bytesRead = static_cast(result); @@ -213,7 +212,7 @@ namespace AZ::IO return IO::ResultCode::Error; } - size_t result = m_archive->FWrite(buffer, 1, size, fileHandle); + size_t result = m_archive->FWrite(buffer, size, fileHandle); if (bytesWritten) { *bytesWritten = static_cast(result); @@ -357,14 +356,8 @@ namespace AZ::IO return IO::ResultCode::Error; } - // avoid using AZStd::string if possible - use OSString instead of StringFunc - AZ::OSString destPath(destinationFilePath); + IO::Path destPath(IO::PathView(destinationFilePath).ParentPath()); - AZ::OSString::size_type pos = destPath.find_last_of(AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR); - if (pos != AZ::OSString::npos) - { - destPath.resize(pos); - } CreatePath(destPath.c_str()); if (!Open(destinationFilePath, IO::OpenMode::ModeWrite | IO::OpenMode::ModeBinary, destinationFile)) @@ -466,31 +459,25 @@ namespace AZ::IO return IO::ResultCode::Error; } - AZStd::fixed_string total = filePath; + AZ::IO::FixedMaxPath total = filePath; if (total.empty()) { return IO::ResultCode::Error; } - if (!total.ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR) && !total.ends_with(AZ_WRONG_FILESYSTEM_SEPARATOR)) - { - total.append(AZ_CORRECT_FILESYSTEM_SEPARATOR_STRING); - } - - total.append(filter); + total /= filter; AZ::IO::ArchiveFileIterator fileIterator = m_archive->FindFirst(total.c_str()); if (!fileIterator) { return IO::ResultCode::Success; // its not an actual fatal error to not find anything. } - for (;fileIterator; fileIterator = m_archive->FindNext(fileIterator)) + for (; fileIterator; fileIterator = m_archive->FindNext(fileIterator)) { - total = AZStd::fixed_string::format("%s/%.*s", filePath, aznumeric_cast(fileIterator.m_filename.size()), fileIterator.m_filename.data()); - AZStd::optional resolvedAliasLength = ConvertToAlias(total.data(), total.capacity()); - if (resolvedAliasLength) + total = filePath; + total /= fileIterator.m_filename; + if (ConvertToAlias(total, total)) { - total.resize_no_construct(*resolvedAliasLength); if (!callback(total.c_str())) { break; @@ -510,8 +497,13 @@ namespace AZ::IO const auto fileIt = m_trackedFiles.find(fileHandle); if (fileIt != m_trackedFiles.end()) { - AZ_Assert(filenameSize >= fileIt->second.length(), "Filename size %" PRIu64 " is larger than the size of the tracked file %s:%zu", fileIt->second.c_str(), fileIt->second.size()); - azstrncpy(filename, filenameSize, fileIt->second.c_str(), fileIt->second.length()); + const AZStd::string_view trackedFileView = fileIt->second.Native(); + if (filenameSize <= trackedFileView.size()) + { + return false; + } + size_t trackedFileViewLength = trackedFileView.copy(filename, trackedFileView.size()); + filename[trackedFileViewLength] = '\0'; return true; } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFileIO.h b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFileIO.h index c99fbb14c5..21cef18a7a 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFileIO.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFileIO.h @@ -13,7 +13,6 @@ #include #include #include -#include namespace AZ::IO @@ -78,7 +77,7 @@ namespace AZ::IO protected: // we keep a list of file names ever opened so that we can easily return it. mutable AZStd::recursive_mutex m_operationGuard; - AZStd::unordered_map, AZStd::equal_to, AZ::OSStdAllocator> m_trackedFiles; + AZStd::unordered_map m_trackedFiles; AZStd::fixed_vector m_copyBuffer; IArchive* m_archive; }; diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.cpp index c4c845a832..d7a92efbf6 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.cpp @@ -15,34 +15,6 @@ namespace AZ::IO { - size_t ArchiveFileIteratorHash::operator()(const AZ::IO::ArchiveFileIterator& iter) const - { - return iter.GetHash(); - } - - bool AZStdStringLessCaseInsensitive::operator()(AZStd::string_view left, AZStd::string_view right) const - { - // If one or both strings are 0-length, return true if the left side is smaller, false if they're equal or left is larger. - size_t compareLength = (AZStd::min)(left.size(), right.size()); - if (compareLength == 0) - { - return left.size() < right.size(); - } - - // They're both non-zero, so compare the strings up until the length of the shorter string. - int compareResult = azstrnicmp(left.data(), right.data(), compareLength); - - // If both strings are equal for the number of characters compared, return true if the left side is shorter, false if - // they're equal or left is longer. - if (compareResult == 0) - { - return left.size() < right.size(); - } - - // Return true if the left side should come first alphabetically, false if the right side should. - return compareResult < 0; - } - FileDesc::FileDesc(Attribute fileAttribute, uint64_t fileSize, time_t accessTime, time_t creationTime, time_t writeTime) : nAttrib{ fileAttribute } , nSize{ fileSize } @@ -52,10 +24,9 @@ namespace AZ::IO { } - ArchiveFileIterator::ArchiveFileIterator(FindData* findData, AZStd::string_view filename, const FileDesc& fileDesc) + // ArchiveFileIterator + ArchiveFileIterator::ArchiveFileIterator(FindData* findData) : m_findData{ findData } - , m_filename{ filename } - , m_fileDesc{ fileDesc } { } @@ -73,21 +44,36 @@ namespace AZ::IO return operator++(); } - bool ArchiveFileIterator::operator==(const AZ::IO::ArchiveFileIterator& rhs) const - { - return GetHash() == rhs.GetHash(); - } ArchiveFileIterator::operator bool() const { return m_findData && m_lastFetchValid; } - size_t ArchiveFileIterator::GetHash() const + // FindData::ArchiveFile + FindData::ArchiveFile::ArchiveFile() = default; + FindData::ArchiveFile::ArchiveFile(AZStd::string_view filename, const FileDesc& fileDesc) + : m_filename(filename) + , m_fileDesc(fileDesc) + { + } + size_t FindData::ArchiveFile::GetHash() const { return AZStd::hash{}(m_filename.c_str()); } + bool FindData::ArchiveFile::operator==(const ArchiveFile& rhs) const + { + return GetHash() == rhs.GetHash(); + } + + // FindData::ArchiveFilehash + size_t FindData::ArchiveFileHash::operator()(const ArchiveFile& archiveFile) const + { + return archiveFile.GetHash(); + } + + // FindData void FindData::Scan(IArchive* archive, AZStd::string_view szDir, bool bAllowUseFS, bool bScanZips) { // get the priority into local variable to avoid it changing in the course of @@ -119,40 +105,37 @@ namespace AZ::IO void FindData::ScanFS([[maybe_unused]] IArchive* archive, AZStd::string_view szDirIn) { - AZStd::string searchDirectory; - AZStd::string pattern; - { - AZ::IO::PathString directory{ szDirIn }; - AZ::StringFunc::Path::GetFullPath(directory.c_str(), searchDirectory); - AZ::StringFunc::Path::GetFullFileName(directory.c_str(), pattern); - } - AZ::IO::FileIOBase::GetDirectInstance()->FindFiles(searchDirectory.c_str(), pattern.c_str(), [&](const char* filePath) -> bool + AZ::IO::PathView directory{ szDirIn }; + AZ::IO::FixedMaxPath searchDirectory = directory.ParentPath(); + AZ::IO::FixedMaxPath pattern = directory.Filename(); + auto ScanFileSystem = [this](const char* filePath) -> bool { - AZ::IO::ArchiveFileIterator fileIterator{ nullptr, AZ::IO::PathView(filePath).Filename().Native(), {} }; + ArchiveFile archiveFile{ AZ::IO::PathView(filePath).Filename().Native(), {} }; if (AZ::IO::FileIOBase::GetDirectInstance()->IsDirectory(filePath)) { - fileIterator.m_fileDesc.nAttrib = fileIterator.m_fileDesc.nAttrib | AZ::IO::FileDesc::Attribute::Subdirectory; - m_fileSet.emplace(AZStd::move(fileIterator)); + archiveFile.m_fileDesc.nAttrib = archiveFile.m_fileDesc.nAttrib | AZ::IO::FileDesc::Attribute::Subdirectory; + m_fileSet.emplace(AZStd::move(archiveFile)); } else { if (AZ::IO::FileIOBase::GetDirectInstance()->IsReadOnly(filePath)) { - fileIterator.m_fileDesc.nAttrib = fileIterator.m_fileDesc.nAttrib | AZ::IO::FileDesc::Attribute::ReadOnly; + archiveFile.m_fileDesc.nAttrib = archiveFile.m_fileDesc.nAttrib | AZ::IO::FileDesc::Attribute::ReadOnly; } AZ::u64 fileSize = 0; AZ::IO::FileIOBase::GetDirectInstance()->Size(filePath, fileSize); - fileIterator.m_fileDesc.nSize = fileSize; - fileIterator.m_fileDesc.tWrite = AZ::IO::FileIOBase::GetDirectInstance()->ModificationTime(filePath); + archiveFile.m_fileDesc.nSize = fileSize; + archiveFile.m_fileDesc.tWrite = AZ::IO::FileIOBase::GetDirectInstance()->ModificationTime(filePath); // These times are not supported by our file interface - fileIterator.m_fileDesc.tAccess = fileIterator.m_fileDesc.tWrite; - fileIterator.m_fileDesc.tCreate = fileIterator.m_fileDesc.tWrite; - m_fileSet.emplace(AZStd::move(fileIterator)); + archiveFile.m_fileDesc.tAccess = archiveFile.m_fileDesc.tWrite; + archiveFile.m_fileDesc.tCreate = archiveFile.m_fileDesc.tWrite; + m_fileSet.emplace(AZStd::move(archiveFile)); } return true; - }); + }; + AZ::IO::FileIOBase::GetDirectInstance()->FindFiles(searchDirectory.c_str(), pattern.c_str(), ScanFileSystem); } ////////////////////////////////////////////////////////////////////////// @@ -180,7 +163,7 @@ namespace AZ::IO fileDesc.nAttrib = AZ::IO::FileDesc::Attribute::ReadOnly | AZ::IO::FileDesc::Attribute::Archive; fileDesc.nSize = fileEntry->desc.lSizeUncompressed; fileDesc.tWrite = fileEntry->GetModificationTime(); - m_fileSet.emplace(AZ::IO::ArchiveFileIterator{ this, fname, fileDesc }); + m_fileSet.emplace(fname, fileDesc); } ZipDir::FindDir findDirectoryEntry(zipCache); @@ -193,7 +176,7 @@ namespace AZ::IO } AZ::IO::FileDesc fileDesc; fileDesc.nAttrib = AZ::IO::FileDesc::Attribute::ReadOnly | AZ::IO::FileDesc::Attribute::Archive | AZ::IO::FileDesc::Attribute::Subdirectory; - m_fileSet.emplace(AZ::IO::ArchiveFileIterator{ this, fname, fileDesc }); + m_fileSet.emplace(fname, fileDesc); } }; @@ -208,30 +191,16 @@ namespace AZ::IO // so there's really no way to filter out opening the pack and looking at the files inside. // however, the bind root is not part of the inner zip entry name either // and the ZipDir::FindFile actually expects just the chopped off piece. - // we have to find whats in common between them and check that: + // we have to find the common path segments between them and check that: - auto resolvedBindRoot = AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(it->m_pathBindRoot); - if (!resolvedBindRoot) + AZ::IO::FixedMaxPath bindRoot; + if (!AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(bindRoot, it->m_pathBindRoot)) { AZ_Assert(false, "Unable to resolve Path for archive %s bind root %s", it->GetFullPath(), it->m_pathBindRoot.c_str()); return; } - AZ::IO::FixedMaxPath bindRoot{ *resolvedBindRoot }; - auto [bindRootIter, sourcePathIter] = AZStd::mismatch(AZStd::begin(bindRoot), AZStd::end(bindRoot), - AZStd::begin(sourcePath), AZStd::end(sourcePath)); - if (sourcePathIter == AZStd::begin(sourcePath)) - { - // The path has no characters in common , early out the search as filepath is not part of the iterated zip - continue; - } - - AZ::IO::FixedMaxPath sourcePathRemainder; - for (; sourcePathIter != AZStd::end(sourcePath); ++sourcePathIter) - { - sourcePathRemainder /= *sourcePathIter; - } // Example: // "@assets@\\levels\\*" <--- szDir // "@assets@\\" <--- mount point @@ -256,18 +225,26 @@ namespace AZ::IO // then it means that the pack's mount point itself might be a return value, not the files inside the pack // in that case, we compare the mount point remainder itself with the search filter + auto [bindRootIter, sourcePathIter] = AZStd::mismatch(bindRoot.begin(), bindRoot.end(), + sourcePath.begin(), sourcePath.end()); if (bindRootIter != bindRoot.end()) { + AZ::IO::FixedMaxPath sourcePathRemainder; + for (; sourcePathIter != sourcePath.end(); ++sourcePathIter) + { + sourcePathRemainder /= *sourcePathIter; + } + // Retrieve next path component of the mount point remainder - if (!bindRootIter->empty() && AZStd::wildcard_match(sourcePathRemainder.Native(), bindRootIter->Native())) + if (!bindRootIter->empty() && bindRootIter->Match(sourcePathRemainder.Native())) { AZ::IO::FileDesc fileDesc{ AZ::IO::FileDesc::Attribute::ReadOnly | AZ::IO::FileDesc::Attribute::Archive | AZ::IO::FileDesc::Attribute::Subdirectory }; - m_fileSet.emplace(AZ::IO::ArchiveFileIterator{ this, bindRootIter->Native(), fileDesc }); + m_fileSet.emplace(AZStd::move(bindRootIter->Native()), fileDesc); } } else { - + AZ::IO::FixedMaxPath sourcePathRemainder = sourcePath.LexicallyRelative(bindRoot); // if we get here, it means that the search pattern's root and the mount point for this pack are identical // which means we may search inside the pack. ScanInZip(it->pZip.get(), sourcePathRemainder.Native()); @@ -280,17 +257,17 @@ namespace AZ::IO { if (m_fileSet.empty()) { - AZ::IO::ArchiveFileIterator emptyFileIterator; - emptyFileIterator.m_lastFetchValid = false; - emptyFileIterator.m_findData = this; - return emptyFileIterator; + return {}; } // Remove Fetched item from the FindData map so that the iteration continues - AZ::IO::ArchiveFileIterator fileIterator{ *m_fileSet.begin() }; + AZ::IO::ArchiveFileIterator fileIterator; + auto archiveFileIt = m_fileSet.begin(); + fileIterator.m_filename = archiveFileIt->m_filename; + fileIterator.m_fileDesc = archiveFileIt->m_fileDesc; fileIterator.m_lastFetchValid = true; fileIterator.m_findData = this; - m_fileSet.erase(m_fileSet.begin()); + m_fileSet.erase(archiveFileIt); return fileIterator; } } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.h b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.h index 12544d124d..ebdbf45626 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.h @@ -36,56 +36,72 @@ namespace AZ::IO AZ_DEFINE_ENUM_BITWISE_OPERATORS(AZ::IO::FileDesc::Attribute); + inline constexpr size_t ArchiveFilenameMaxLength = 256; + using ArchiveFileString = AZStd::fixed_string; + class FindData; + //! This is not really an iterator, but a handle + //! that extends ownership of any found filenames from an archive file or the file system struct ArchiveFileIterator { ArchiveFileIterator() = default; - ArchiveFileIterator(FindData* findData, AZStd::string_view filename, const FileDesc& fileDesc); + explicit ArchiveFileIterator(FindData* findData); ArchiveFileIterator operator++(); ArchiveFileIterator operator++(int); - bool operator==(const AZ::IO::ArchiveFileIterator& rhs) const; - explicit operator bool() const; - size_t GetHash() const; - - inline static constexpr size_t FilenameMaxLength = 256; - AZStd::fixed_string m_filename; + ArchiveFileString m_filename; FileDesc m_fileDesc; - AZStd::intrusive_ptr m_findData{}; private: friend class FindData; + friend class Archive; + AZStd::intrusive_ptr m_findData; bool m_lastFetchValid{}; }; - struct ArchiveFileIteratorHash - { - size_t operator()(const AZ::IO::ArchiveFileIterator& iter) const; - }; - struct AZStdStringLessCaseInsensitive - { - bool operator()(AZStd::string_view left, AZStd::string_view right) const; - - using is_transparent = void; - }; class FindData : public AZStd::intrusive_base { public: AZ_CLASS_ALLOCATOR(FindData, AZ::SystemAllocator, 0); FindData() = default; - AZ::IO::ArchiveFileIterator Fetch(); + ArchiveFileIterator Fetch(); void Scan(IArchive* archive, AZStd::string_view path, bool bAllowUseFS = false, bool bScanZips = true); protected: void ScanFS(IArchive* archive, AZStd::string_view path); + // Populates the FileSet with files within the that match the path pattern that is + // if it refers to a file within a bound archive root or returns the archive root + // path if the path pattern matches it. void ScanZips(IArchive* archive, AZStd::string_view path); - using FileSet = AZStd::unordered_set; + class ArchiveFile + { + public: + friend class FindData; + + ArchiveFile(); + ArchiveFile(AZStd::string_view filename, const FileDesc& fileDesc); + + size_t GetHash() const; + bool operator==(const ArchiveFile& rhs) const; + + private: + ArchiveFileString m_filename; + FileDesc m_fileDesc; + }; + + struct ArchiveFileHash + { + size_t operator()(const ArchiveFile& archiveFile) const; + }; + + using FileSet = AZStd::unordered_set; FileSet m_fileSet; }; + } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/IArchive.h b/Code/Framework/AzFramework/AzFramework/Archive/IArchive.h index 5ac417564a..bd9615110a 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/IArchive.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/IArchive.h @@ -12,12 +12,10 @@ #include #include #include -#include #include #include #include #include -#include #include @@ -106,66 +104,6 @@ namespace AZ::IO { AZ_RTTI(IArchive, "{764A2260-FF8A-4C86-B958-EBB0B69D9DFA}"); using FileTime = uint64_t; - // Flags used in file path resolution rules - enum EPathResolutionRules - { - // If used, the source path will be treated as the destination path - // and no transformations will be done. Pass this flag when the path is to be the actual - // path on the disk/in the packs and doesn't need adjustment (or after it has come through adjustments already) - // if this is set, AdjustFileName will not map the input path into the folder (Ex: Shaders will not be converted to Game\Shaders) - FLAGS_PATH_REAL = 1 << 16, - - // AdjustFileName will always copy the file path to the destination path: - // regardless of the returned value, szDestpath can be used - FLAGS_COPY_DEST_ALWAYS = 1 << 17, - - // Adds trailing slash to the path - FLAGS_ADD_TRAILING_SLASH = 1L << 18, - - // if this is set, AdjustFileName will not make relative paths into full paths - FLAGS_NO_FULL_PATH = 1 << 21, - - // if this is set, AdjustFileName will redirect path to disc - FLAGS_REDIRECT_TO_DISC = 1 << 22, - - // if this is set, AdjustFileName will not adjust path for writing files - FLAGS_FOR_WRITING = 1 << 23, - - // if this is set, the archive would be stored in memory (gpu) - FLAGS_PAK_IN_MEMORY = 1 << 25, - - // Store all file names as crc32 in a flat directory structure. - FLAGS_FILENAMES_AS_CRC32 = 1 << 26, - - // if this is set, AdjustFileName will try to find the file under any mod paths we know about - FLAGS_CHECK_MOD_PATHS = 1 << 27, - - // if this is set, AdjustFileName will always check the filesystem/disk and not check inside open archives - FLAGS_NEVER_IN_PAK = 1 << 28, - - // returns existing file name from the local data or existing cache file name - // used by the resource compiler to pass the real file name - FLAGS_RESOLVE_TO_CACHE = 1 << 29, - - // if this is set, the archive would be stored in memory (cpu) - FLAGS_PAK_IN_MEMORY_CPU = 1 << 30, - - // if this is set, the level pak is inside another archive - FLAGS_LEVEL_PAK_INSIDE_PAK = 1 << 31, - }; - - // Used for widening FOpen functionality. They're ignored for the regular File System files. - enum EFOpenFlags - { - // If possible, will prevent the file from being read from memory. - FOPEN_HINT_DIRECT_OPERATION = 1, - // Will prevent a "missing file" warnings to be created. - FOPEN_HINT_QUIET = 1 << 1, - // File should be on disk - FOPEN_ONDISK = 1 << 2, - // Open is done by the streaming thread. - FOPEN_FORSTREAMING = 1 << 3, - }; // enum ERecordFileOpenList @@ -175,8 +113,6 @@ namespace AZ::IO RFOM_Level, // during level loading till export2game -> resourcelist.txt, used to generate the list for level2level loading RFOM_NextLevel // used for level2level loading }; - // the size of the buffer that receives the full path to the file - inline static constexpr size_t MaxPath = 1024; //file location enum used in isFileExist to control where the archive system looks for the file. enum EFileSearchLocation @@ -205,63 +141,31 @@ namespace AZ::IO virtual ~IArchive() = default; - /** - * Deprecated: Use the AZ::IO::FileIOBase::ResolvePath function below that doesn't accept the nFlags or skipMods parameters - * given the source relative path, constructs the full path to the file according to the flags - * returns the pointer to the constructed path (can be either szSourcePath, or szDestPath, or NULL in case of error - */ - // - virtual const char* AdjustFileName(AZStd::string_view src, char* dst, size_t dstSize, uint32_t nFlags, bool skipMods = false) = 0; - - virtual bool Init(AZStd::string_view szBasePath) = 0; - virtual void Release() = 0; - - // Summary: - // Returns true if given archivepath is installed to HDD - // If no file path is given it will return true if whole application is installed to HDD - virtual bool IsInstalledToHDD(AZStd::string_view acFilePath = 0) const = 0; - // after this call, the archive file will be searched for files when they aren't on the OS file system // Arguments: // pName - must not be 0 - virtual bool OpenPack(AZStd::string_view pName, uint32_t nFlags = FLAGS_PATH_REAL, AZStd::intrusive_ptr pData = {}, + virtual bool OpenPack(AZStd::string_view pName, AZStd::intrusive_ptr pData = {}, AZ::IO::FixedMaxPathString* pFullPath = nullptr, bool addLevels = true) = 0; // after this call, the archive file will be searched for files when they aren't on the OS file system - virtual bool OpenPack(AZStd::string_view pBindingRoot, AZStd::string_view pName, uint32_t nFlags = FLAGS_PATH_REAL, + virtual bool OpenPack(AZStd::string_view pBindingRoot, AZStd::string_view pName, AZStd::intrusive_ptr pData = {}, AZ::IO::FixedMaxPathString* pFullPath = nullptr, bool addLevels = true) = 0; // after this call, the file will be unlocked and closed, and its contents won't be used to search for files - virtual bool ClosePack(AZStd::string_view pName, uint32_t nFlags = FLAGS_PATH_REAL) = 0; + virtual bool ClosePack(AZStd::string_view pName) = 0; // opens pack files by the path and wildcard - virtual bool OpenPacks(AZStd::string_view pWildcard, uint32_t nFlags = FLAGS_PATH_REAL, AZStd::vector* pFullPaths = nullptr) = 0; + virtual bool OpenPacks(AZStd::string_view pWildcard, AZStd::vector* pFullPaths = nullptr) = 0; // opens pack files by the path and wildcard - virtual bool OpenPacks(AZStd::string_view pBindingRoot, AZStd::string_view pWildcard, uint32_t nFlags = FLAGS_PATH_REAL, + virtual bool OpenPacks(AZStd::string_view pBindingRoot, AZStd::string_view pWildcard, AZStd::vector* pFullPaths = nullptr) = 0; // closes pack files by the path and wildcard - virtual bool ClosePacks(AZStd::string_view pWildcard, uint32_t nFlags = FLAGS_PATH_REAL) = 0; + virtual bool ClosePacks(AZStd::string_view pWildcard) = 0; //returns if a archive exists matching the wildcard virtual bool FindPacks(AZStd::string_view pWildcardIn) = 0; // Set access status of a archive files with a wildcard - virtual bool SetPacksAccessible(bool bAccessible, AZStd::string_view pWildcard, uint32_t nFlags = FLAGS_PATH_REAL) = 0; + virtual bool SetPacksAccessible(bool bAccessible, AZStd::string_view pWildcard) = 0; // Set access status of a pack file - virtual bool SetPackAccessible(bool bAccessible, AZStd::string_view pName, uint32_t nFlags = FLAGS_PATH_REAL) = 0; - - // Load or unload archive file completely to memory. - virtual bool LoadPakToMemory(AZStd::string_view pName, EInMemoryArchiveLocation eLoadToMemory, AZStd::intrusive_ptr pMemoryBlock = nullptr) = 0; - virtual void LoadPaksToMemory(int nMaxArchiveSize, bool bLoadToMemory) = 0; - - // Processes an alias command line containing multiple aliases. - virtual void ParseAliases(AZStd::string_view szCommandLine) = 0; - // adds or removes an alias from the list - virtual void SetAlias(AZStd::string_view szName, AZStd::string_view szAlias, bool bAdd) = 0; - // gets an alias from the list, if any exist. - // if bReturnSame==true, it will return the input name if an alias doesn't exist. Otherwise returns NULL - virtual const char* GetAlias(AZStd::string_view szName, bool bReturnSame = true) = 0; - - // lock all the operations - virtual void Lock() = 0; - virtual void Unlock() = 0; + virtual bool SetPackAccessible(bool bAccessible, AZStd::string_view pName) = 0; // Set and Get the localization folder name (Languages, Localization, ...) virtual void SetLocalizationFolder(AZStd::string_view sLocalizationFolder) = 0; @@ -273,28 +177,19 @@ namespace AZ::IO // ex: AZ::IO::HandleType fileHandle = FOpen( "test.txt","rbx" ); // mode x is a direct access mode, when used file reads will go directly into the low level file system without any internal data caching. // Text mode is not supported for files in Archives. - // for nFlags @see IArchive::EFOpenFlags - virtual AZ::IO::HandleType FOpen(AZStd::string_view pName, const char* mode, uint32_t nFlags = 0) = 0; - - // Read raw data from file, no endian conversion. - virtual size_t FReadRaw(void* data, size_t length, size_t elems, AZ::IO::HandleType fileHandle) = 0; - - // Read all file contents into the provided memory, nSizeOfFile must be the same as returned by GetFileSize(handle) - // Current seek pointer is ignored and reseted to 0. - // no endian conversion. - virtual size_t FReadRawAll(void* data, size_t nFileSize, AZ::IO::HandleType fileHandle) = 0; + virtual AZ::IO::HandleType FOpen(AZStd::string_view pName, const char* mode) = 0; // Get pointer to the internally cached, loaded data of the file. // WARNING! The returned pointer is only valid while the fileHandle has not been closed. virtual void* FGetCachedFileData(AZ::IO::HandleType fileHandle, size_t& nFileSize) = 0; + // Read raw data from file, no endian conversion. + virtual size_t FRead(void* data, size_t bytesToRead, AZ::IO::HandleType fileHandle) = 0; + // Write file data, cannot be used for writing into the Archive. - // Use INestedArchive interface for writing into the archivefiles. - virtual size_t FWrite(const void* data, size_t length, size_t elems, AZ::IO::HandleType fileHandle) = 0; + // Use INestedArchive interface for writing into the archive files. + virtual size_t FWrite(const void* data, size_t bytesToWrite, AZ::IO::HandleType fileHandle) = 0; - virtual int FPrintf(AZ::IO::HandleType fileHandle, const char* format, ...) = 0; - virtual char* FGets(char*, int, AZ::IO::HandleType) = 0; - virtual int Getc(AZ::IO::HandleType) = 0; virtual size_t FGetSize(AZ::IO::HandleType fileHandle) = 0; virtual size_t FGetSize(AZStd::string_view pName, bool bAllowUseFileSystem = false) = 0; virtual bool IsInPak(AZ::IO::HandleType fileHandle) = 0; @@ -318,7 +213,6 @@ namespace AZ::IO virtual AZStd::intrusive_ptr PoolAllocMemoryBlock(size_t nSize, const char* sUsage, size_t nAlign = 1) = 0; // Arguments: - // nFlags is a combination of EPathResolutionRules flags. virtual ArchiveFileIterator FindFirst(AZStd::string_view pDir, EFileSearchType searchType = eFileSearchType_AllowInZipsOnly) = 0; virtual ArchiveFileIterator FindNext(AZ::IO::ArchiveFileIterator handle) = 0; virtual bool FindClose(AZ::IO::ArchiveFileIterator handle) = 0; @@ -334,9 +228,6 @@ namespace AZ::IO virtual IArchive::SignedFileSize GetFileSizeOnDisk(AZStd::string_view filename) = 0; - // creates a directory - virtual bool MakeDir(AZStd::string_view szPath) = 0; - // open the physical archive file - creates if it doesn't exist // returns NULL if it's invalid or can't open the file // nFlags is a combination of flags from EArchiveFlags enum. @@ -344,8 +235,8 @@ namespace AZ::IO AZStd::intrusive_ptr pData = nullptr) = 0; // returns the path to the archive in which the file was opened - // returns NULL if the file is a physical file, and "" if the path to archive is unknown (shouldn't ever happen) - virtual const char* GetFileArchivePath(AZ::IO::HandleType fileHandle) = 0; + // returns empty path view if the file is a physical file + virtual AZ::IO::PathView GetFileArchivePath(AZ::IO::HandleType fileHandle) = 0; // compresses the raw data into raw data. The buffer for compressed data itself with the heap passed. Uses method 8 (deflate) // returns one of the Z_* errors (Z_OK upon success) @@ -378,25 +269,7 @@ namespace AZ::IO // get the current mode, can be set by RecordFileOpen() virtual IArchive::ERecordFileOpenList GetRecordFileOpenList() = 0; - // computes CRC (zip compatible) for a file - // useful if a huge uncompressed file is generation in non continuous way - // good for big files - low memory overhead (1MB) - // Arguments: - // szPath - must not be 0 - // Returns: - // error code - virtual uint32_t ComputeCRC(AZStd::string_view szPath, uint32_t nFileOpenFlags = 0) = 0; - - // computes MD5 checksum for a file - // good for big files - low memory overhead (1MB) - // Arguments: - // szPath - must not be 0 - // md5 - destination array of uint8_t [16] - // Returns: - // true if success, false on failure - virtual bool ComputeMD5(AZStd::string_view szPath, uint8_t* md5, uint32_t nFileOpenFlags = 0, bool useDirectFileAccess = false) = 0; - - // useful for gathering file access statistics, assert if it was inserted already but then it does not become insersted + // useful for gathering file access statistics, assert if it was inserted already but then it does not become inserted // Arguments: // pSink - must not be 0 virtual void RegisterFileAccessSink(IArchiveFileAccessSink* pSink) = 0; @@ -408,8 +281,6 @@ namespace AZ::IO // When enabled, files accessed at runtime will be tracked virtual void DisableRuntimeFileAccess(bool status) = 0; virtual bool DisableRuntimeFileAccess(bool status, AZStd::thread_id threadId) = 0; - virtual bool CheckFileAccessDisabled(AZStd::string_view name, const char* mode) = 0; - virtual void SetRenderThreadId(AZStd::thread_id renderThreadId) = 0; // gets the current pak priority virtual ArchiveLocationPriority GetPakPriority() const = 0; @@ -431,21 +302,6 @@ namespace AZ::IO using LevelPackCloseEvent = AZ::Event; virtual auto GetLevelPackCloseEvent()->LevelPackCloseEvent* = 0; - // Type-safe endian conversion read. - template - size_t FRead(T* data, size_t elems, AZ::IO::HandleType fileHandle, bool bSwapEndian = false) - { - size_t count = FReadRaw(data, sizeof(T), elems, fileHandle); - SwapEndian(data, count, bSwapEndian); - return count; - } - // Type-independent Write. - template - void FWrite(T* data, size_t elems, AZ::IO::HandleType fileHandle) - { - FWrite((void*)data, sizeof(T), elems, fileHandle); - } - inline static constexpr IArchive::SignedFileSize FILE_NOT_PRESENT = -1; }; diff --git a/Code/Framework/AzFramework/AzFramework/Archive/INestedArchive.h b/Code/Framework/AzFramework/AzFramework/Archive/INestedArchive.h index b45d705259..f85fd273ce 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/INestedArchive.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/INestedArchive.h @@ -9,9 +9,9 @@ #pragma once +#include #include #include -#include #include namespace AZ::IO @@ -71,28 +71,10 @@ namespace AZ::IO // multiple times FLAGS_DONT_COMPACT = 1 << 5, - // flag is set when complete pak has been loaded into memory - FLAGS_IN_MEMORY = 1 << 6, - FLAGS_IN_MEMORY_CPU = 1 << 7, - FLAGS_IN_MEMORY_MASK = FLAGS_IN_MEMORY | FLAGS_IN_MEMORY_CPU, - - // Store all file names as crc32 in a flat directory structure. - FLAGS_FILENAMES_AS_CRC32 = 1 << 8, - - // flag is set when pak is stored on HDD - FLAGS_ON_HDD = 1 << 9, - - //Override pak - paks opened with this flag go at the end of the list and contents will be found before other paks - //Used for patching - FLAGS_OVERRIDE_PAK = 1 << 10, - // Disable a pak file without unloading it, this flag is used in combination with patches and multiplayer - // to ensure that specific paks stay in the position(to keep the same priority) but beeing disabled + // to ensure that specific paks stay in the position(to keep the same priority) but being disabled // when running multiplayer FLAGS_DISABLE_PAK = 1 << 11, - - // flag is set when pak is inside another pak - FLAGS_INSIDE_PAK = 1 << 12, }; using Handle = void*; @@ -122,7 +104,7 @@ namespace AZ::IO virtual int StartContinuousFileUpdate(AZStd::string_view szRelativePath, uint64_t nSize) = 0; // Summary: - // Adds a new file to the zip or update an existing's segment if it is not compressed - just stored + // Adds a new file to the zip or update an existing segment if it is not compressed - just stored // adds a directory (creates several nested directories if needed) // ( name might be misleading as if nOverwriteSeekPos is used the update is not continuous ) // Arguments: @@ -164,7 +146,7 @@ namespace AZ::IO // Summary: // Get the full path to the archive file. - virtual const char* GetFullPath() const = 0; + virtual AZ::IO::PathView GetFullPath() const = 0; // Summary: // Get the flags of this object. diff --git a/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.cpp b/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.cpp index dc1d4aa864..1e0f237df5 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.cpp @@ -174,7 +174,7 @@ namespace AZ::IO return m_pCache->ReadFile(reinterpret_cast(fileHandle), nullptr, pBuffer); } - const char* NestedArchive::GetFullPath() const + AZ::IO::PathView NestedArchive::GetFullPath() const { return m_pCache->GetFilePath(); } @@ -193,19 +193,9 @@ namespace AZ::IO if (nFlagsToSet & FLAGS_RELATIVE_PATHS_ONLY) { m_nFlags |= FLAGS_RELATIVE_PATHS_ONLY; - } - - if (nFlagsToSet & FLAGS_ON_HDD) - { - m_nFlags |= FLAGS_ON_HDD; - } - - if (nFlagsToSet & FLAGS_RELATIVE_PATHS_ONLY || - nFlagsToSet & FLAGS_ON_HDD) - { - // we don't support changing of any other flags return true; } + return false; } @@ -252,20 +242,12 @@ namespace AZ::IO return AZ::IO::FixedMaxPathString{ szRelativePath }; } - if ((szRelativePath.size() > 1 && szRelativePath[1] == ':') || (m_nFlags & FLAGS_ABSOLUTE_PATHS)) + if ((m_nFlags & FLAGS_ABSOLUTE_PATHS) == FLAGS_ABSOLUTE_PATHS) { // make the normalized full path and try to match it against the binding root of this object - auto resolvedPath = AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(szRelativePath); - - // Make sure the resolve path is longer than the bind root and that it starts with the bind root - if (!resolvedPath || resolvedPath->Native().size() <= m_strBindRoot.size() || azstrnicmp(resolvedPath->c_str(), m_strBindRoot.c_str(), m_strBindRoot.size()) != 0) - { - return {}; - } - - // Remove the bind root prefix from the resolved path - resolvedPath->Native().erase(0, m_strBindRoot.size() + 1); - return resolvedPath->Native(); + AZ::IO::FixedMaxPath resolvedPath; + AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(resolvedPath, szRelativePath); + return resolvedPath.LexicallyProximate(m_strBindRoot).Native(); } return AZ::IO::FixedMaxPathString{ szRelativePath }; diff --git a/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.h b/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.h index eac98b2f07..34bbcdc201 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.h @@ -19,15 +19,15 @@ namespace AZ::IO { bool operator()(const INestedArchive* left, const INestedArchive* right) const { - return azstricmp(left->GetFullPath(), right->GetFullPath()) < 0; + return left->GetFullPath() < right->GetFullPath(); } bool operator()(AZStd::string_view left, const INestedArchive* right) const { - return azstrnicmp(left.data(), right->GetFullPath(), left.size()) < 0; + return AZ::IO::PathView(left) < right->GetFullPath(); } bool operator()(const INestedArchive* left, AZStd::string_view right) const { - return azstrnicmp(left->GetFullPath(), right.data(), right.size()) < 0; + return left->GetFullPath() < AZ::IO::PathView(right); } }; @@ -78,7 +78,7 @@ namespace AZ::IO int ReadFile(Handle fileHandle, void* pBuffer) override; // returns the full path to the archive file - const char* GetFullPath() const override; + AZ::IO::PathView GetFullPath() const override; ZipDir::Cache* GetCache(); uint32_t GetFlags() const override; @@ -95,7 +95,7 @@ namespace AZ::IO ZipDir::CachePtr m_pCache; // the binding root may be empty string - in this case, the absolute path binding won't work - AZStd::string m_strBindRoot; + AZ::IO::Path m_strBindRoot; IArchive* m_archive{}; uint32_t m_nFlags{}; }; diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.cpp index 81f26d78b8..328541c4d0 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include @@ -104,24 +103,21 @@ namespace AZ::IO::ZipDir : m_pCache(pCache) , m_bCommitted(false) { - AZ::IO::PathString normalizedPath{ szRelativePath }; - AZ::StringFunc::Path::Normalize(normalizedPath); - AZStd::to_lower(AZStd::begin(normalizedPath), AZStd::end(normalizedPath)); // Update the cache string pool with the relative path to the file - auto pathIt = m_pCache->m_relativePathPool.emplace(normalizedPath); + auto pathIt = m_pCache->m_relativePathPool.emplace(AZ::IO::PathView(szRelativePath).LexicallyNormal()); m_szRelativePath = *pathIt.first; // this is the name of the directory - create it or find it - m_pFileEntry = m_pCache->GetRoot()->Add(m_szRelativePath); + m_pFileEntry = m_pCache->GetRoot()->Add(m_szRelativePath.Native()); if (m_pFileEntry && az_archive_zip_directory_cache_verbosity) { - AZ_TracePrintf("Archive", R"(File "%s" has been added to archive at root "%s")", normalizedPath.c_str(), pCache->GetFilePath()); + AZ_TracePrintf("Archive", R"(File "%s" has been added to archive at root "%s")", pathIt.first->c_str(), pCache->GetFilePath()); } } ~FileEntryTransactionAdd() { if (m_pFileEntry && !m_bCommitted) { - m_pCache->RemoveFile(m_szRelativePath); + m_pCache->RemoveFile(m_szRelativePath.Native()); m_pCache->m_relativePathPool.erase(m_szRelativePath); } } @@ -131,11 +127,11 @@ namespace AZ::IO::ZipDir } AZStd::string_view GetRelativePath() const { - return m_szRelativePath; + return m_szRelativePath.Native(); } private: Cache* m_pCache; - AZStd::string_view m_szRelativePath; + AZ::IO::PathView m_szRelativePath; FileEntry* m_pFileEntry; bool m_bCommitted; }; @@ -587,34 +583,27 @@ namespace AZ::IO::ZipDir // deletes the file from the archive ErrorEnum Cache::RemoveFile(AZStd::string_view szRelativePathSrc) { - // Normalize and lower case the relative path - AZ::IO::PathString szRelativePath{ szRelativePathSrc }; - AZ::StringFunc::Path::Normalize(szRelativePath); - AZStd::to_lower(AZStd::begin(szRelativePath), AZStd::end(szRelativePath)); - AZStd::string_view normalizedRelativePath = szRelativePath; - - // find the last slash in the path - size_t slashOffset = normalizedRelativePath.find_last_of(AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR); + AZ::IO::PathView szRelativePath{ szRelativePathSrc }; AZStd::string_view fileName; // the name of the file to delete FileEntryTree* pDir; // the dir from which the subdir will be deleted - if (slashOffset != AZStd::string_view::npos) + if (szRelativePath.HasParentPath()) { FindDir fd(GetRoot()); // the directory to remove - pDir = fd.FindExact(normalizedRelativePath.substr(0, slashOffset)); + pDir = fd.FindExact(szRelativePath.ParentPath()); if (!pDir) { return ZD_ERROR_DIR_NOT_FOUND;// there is no such directory } - fileName = normalizedRelativePath.substr(slashOffset + 1); + fileName = szRelativePath.Filename().Native(); } else { pDir = GetRoot(); - fileName = normalizedRelativePath; + fileName = szRelativePath.Native(); } ErrorEnum e = pDir->RemoveFile(fileName); @@ -625,7 +614,7 @@ namespace AZ::IO::ZipDir if (az_archive_zip_directory_cache_verbosity) { AZ_TracePrintf("Archive", R"(File "%.*s" has been remove from archive at root "%s")", - aznumeric_cast(fileName.size()), fileName.data(), GetFilePath()); + AZ_STRING_ARG(szRelativePath.Native()), GetFilePath()); } } return e; @@ -635,45 +624,38 @@ namespace AZ::IO::ZipDir // deletes the directory, with all its descendants (files and subdirs) ErrorEnum Cache::RemoveDir(AZStd::string_view szRelativePathSrc) { - // Normalize and lower case the relative path - AZ::IO::PathString szRelativePath{ szRelativePathSrc }; - AZ::StringFunc::Path::Normalize(szRelativePath); - AZStd::to_lower(AZStd::begin(szRelativePath), AZStd::end(szRelativePath)); - AZStd::string_view normalizedRelativePath = szRelativePath; - - // find the last slash in the path - size_t slashOffset = normalizedRelativePath.find_last_of(AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR); + AZ::IO::PathView szRelativePath{ szRelativePathSrc }; AZStd::string_view dirName; // the name of the dir to delete FileEntryTree* pDir; // the dir from which the subdir will be deleted - if (slashOffset != AZStd::string_view::npos) + if (szRelativePath.HasParentPath()) { FindDir fd(GetRoot()); // the directory to remove - pDir = fd.FindExact(normalizedRelativePath.substr(0, slashOffset)); + pDir = fd.FindExact(szRelativePath.ParentPath()); if (!pDir) { return ZD_ERROR_DIR_NOT_FOUND;// there is no such directory } - dirName = normalizedRelativePath.substr(slashOffset + 1); + dirName = szRelativePath.Filename().Native(); } else { pDir = GetRoot(); - dirName = normalizedRelativePath; + dirName = szRelativePath.Native(); } - ErrorEnum e = pDir->RemoveDir(normalizedRelativePath); + ErrorEnum e = pDir->RemoveDir(dirName); if (e == ZD_ERROR_SUCCESS) { m_nFlags |= FLAGS_UNCOMPACTED | FLAGS_CDR_DIRTY; if (az_archive_zip_directory_cache_verbosity) { - AZ_TracePrintf("Archive", R"(File "%.*s" has been remove from archive at root "%s")", - aznumeric_cast(normalizedRelativePath.size()), normalizedRelativePath.data(), GetFilePath()); + AZ_TracePrintf("Archive", R"(Directory "%.*s" has been remove from archive at root "%s")", + AZ_STRING_ARG(szRelativePath.Native()), GetFilePath()); } } return e; @@ -769,9 +751,7 @@ namespace AZ::IO::ZipDir // finds the file by exact path FileEntry* Cache::FindFile(AZStd::string_view szPathSrc, [[maybe_unused]] bool bFullInfo) { - AZ::IO::PathString szPath{ szPathSrc }; - AZ::StringFunc::Path::Normalize(szPath); - AZStd::to_lower(AZStd::begin(szPath), AZStd::end(szPath)); + AZ::IO::PathView szPath{ szPathSrc }; ZipDir::FindFile fd(GetRoot()); FileEntry* fileEntry = fd.FindExact(szPath); @@ -779,19 +759,13 @@ namespace AZ::IO::ZipDir { if (az_archive_zip_directory_cache_verbosity) { - AZ_TracePrintf("Archive", "FindExact failed to find file %s at root %s", szPath.c_str(), GetFilePath()); + AZ_TracePrintf("Archive", "FindExact failed to find file %.*s at root %s", AZ_STRING_ARG(szPath.Native()), GetFilePath()); } return {}; } return fileEntry; } - // returns the size of memory occupied by the instance referred to by this cache - size_t Cache::GetSize() const - { - return sizeof(*this) + m_strFilePath.capacity() + m_treeDir.GetSize() - sizeof(m_treeDir); - } - // refreshes information about the given file entry into this file entry ErrorEnum Cache::Refresh(FileEntryBase* pFileEntry) { diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.h b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.h index 35bf0ae251..646410f8db 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include #include @@ -89,9 +90,6 @@ namespace AZ::IO::ZipDir // refreshes information about the given file entry into this file entry ErrorEnum Refresh(FileEntryBase* pFileEntry); - // returns the size of memory occupied by the instance of this cache - size_t GetSize() const; - // QUICK check to determine whether the file entry belongs to this object bool IsOwnerOf(const FileEntry* pFileEntry) const { @@ -100,9 +98,9 @@ namespace AZ::IO::ZipDir // returns the string - path to the zip file from which this object was constructed. // this will be "" if the object was constructed with a factory that wasn't created with FLAGS_MEMORIZE_ZIP_PATH - const char* GetFilePath() const + AZ::IO::PathView GetFilePath() const { - return m_strFilePath.c_str(); + return m_strFilePath; } FileEntryTree* GetRoot() @@ -135,10 +133,10 @@ namespace AZ::IO::ZipDir FileEntryTree m_treeDir; AZ::IO::HandleType m_fileHandle; AZ::IAllocatorAllocate* m_allocator; - AZStd::string m_strFilePath; + AZ::IO::Path m_strFilePath; // String Pool for persistently storing paths as long as they reside in the cache - AZStd::unordered_set m_relativePathPool; + AZStd::unordered_set m_relativePathPool; // offset to the start of CDR in the file,even if there's no CDR there currently // when a new file is added, it can start from here, but this value will need to be updated then diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.cpp index 0092c7c8f8..6adab594ed 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.cpp @@ -41,13 +41,6 @@ namespace AZ::IO::ZipDir m_encryptedHeaders = ZipFile::HEADERS_NOT_ENCRYPTED; m_signedHeaders = ZipFile::HEADERS_NOT_SIGNED; - if (m_nFlags & FLAGS_FILENAMES_AS_CRC32) - { - m_bBuildFileEntryMap = false; - m_bBuildFileEntryTree = false; - m_bBuildOptimizedFileEntry = true; - } - if (m_nFlags & FLAGS_READ_INSIDE_PAK) { m_fileExt.m_fileIOBase = AZ::IO::FileIOBase::GetInstance(); @@ -88,12 +81,12 @@ namespace AZ::IO::ZipDir if (m_fileExt.m_fileHandle == AZ::IO::InvalidHandle) { - THROW_ZIPDIR_ERROR(ZD_ERROR_IO_FAILED, "Could not open file in binary mode for reading"); + AZ_Warning("Archive", false, R"(ZD_ERROR_IO_FAILED: Could not open file "%s" in binary mode for reading)", szFileName); return {}; } if (!ReadCache(*pCache)) { - THROW_ZIPDIR_ERROR(ZD_ERROR_IO_FAILED, "Could not read the CDR of the pack file."); + AZ_Warning("Archive", false, R"(ZD_ERROR_IO_FAILED: Could not read the CDR of the pack file "%s".)", pCache->m_strFilePath.c_str()); return {}; } } @@ -113,12 +106,12 @@ namespace AZ::IO::ZipDir size_t nFileSize = (size_t)Tell(); Seek(0, SEEK_SET); - AZ_Assert(nFileSize != 0, "File of size 0 will not be open for reading"); + AZ_Warning("Archive", nFileSize != 0, R"(ZD_ERROR_IO_FAILED: File "%s" with size 0 will not be open for reading)", szFileName); if (nFileSize) { if (!ReadCache(*pCache)) { - THROW_ZIPDIR_ERROR(ZD_ERROR_IO_FAILED, "Could not open file in binary mode for reading"); + AZ_Warning("Archive", false, R"(ZD_ERROR_IO_FAILED: Could not open file "%s" in binary mode for reading)", szFileName); return {}; } bOpenForWriting = false; @@ -143,7 +136,7 @@ namespace AZ::IO::ZipDir if (m_fileExt.m_fileHandle == AZ::IO::InvalidHandle) { - THROW_ZIPDIR_ERROR(ZD_ERROR_IO_FAILED, "Could not open file in binary mode for appending (read/write)"); + AZ_Warning("Archive", false, R"(ZD_ERROR_IO_FAILED: Could not open file "%s" in binary mode for appending (read/write))", szFileName); return {}; } } @@ -211,7 +204,7 @@ namespace AZ::IO::ZipDir if (m_headerExtended.nHeaderSize != sizeof(m_headerExtended)) { // Extended Header is not valid - THROW_ZIPDIR_ERROR(ZD_ERROR_DATA_IS_CORRUPT, "Bad extended header"); + AZ_Warning("Archive", false, "ZD_ERROR_DATA_IS_CORRUPT: Bad extended header"); return false; } //We have the header, so read the encryption and signing techniques @@ -224,7 +217,7 @@ namespace AZ::IO::ZipDir if (m_headerExtended.nEncryption != ZipFile::HEADERS_NOT_ENCRYPTED && m_encryptedHeaders != ZipFile::HEADERS_NOT_ENCRYPTED) { //Encryption technique has been specified in both the disk number (old technique) and the custom header (new technique). - THROW_ZIPDIR_ERROR(ZD_ERROR_DATA_IS_CORRUPT, "Unexpected encryption technique in header"); + AZ_Warning("Archive", false, "ZD_ERROR_DATA_IS_CORRUPT: Unexpected encryption technique in header"); return false; } else @@ -240,7 +233,7 @@ namespace AZ::IO::ZipDir break; default: // Unexpected technique - THROW_ZIPDIR_ERROR(ZD_ERROR_DATA_IS_CORRUPT, "Bad encryption technique in header"); + AZ_Warning("Archive", false, "ZD_ERROR_DATA_IS_CORRUPT: Bad encryption technique in header"); return false; } } @@ -255,7 +248,7 @@ namespace AZ::IO::ZipDir break; default: // Unexpected technique - THROW_ZIPDIR_ERROR(ZD_ERROR_DATA_IS_CORRUPT, "Bad signing technique in header"); + AZ_Warning("Archive", false, "ZD_ERROR_DATA_IS_CORRUPT: Bad signing technique in header"); return false; } @@ -266,7 +259,7 @@ namespace AZ::IO::ZipDir Read(&m_headerSignature, sizeof(m_headerSignature)); if (m_headerSignature.nHeaderSize != sizeof(m_headerSignature)) { - THROW_ZIPDIR_ERROR(ZD_ERROR_DATA_IS_CORRUPT, "Bad signature header"); + AZ_Warning("Archive", false, "ZD_ERROR_DATA_IS_CORRUPT: Bad signature header"); return false; } } @@ -274,7 +267,7 @@ namespace AZ::IO::ZipDir else { // Unexpected technique - THROW_ZIPDIR_ERROR(ZD_ERROR_DATA_IS_CORRUPT, "Comment field is the wrong length"); + AZ_Warning("Archive", false, "ZD_ERROR_DATA_IS_CORRUPT: Comment field is the wrong length"); return false; } } @@ -285,7 +278,7 @@ namespace AZ::IO::ZipDir || m_CDREnd.nCDRStartDisk != 0 || m_CDREnd.numEntriesOnDisk != m_CDREnd.numEntriesTotal) { - THROW_ZIPDIR_ERROR(ZD_ERROR_UNSUPPORTED, "Multivolume archive detected. Current version of ZipDir does not support multivolume archives"); + AZ_Warning("Archive", false, "ZD_ERROR_UNSUPPORTED: Multivolume archive detected.Current version of ZipDir does not support multivolume archives"); return false; } @@ -295,7 +288,7 @@ namespace AZ::IO::ZipDir || m_CDREnd.lCDRSize > m_nCDREndPos || m_CDREnd.lCDROffset + m_CDREnd.lCDRSize > m_nCDREndPos) { - THROW_ZIPDIR_ERROR(ZD_ERROR_DATA_IS_CORRUPT, "The central directory offset or size are out of range, the pak is probably corrupt, try to repare or delete the file"); + AZ_Warning("Archive", false, "ZD_ERROR_DATA_IS_CORRUPT: The central directory offset or size are out of range, the pak is probably corrupt, try to repare or delete the file"); return false; } @@ -394,7 +387,12 @@ namespace AZ::IO::ZipDir // if there's nothing to search if (nNewBufPos >= nOldBufPos) { - THROW_ZIPDIR_ERROR(ZD_ERROR_NO_CDR, "Cannot find Central Directory Record in pak. This is either not a pak file, or a pak file without Central Directory. It does not mean that the data is permanently lost, but it may be severely damaged. Please repair the file with external tools, there may be enough information left to recover the file completely."); // we didn't find anything + AZ_Warning("Archive", false, "ZD_ERROR_NO_CDR: Cannot find Central Directory Record in pak." + " This is either not a pak file, or a pak file without Central Directory." + " It does not mean that the data is permanently lost," + " but it may be severely damaged." + " Please repair the file with external tools," + " there may be enough information left to recover the file completely."); // we didn't find anything return false; } @@ -418,7 +416,11 @@ namespace AZ::IO::ZipDir } else { - THROW_ZIPDIR_ERROR(ZD_ERROR_DATA_IS_CORRUPT, "Central Directory Record is followed by a comment of inconsistent length. This might be a minor misconsistency, please try to repair the file. However, it is dangerous to open the file because I will have to guess some structure offsets, which can lead to permanent unrecoverable damage of the archive content"); + AZ_Warning("Archive", false, "ZD_ERROR_DATA_IS_CORRUPT:" + " Central Directory Record is followed by a comment of inconsistent length." + " This might be a minor misconsistency, please try to repair the file.However," + " it is dangerous to open the file because I will have to guess some structure offsets," + " which can lead to permanent unrecoverable damage of the archive content"); return false; } } @@ -436,7 +438,7 @@ namespace AZ::IO::ZipDir nOldBufPos = nNewBufPos; memmove(&pReservedBuffer[CDRSearchWindowSize], pWindow, sizeof(ZipFile::CDREnd) - 1); } - THROW_ZIPDIR_ERROR(ZD_ERROR_UNEXPECTED, "The program flow may not have possibly lead here. This error is unexplainable"); // we shouldn't be here + AZ_Assert(false, "ZD_ERROR_UNEXPECTED: The program flow may not have possibly lead here. This error is unexplainable"); // we shouldn't be here return false; } @@ -460,13 +462,13 @@ namespace AZ::IO::ZipDir if (pBuffer.empty()) // couldn't allocate enough memory for temporary copy of CDR { - THROW_ZIPDIR_ERROR(ZD_ERROR_NO_MEMORY, "Not enough memory to cache Central Directory record for fast initialization. This error may not happen on non-console systems"); + AZ_Warning("Archive", false, "ZD_ERROR_NO_MEMORY: Not enough memory to cache Central Directory record for fast initialization. This error may not happen on non-console systems"); return false; } if (!ReadHeaderData(&pBuffer[0], m_CDREnd.lCDRSize)) { - THROW_ZIPDIR_ERROR(ZD_ERROR_CORRUPTED_DATA, "Archive contains corrupted CDR."); + AZ_Warning("Archive", false, "ZD_ERROR_CORRUPTED_DATA: Archive contains corrupted CDR."); return false; } @@ -482,7 +484,7 @@ namespace AZ::IO::ZipDir if ((pFile->nVersionNeeded & 0xFF) > 20) { - THROW_ZIPDIR_ERROR(ZD_ERROR_UNSUPPORTED, "Cannot read the archive file (nVersionNeeded > 20)."); + AZ_Warning("Archive", false, "ZD_ERROR_UNSUPPORTED: Cannot read the archive file (nVersionNeeded > 20)."); return false; } //if (pFile->lSignature != pFile->SIGNATURE) // Timur, Dont compare signatures as signatue in memory can be overwritten by the code below @@ -492,7 +494,8 @@ namespace AZ::IO::ZipDir // if the record overlaps with the End Of CDR structure, something is wrong if (pEndOfRecord > pEndOfData) { - THROW_ZIPDIR_ERROR(ZD_ERROR_CDR_IS_CORRUPT, "Central Directory record is either corrupt, or truncated, or missing. Cannot read the archive directory"); + AZ_Warning("Archive", false, "ZD_ERROR_CDR_IS_CORRUPT: Central Directory record is either corrupt, or truncated, or missing." + " Cannot read the archive directory"); return false; } @@ -555,13 +558,17 @@ namespace AZ::IO::ZipDir { if (pFileHeader->lLocalHeaderOffset > m_CDREnd.lCDROffset) { - THROW_ZIPDIR_ERROR(ZD_ERROR_CDR_IS_CORRUPT, "Central Directory contains file descriptors pointing outside the archive file boundaries. The archive file is either truncated or damaged. Please try to repair the file"); // the file offset is beyond the CDR: impossible + AZ_Warning("Archive", false, "ZD_ERROR_CDR_IS_CORRUPT:" + " Central Directory contains file descriptors pointing outside the archive file boundaries." + " The archive file is either truncated or damaged.Please try to repair the file"); // the file offset is beyond the CDR: impossible return; } if ((pFileHeader->nMethod == ZipFile::METHOD_STORE || pFileHeader->nMethod == ZipFile::METHOD_STORE_AND_STREAMCIPHER_KEYTABLE) && pFileHeader->desc.lSizeUncompressed != pFileHeader->desc.lSizeCompressed) { - THROW_ZIPDIR_ERROR(ZD_ERROR_VALIDATION_FAILED, "File with STORE compression method declares its compressed size not matching its uncompressed size. File descriptor is inconsistent, archive content may be damaged, please try to repair the archive"); + AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED:" + " File with STORE compression method declares its compressed size not matching its uncompressed size." + " File descriptor is inconsistent, archive content may be damaged, please try to repair the archive"); return; } @@ -617,7 +624,9 @@ namespace AZ::IO::ZipDir //|| pFileHeader->nLastModTime != pLocalFileHeader->nLastModTime ) { - THROW_ZIPDIR_ERROR(ZD_ERROR_VALIDATION_FAILED, "The local file header descriptor doesn't match the basic parameters declared in the global file header in the file. The archive content is misconsistent and may be damaged. Please try to repair the archive"); + AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED:" + " The local file header descriptor doesn't match the basic parameters declared in the global file header in the file." + " The archive content is misconsistent and may be damaged. Please try to repair the archive"); return; } @@ -628,7 +637,9 @@ namespace AZ::IO::ZipDir if (!AZStd::equal(zipFileDataBegin, zipFileDataEnd, reinterpret_cast(pFileHeader + 1), CompareNoCase)) { // either file name, or the extra field do not match - THROW_ZIPDIR_ERROR(ZD_ERROR_VALIDATION_FAILED, "The local file header contains file name which does not match the file name of the global file header. The archive content is misconsistent with its directory. Please repair the archive"); + AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED:" + " The local file header contains file name which does not match the file name of the global file header." + " The archive content is misconsistent with its directory. Please repair the archive"); return; } @@ -642,7 +653,9 @@ namespace AZ::IO::ZipDir if (fileEntry.nFileDataOffset >= m_nCDREndPos) { - THROW_ZIPDIR_ERROR(ZD_ERROR_VALIDATION_FAILED, "The global file header declares the file which crosses the boundaries of the archive. The archive is either corrupted or truncated, please try to repair it"); + AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED:" + " The global file header declares the file which crosses the boundaries of the archive." + " The archive is either corrupted or truncated, please try to repair it"); return; } @@ -686,29 +699,29 @@ namespace AZ::IO::ZipDir case Z_OK: break; case Z_MEM_ERROR: - THROW_ZIPDIR_ERROR(ZD_ERROR_ZLIB_NO_MEMORY, "ZLib reported out-of-memory error"); + AZ_Warning("Archive", false, "ZD_ERROR_ZLIB_NO_MEMORY: ZLib reported out-of-memory error"); return; case Z_BUF_ERROR: - THROW_ZIPDIR_ERROR(ZD_ERROR_ZLIB_CORRUPTED_DATA, "ZLib reported compressed stream buffer error"); + AZ_Warning("Archive", false, "ZD_ERROR_ZLIB_CORRUPTED_DATA: ZLib reported compressed stream buffer error"); return; case Z_DATA_ERROR: - THROW_ZIPDIR_ERROR(ZD_ERROR_ZLIB_CORRUPTED_DATA, "ZLib reported compressed stream data error"); + AZ_Warning("Archive", false, "ZD_ERROR_ZLIB_CORRUPTED_DATA: ZLib reported compressed stream data error"); return; default: - THROW_ZIPDIR_ERROR(ZD_ERROR_ZLIB_FAILED, "ZLib reported an unexpected unknown error"); + AZ_Warning("Archive", false, "ZD_ERROR_ZLIB_FAILED: ZLib reported an unexpected unknown error"); return; } if (nDestSize != fileEntry.desc.lSizeUncompressed) { - THROW_ZIPDIR_ERROR(ZD_ERROR_CORRUPTED_DATA, "Uncompressed stream doesn't match the size of uncompressed file stored in the archive file headers"); + AZ_Warning("Archive", false, "ZD_ERROR_CORRUPTED_DATA: Uncompressed stream doesn't match the size of uncompressed file stored in the archive file headers"); return; } uLong uCRC32 = AZ::Crc32((Bytef*)pUncompressed, nDestSize); if (uCRC32 != fileEntry.desc.lCRC32) { - THROW_ZIPDIR_ERROR(ZD_ERROR_CRC32_CHECK, "Uncompressed stream CRC32 check failed"); + AZ_Warning("Archive", false, "ZD_ERROR_CRC32_CHECK: Uncompressed stream CRC32 check failed"); return; } } @@ -737,7 +750,7 @@ namespace AZ::IO::ZipDir { if (FSeek(&m_fileExt, nPos, nOrigin)) { - THROW_ZIPDIR_ERROR(ZD_ERROR_IO_FAILED, "Cannot fseek() to the new position in the file. This is unexpected error and should not happen under any circumstances. Perhaps some network or disk failure error has caused this"); + AZ_Warning("Archive", false, "ZD_ERROR_IO_FAILED: Cannot fseek() to the new position in the file. This is unexpected error and should not happen under any circumstances. Perhaps some network or disk failure error has caused this"); return; } } @@ -747,7 +760,7 @@ namespace AZ::IO::ZipDir int64_t nPos = FTell(&m_fileExt); if (nPos == -1) { - THROW_ZIPDIR_ERROR(ZD_ERROR_IO_FAILED, "Cannot ftell() position in the archive. This is unexpected error and should not happen under any circumstances. Perhaps some network or disk failure error has caused this"); + AZ_Warning("Archive", false, "ZD_ERROR_IO_FAILED: Cannot ftell() position in the archive. This is unexpected error and should not happen under any circumstances. Perhaps some network or disk failure error has caused this"); return 0; } return nPos; @@ -757,7 +770,7 @@ namespace AZ::IO::ZipDir { if (FRead(&m_fileExt, pDest, nSize, 1) != 1) { - THROW_ZIPDIR_ERROR(ZD_ERROR_IO_FAILED, "Cannot fread() a portion of data from archive"); + AZ_Warning("Archive", false, "ZD_ERROR_IO_FAILED: Cannot fread() a portion of data from archive"); return false; } return true; diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.h b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.h index 1f27cb5504..c31d4d7dfd 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.h @@ -33,20 +33,13 @@ namespace AZ::IO::ZipDir // if this is set, the archive will be created anew (the existing file will be overwritten) FLAGS_CREATE_NEW = 1 << 3, - // Cache will be loaded completely into the memory. - FLAGS_IN_MEMORY = 1 << 4, - FLAGS_IN_MEMORY_CPU = 1 << 5, - - // Store all file names as crc32 in a flat directory structure. - FLAGS_FILENAMES_AS_CRC32 = 1 << 6, - // if this is set, zip path will be searched inside other zips FLAGS_READ_INSIDE_PAK = 1 << 7, }; // initializes the internal structures // nFlags can have FLAGS_READ_ONLY flag, in this case the object will be opened only for reading - CacheFactory (InitMethodEnum nInitMethod, uint32_t nFlags = 0); + CacheFactory(InitMethodEnum nInitMethod, uint32_t nFlags = 0); ~CacheFactory(); // the new function creates a new cache diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirFind.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirFind.cpp index 30c8676f75..4fb1a54ded 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirFind.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirFind.cpp @@ -17,7 +17,7 @@ namespace AZ::IO::ZipDir { - bool FindFile::FindFirst(AZStd::string_view szWildcard) + bool FindFile::FindFirst(AZ::IO::PathView szWildcard) { if (!PreFind(szWildcard)) { @@ -29,7 +29,7 @@ namespace AZ::IO::ZipDir return SkipNonMatchingFiles(); } - bool FindDir::FindFirst(AZStd::string_view szWildcard) + bool FindDir::FindFirst(AZ::IO::PathView szWildcard) { if (!PreFind(szWildcard)) { @@ -42,37 +42,20 @@ namespace AZ::IO::ZipDir } // matches the file wildcard in the m_szWildcard to the given file/dir name - // this takes into account the fact that xxx. is the alias name for xxx - bool FindData::MatchWildcard(AZStd::string_view szName) + bool FindData::MatchWildcard(AZ::IO::PathView szName) { - if (AZStd::wildcard_match(m_szWildcard, szName)) - { - return true; - } - - // check if the file object name contains extension sign (.) - size_t extensionOffset = szName.find('.'); - if (extensionOffset != AZStd::string_view::npos) - { - return false; - } - - // no extension sign - add it - AZStd::fixed_string szAlias{ szName }; - szAlias.push_back('.'); - - return AZStd::wildcard_match(m_szWildcard, szAlias); + return szName.Match(m_szWildcard.Native()); } - FileEntry* FindFile::FindExact(AZStd::string_view szPath) + FileEntry* FindFile::FindExact(AZ::IO::PathView szPath) { if (!PreFind(szPath)) { return nullptr; } - FileEntryTree::FileMap::iterator itFile = m_pDirHeader->FindFile(m_szWildcard.c_str()); + FileEntryTree::FileMap::iterator itFile = m_pDirHeader->FindFile(m_szWildcard); if (itFile == m_pDirHeader->GetFileEnd()) { m_pDirHeader = nullptr; // we didn't find it, fail the search @@ -84,7 +67,7 @@ namespace AZ::IO::ZipDir return m_pDirHeader->GetFileEntry(m_itFile); } - FileEntryTree* FindDir::FindExact(AZStd::string_view szPath) + FileEntryTree* FindDir::FindExact(AZ::IO::PathView szPath) { if (!PreFind(szPath)) { @@ -97,40 +80,50 @@ namespace AZ::IO::ZipDir ////////////////////////////////////////////////////////////////////////// // after this call returns successfully (with true returned), the m_szWildcard - // contains the file name/wildcard and m_pDirHeader contains the directory where + // contains the file name/glob and m_pDirHeader contains the directory where // the file (s) are to be found - bool FindData::PreFind(AZStd::string_view szWildcard) + bool FindData::PreFind(AZ::IO::PathView pathGlob) { if (!m_pRoot) { return false; } - // start the search from the root - m_pDirHeader = m_pRoot; - m_szWildcard = szWildcard; - - // for each path directory, copy it into the wildcard buffer and try to find the subdirectory - for (AZStd::optional pathEntry = AZ::StringFunc::TokenizeNext(szWildcard, AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR); pathEntry; - pathEntry = AZ::StringFunc::TokenizeNext(szWildcard, AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR)) + FileEntryTree* entryTreeHeader = m_pRoot; + // If there is a root path in the glob path, attempt to locate it from the root + if (AZ::IO::PathView rootPath = m_szWildcard.RootPath(); !rootPath.empty()) { - // Update wildcard to new path entry - m_szWildcard = *pathEntry; + FileEntryTree* dirEntry = entryTreeHeader->FindDir(rootPath); + if (dirEntry == nullptr) + { + return false; + } - // If the wildcard parameter that has been passed to TokenizeNext is empty - // Then pathEntry is the final portion of the path - if (!szWildcard.empty()) + entryTreeHeader = dirEntry->GetDirectory(); + pathGlob = pathGlob.RelativePath(); + } + + + AZ::IO::PathView filenameSegment = pathGlob; + // Recurse through the directories within the file tree for each remaining parent path segment + // of pathGlob parameter + auto parentPathIter = pathGlob.begin(); + for (auto filenamePathIter = parentPathIter == pathGlob.end() ? pathGlob.end() : AZStd::next(parentPathIter, 1); + filenamePathIter != pathGlob.end(); ++parentPathIter, ++filenamePathIter) + { + FileEntryTree* dirEntry = entryTreeHeader->FindDir(*parentPathIter); + if (dirEntry == nullptr) { - FileEntryTree* dirEntry = m_pDirHeader->FindDir(*pathEntry); - if (!dirEntry) - { - m_pDirHeader = nullptr; // an intermediate directory has not been found continue the search - return false; - } - m_pDirHeader = dirEntry->GetDirectory(); + return false; } + entryTreeHeader = dirEntry->GetDirectory(); + filenameSegment = *filenamePathIter; } + // At this point the all the intermediate directories have been found + // so update the directory header to point at the last file entry tree + m_pDirHeader = entryTreeHeader; + m_szWildcard = filenameSegment; return true; } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirFind.h b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirFind.h index 2bb3932aa9..f9b773a42d 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirFind.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirFind.h @@ -38,11 +38,10 @@ namespace AZ::IO::ZipDir // after this call returns successfully (with true returned), the m_szWildcard // contains the file name/wildcard and m_pDirHeader contains the directory where // the file (s) are to be found - bool PreFind(AZStd::string_view szWildcard); + bool PreFind(AZ::IO::PathView szWildcard); // matches the file wildcard in the m_szWildcard to the given file/dir name - // this takes into account the fact that xxx. is the alias name for xxx - bool MatchWildcard(AZStd::string_view szName); + bool MatchWildcard(AZ::IO::PathView szName); // the directory inside which the current object (file or directory) is being searched FileEntryTree* m_pDirHeader{}; @@ -50,7 +49,7 @@ namespace AZ::IO::ZipDir FileEntryTree* m_pRoot{}; // the root of the zip file in which to search // the actual wildcard being used in the current scan - the file name wildcard only! - AZStd::fixed_string m_szWildcard; + AZ::IO::FixedMaxPath m_szWildcard; }; class FindFile @@ -66,9 +65,9 @@ namespace AZ::IO::ZipDir { } // if bExactFile is passed, only the file is searched, and besides with the exact name as passed (no wildcards) - bool FindFirst(AZStd::string_view szWildcard); + bool FindFirst(AZ::IO::PathView szWildcard); - FileEntry* FindExact(AZStd::string_view szPath); + FileEntry* FindExact(AZ::IO::PathView szPath); // goes on to the next file entry bool FindNext(); @@ -94,9 +93,9 @@ namespace AZ::IO::ZipDir { } // if bExactFile is passed, only the file is searched, and besides with the exact name as passed (no wildcards) - bool FindFirst(AZStd::string_view szWildcard); + bool FindFirst(AZ::IO::PathView szWildcard); - FileEntryTree* FindExact(AZStd::string_view szPath); + FileEntryTree* FindExact(AZ::IO::PathView szPath); // goes on to the next file entry bool FindNext(); diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirList.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirList.cpp index 2f96a93a82..729f394b9d 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirList.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirList.cpp @@ -68,14 +68,14 @@ namespace AZ::IO::ZipDir { for (FileEntryTree::SubdirMap::iterator it = pTree->GetDirBegin(); it != pTree->GetDirEnd(); ++it) { - AddAllFiles(it->second.get(), AZStd::string::format("%.*s%.*s/", aznumeric_cast(strRoot.size()), strRoot.data(), aznumeric_cast(it->first.size()), it->first.data())); + AddAllFiles(it->second.get(), (AZ::IO::Path(strRoot) / it->first).Native()); } for (FileEntryTree::FileMap::iterator it = pTree->GetFileBegin(); it != pTree->GetFileEnd(); ++it) { FileRecord rec; rec.pFileEntryBase = pTree->GetFileEntry(it); - rec.strPath = AZStd::string::format("%.*s%.*s", aznumeric_cast(strRoot.size()), strRoot.data(), aznumeric_cast(it->first.size()), it->first.data()); + rec.strPath = (AZ::IO::Path(strRoot) / it->first).Native(); push_back(rec); } } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.cpp index 1d09705900..f9aa249339 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.cpp @@ -432,18 +432,18 @@ namespace AZ::IO::ZipDir bool CZipFile::EvaluateSectorSize(const char* filename) { - char volume[AZ_MAX_PATH_LEN]; + AZ::IO::FixedMaxPath volume; - if (AZ::StringFunc::Path::IsRelative(filename)) + if (AZ::IO::PathView(filename).IsRelative()) { - AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(filename, volume, AZ_ARRAY_SIZE(volume)); + AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(volume, filename); } else { - azstrcpy(volume, AZ_ARRAY_SIZE(volume), filename); + volume = filename; } - AZ::IO::FixedMaxPathString drive{ AZ::IO::PathView(volume).RootName().Native() }; + AZ::IO::FixedMaxPath drive = volume.RootName(); if (drive.empty()) { return false; @@ -666,7 +666,9 @@ namespace AZ::IO::ZipDir DirEntry* pEnd = pBegin + this->numDirs; DirEntry* pEntry = AZStd::lower_bound(pBegin, pEnd, szName, pred); #if AZ_TRAIT_LEGACY_CRYPAK_UNIX_LIKE_FILE_SYSTEM - if (pEntry != pEnd && !azstrnicmp(szName.data(), pEntry->GetName(pNamePool), szName.size())) + AZ::IO::PathView searchPath(szName, AZ::IO::WindowsPathSeparator); + AZ::IO::PathView entryPath(pEntry->GetName(pNamePool), AZ::IO::WindowsPathSeparator); + if (pEntry != pEnd && searchPath == entryPath) #else if (pEntry != pEnd && szName == pEntry->GetName(pNamePool)) #endif @@ -690,7 +692,9 @@ namespace AZ::IO::ZipDir FileEntry* pEnd = pBegin + this->numFiles; FileEntry* pEntry = AZStd::lower_bound(pBegin, pEnd, szName, pred); #if AZ_TRAIT_LEGACY_CRYPAK_UNIX_LIKE_FILE_SYSTEM - if (pEntry != pEnd && !azstrnicmp(szName.data(), pEntry->GetName(pNamePool), szName.size())) + AZ::IO::PathView searchPath(szName, AZ::IO::WindowsPathSeparator); + AZ::IO::PathView entryPath(pEntry->GetName(pNamePool), AZ::IO::WindowsPathSeparator); + if (pEntry != pEnd && searchPath == entryPath) #else if (pEntry != pEnd && szName == pEntry->GetName(pNamePool)) #endif @@ -990,13 +994,6 @@ namespace AZ::IO::ZipDir } ////////////////////////////////////////////////////////////////////////// - uint32_t FileNameHash(AZStd::string_view filename) - { - AZ::IO::StackString pathname{ filename }; - AZStd::replace(AZStd::begin(pathname), AZStd::end(pathname), AZ_WRONG_DATABASE_SEPARATOR, AZ_CORRECT_DATABASE_SEPARATOR); - - return AZ::Crc32(pathname); - } int64_t FSeek(CZipFile* file, int64_t origin, int command) { diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.h b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.h index 2f9046f53e..7e1d54b405 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.h @@ -119,8 +119,6 @@ namespace AZ::IO::ZipDir const char* m_szDescription; }; -#define THROW_ZIPDIR_ERROR(ZD_ERR, DESC) AZ_Warning("Archive", false, DESC) - // possible initialization methods enum InitMethodEnum { @@ -157,8 +155,6 @@ namespace AZ::IO::ZipDir int FEof(CZipFile* zipFile); - uint32_t FileNameHash(AZStd::string_view filename); - ////////////////////////////////////////////////////////////////////////// struct SExtraZipFileData diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirTree.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirTree.cpp index 213287ef74..e772cb596a 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirTree.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirTree.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -18,37 +17,42 @@ namespace AZ::IO::ZipDir { // Adds or finds the file. Returns non-initialized structure if it was added, // or an IsInitialized() structure if it was found - FileEntry* FileEntryTree::Add(AZStd::string_view szPath) + FileEntry* FileEntryTree::Add(AZ::IO::PathView inputPathView) { - AZStd::optional pathEntry = AZ::StringFunc::TokenizeNext(szPath, AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR); - if (!pathEntry) + if (inputPathView.empty()) { AZ_Assert(false, "An empty file path cannot be added to the zip file entry tree"); return nullptr; } // If a path separator was found, add a subdirectory - if (!szPath.empty()) + auto inputPathIter = inputPathView.begin(); + AZ::IO::PathView firstPathSegment(*inputPathIter); + auto inputPathNextIter = inputPathIter == inputPathView.end() ? inputPathView.end() : AZStd::next(inputPathIter, 1); + AZ::IO::PathView remainingPath = inputPathNextIter != inputPathView.end() ? + AZStd::string_view(inputPathNextIter->Native().begin(), inputPathView.Native().end()) + : AZStd::string_view{}; + if (!remainingPath.empty()) { - auto dirEntryIter = m_mapDirs.find(*pathEntry); + auto dirEntryIter = m_mapDirs.find(firstPathSegment); // we have a subdirectory here - create the file in it if (dirEntryIter == m_mapDirs.end()) { - dirEntryIter = m_mapDirs.emplace(*pathEntry, AZStd::make_unique()).first; + dirEntryIter = m_mapDirs.emplace(firstPathSegment, AZStd::make_unique()).first; } - return dirEntryIter->second->Add(szPath); + return dirEntryIter->second->Add(remainingPath); } // Add the filename - auto fileEntryIter = m_mapFiles.find(*pathEntry); + auto fileEntryIter = m_mapFiles.find(firstPathSegment); if (fileEntryIter == m_mapFiles.end()) { - fileEntryIter = m_mapFiles.emplace(*pathEntry, AZStd::make_unique()).first; + fileEntryIter = m_mapFiles.emplace(firstPathSegment, AZStd::make_unique()).first; } return fileEntryIter->second.get(); } // adds a file to this directory - ErrorEnum FileEntryTree::Add(AZStd::string_view szPath, const FileEntryBase& file) + ErrorEnum FileEntryTree::Add(AZ::IO::PathView szPath, const FileEntryBase& file) { FileEntry* pFile = Add(szPath); if (!pFile) @@ -63,7 +67,7 @@ namespace AZ::IO::ZipDir return ZD_ERROR_SUCCESS; } - // returns the number of files in this tree, including this and sublevels + // returns the number of files in this tree, including this and subdirectories uint32_t FileEntryTree::NumFilesTotal() const { uint32_t numFiles = aznumeric_cast(m_mapFiles.size()); @@ -91,21 +95,6 @@ namespace AZ::IO::ZipDir m_mapFiles.clear(); } - size_t FileEntryTree::GetSize() const - { - size_t nSize = sizeof(*this); - for (const auto& [dirname, dirEntry] : m_mapDirs) - { - nSize += dirname.size() + sizeof(decltype(m_mapDirs)::value_type) + dirEntry->GetSize(); - } - - for (const auto& [filename, fileEntry] : m_mapFiles) - { - nSize += filename.size() + sizeof(decltype(m_mapFiles)::value_type); - } - return nSize; - } - bool FileEntryTree::IsOwnerOf(const FileEntry* pFileEntry) const { for (const auto& [path, fileEntry] : m_mapFiles) @@ -127,7 +116,7 @@ namespace AZ::IO::ZipDir return false; } - FileEntryTree* FileEntryTree::FindDir(AZStd::string_view szDirName) + FileEntryTree* FileEntryTree::FindDir(AZ::IO::PathView szDirName) { if (auto it = m_mapDirs.find(szDirName); it != m_mapDirs.end()) { @@ -137,7 +126,7 @@ namespace AZ::IO::ZipDir return nullptr; } - FileEntryTree::FileMap::iterator FileEntryTree::FindFile(AZStd::string_view szFileName) + FileEntryTree::FileMap::iterator FileEntryTree::FindFile(AZ::IO::PathView szFileName) { return m_mapFiles.find(szFileName); } @@ -152,7 +141,7 @@ namespace AZ::IO::ZipDir return it == GetDirEnd() ? nullptr : it->second.get(); } - ErrorEnum FileEntryTree::RemoveDir(AZStd::string_view szDirName) + ErrorEnum FileEntryTree::RemoveDir(AZ::IO::PathView szDirName) { SubdirMap::iterator itRemove = m_mapDirs.find(szDirName); if (itRemove == m_mapDirs.end()) @@ -164,7 +153,13 @@ namespace AZ::IO::ZipDir return ZD_ERROR_SUCCESS; } - ErrorEnum FileEntryTree::RemoveFile(AZStd::string_view szFileName) + ErrorEnum FileEntryTree::RemoveAll() + { + Clear(); + return ZD_ERROR_SUCCESS; + } + + ErrorEnum FileEntryTree::RemoveFile(AZ::IO::PathView szFileName) { FileMap::iterator itRemove = m_mapFiles.find(szFileName); if (itRemove == m_mapFiles.end()) diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirTree.h b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirTree.h index cfe539e896..9bdc047a7a 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirTree.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirTree.h @@ -10,6 +10,7 @@ #pragma once #include +#include #include #include #include @@ -24,12 +25,12 @@ namespace AZ::IO::ZipDir // adds a file to this directory // Function can modify szPath input - ErrorEnum Add(AZStd::string_view szPath, const FileEntryBase& file); + ErrorEnum Add(AZ::IO::PathView szPath, const FileEntryBase& file); // Adds or finds the file. Returns non-initialized structure if it was added, // or an IsInitialized() structure if it was found // Function can modify szPath input - FileEntry* Add(AZStd::string_view szPath); + FileEntry* Add(AZ::IO::PathView szPath); // returns the number of files in this tree, including this and sublevels uint32_t NumFilesTotal() const; @@ -45,24 +46,18 @@ namespace AZ::IO::ZipDir m_mapFiles.swap(rThat.m_mapFiles); } - size_t GetSize() const; - bool IsOwnerOf(const FileEntry* pFileEntry) const; // subdirectories - using SubdirMap = AZStd::map>; + using SubdirMap = AZStd::map>; // file entries - using FileMap = AZStd::map>; + using FileMap = AZStd::map>; - FileEntryTree* FindDir(AZStd::string_view szDirName); - ErrorEnum RemoveDir (AZStd::string_view szDirName); - ErrorEnum RemoveAll () - { - Clear(); - return ZD_ERROR_SUCCESS; - } - FileMap::iterator FindFile(AZStd::string_view szFileName); - ErrorEnum RemoveFile(AZStd::string_view szFileName); + FileEntryTree* FindDir(AZ::IO::PathView szDirName); + ErrorEnum RemoveDir(AZ::IO::PathView szDirName); + ErrorEnum RemoveAll(); + FileMap::iterator FindFile(AZ::IO::PathView szFileName); + ErrorEnum RemoveFile(AZ::IO::PathView szFileName); // the FileEntryTree is simultaneously an entry in the dir list AND the directory header FileEntryTree* GetDirectory() { @@ -75,8 +70,8 @@ namespace AZ::IO::ZipDir SubdirMap::iterator GetDirBegin() { return m_mapDirs.begin(); } SubdirMap::iterator GetDirEnd() { return m_mapDirs.end(); } uint32_t NumDirs() const { return aznumeric_cast(m_mapDirs.size()); } - AZStd::string_view GetFileName(FileMap::iterator it) { return it->first; } - AZStd::string_view GetDirName(SubdirMap::iterator it) { return it->first; } + AZStd::string_view GetFileName(FileMap::iterator it) { return it->first.Native(); } + AZStd::string_view GetDirName(SubdirMap::iterator it) { return it->first.Native(); } FileEntry* GetFileEntry(FileMap::iterator it); FileEntryTree* GetDirEntry(SubdirMap::iterator it); diff --git a/Code/Framework/AzFramework/AzFramework/IO/LocalFileIO.cpp b/Code/Framework/AzFramework/AzFramework/IO/LocalFileIO.cpp index 50190ac7d5..4072c17ea0 100644 --- a/Code/Framework/AzFramework/AzFramework/IO/LocalFileIO.cpp +++ b/Code/Framework/AzFramework/AzFramework/IO/LocalFileIO.cpp @@ -734,7 +734,7 @@ namespace AZ { if (AZ::StringFunc::StartsWith(pathStrView, aliasKey)) { - // Reduce of the size result result path by the size of the and add the resolved alias size + // Add to the size of result path by the resolved alias length - the alias key length AZStd::string_view postAliasView = pathStrView.substr(aliasKey.size()); size_t requiredFixedMaxPathSize = postAliasView.size(); requiredFixedMaxPathSize += aliasValue.size(); diff --git a/Code/Framework/AzFramework/CMakeLists.txt b/Code/Framework/AzFramework/CMakeLists.txt index 1164d81f04..6f6e47855e 100644 --- a/Code/Framework/AzFramework/CMakeLists.txt +++ b/Code/Framework/AzFramework/CMakeLists.txt @@ -29,7 +29,6 @@ ly_add_target( AZ::AzCore PUBLIC AZ::GridMate - 3rdParty::md5 3rdParty::zlib 3rdParty::zstd 3rdParty::lz4 diff --git a/Code/Framework/AzFramework/Tests/ArchiveCompressionTests.cpp b/Code/Framework/AzFramework/Tests/ArchiveCompressionTests.cpp index b8eb8d2f9f..d2a6d63db5 100644 --- a/Code/Framework/AzFramework/Tests/ArchiveCompressionTests.cpp +++ b/Code/Framework/AzFramework/Tests/ArchiveCompressionTests.cpp @@ -68,7 +68,7 @@ namespace UnitTest return false; } - if (!archive->OpenPack(path, AZ::IO::IArchive::FLAGS_PATH_REAL)) + if (!archive->OpenPack(path)) { return false; } diff --git a/Code/Framework/AzFramework/Tests/ArchiveTests.cpp b/Code/Framework/AzFramework/Tests/ArchiveTests.cpp index f3764a5e96..ecb495cd61 100644 --- a/Code/Framework/AzFramework/Tests/ArchiveTests.cpp +++ b/Code/Framework/AzFramework/Tests/ArchiveTests.cpp @@ -30,16 +30,17 @@ namespace UnitTest : public ScopedAllocatorSetupFixture { public: + // Use an Immediately invoked function to initlaize the m_stackRecordLevels value of the AZ::SystemAllocator::Descriptor class ArchiveTestFixture() - : m_application{ AZStd::make_unique() } + : ScopedAllocatorSetupFixture( + []() { AZ::SystemAllocator::Descriptor desc; desc.m_stackRecordLevels = 30; return desc; }() + ) + , m_application{ AZStd::make_unique() } { } void SetUp() override { - AZ::ComponentApplication::Descriptor descriptor; - descriptor.m_stackRecordLevels = 30; - AZ::SettingsRegistryInterface* registry = AZ::SettingsRegistry::Get(); auto projectPathKey = @@ -47,7 +48,7 @@ namespace UnitTest registry->Set(projectPathKey, "AutomatedTesting"); AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddRuntimeFilePaths(*registry); - m_application->Start(descriptor); + m_application->Start({}); // Without this, the user settings component would attempt to save on finalize/shutdown. Since the file is // shared across the whole engine, if multiple tests are run in parallel, the saving could cause a crash // in the unit tests. @@ -68,7 +69,7 @@ namespace UnitTest return false; } - return archive->OpenPack(path, AZ::IO::IArchive::FLAGS_PATH_REAL) && archive->ClosePack(path); + return archive->OpenPack(path) && archive->ClosePack(path); } template @@ -116,7 +117,7 @@ namespace UnitTest { // Canary tests first - AZ::IO::HandleType fileHandle = archive->FOpen(testFilePath, "rb", 0); + AZ::IO::HandleType fileHandle = archive->FOpen(testFilePath, "rb"); ASSERT_NE(AZ::IO::InvalidHandle, fileHandle); @@ -136,9 +137,9 @@ namespace UnitTest // open already open file and call FGetCachedFileData fileSize = 0; { - AZ::IO::HandleType fileHandle2 = archive->FOpen(testFilePath, "rb", 0); + AZ::IO::HandleType fileHandle2 = archive->FOpen(testFilePath, "rb"); char* pFileBuffer3 = (char*)archive->FGetCachedFileData(fileHandle2, fileSize); - ASSERT_NE(nullptr,pFileBuffer3); + ASSERT_NE(nullptr, pFileBuffer3); EXPECT_EQ(dataLen, fileSize); EXPECT_EQ(0, memcmp(pFileBuffer3, testData, dataLen)); archive->FClose(fileHandle2); @@ -174,7 +175,7 @@ namespace UnitTest // Multithreaded Test #2 reading from the same file concurrently auto concurrentArchiveFileReadFunc = [archive, testFilePath, dataLen, testData]() { - AZ::IO::HandleType threadFileHandle = archive->FOpen(testFilePath, "rb", 0); + AZ::IO::HandleType threadFileHandle = archive->FOpen(testFilePath, "rb"); if (threadFileHandle == AZ::IO::InvalidHandle) { @@ -305,7 +306,7 @@ namespace UnitTest // open and fetch the opened pak file using a *.pak AZStd::vector fullPaths; - archive->OpenPacks("@usercache@/*.pak", AZ::IO::IArchive::EPathResolutionRules::FLAGS_PATH_REAL, &fullPaths); + archive->OpenPacks("@usercache@/*.pak", &fullPaths); EXPECT_TRUE(AZStd::any_of(fullPaths.cbegin(), fullPaths.cend(), [](auto& path) { return path.ends_with("one.pak"); })); EXPECT_TRUE(AZStd::any_of(fullPaths.cbegin(), fullPaths.cend(), [](auto& path) { return path.ends_with("two.pak"); })); } @@ -712,13 +713,16 @@ namespace UnitTest EXPECT_TRUE(cpfio.Exists("@log@/unittesttemp/copiedfile2.xml")); // find files test. - + AZ::IO::FixedMaxPath resolvedTestFilePath; + EXPECT_TRUE(cpfio.ResolvePath(resolvedTestFilePath, AZ::IO::PathView("@assets@/testfile.xml"))); bool foundIt = false; // note that this file exists only in the archive. - cpfio.FindFiles("@assets@", "*.xml", [&foundIt](const char* foundName) + cpfio.FindFiles("@assets@", "*.xml", [&foundIt, &cpfio, &resolvedTestFilePath](const char* foundName) { + AZ::IO::FixedMaxPath resolvedFoundPath; + EXPECT_TRUE(cpfio.ResolvePath(resolvedFoundPath, AZ::IO::PathView(foundName))); // according to the contract stated in the FileIO.h file, we expect full paths. (Aliases are full paths) - if (azstricmp(foundName, "@assets@/testfile.xml") == 0) + if (resolvedTestFilePath == resolvedFoundPath) { foundIt = true; return false; @@ -803,40 +807,49 @@ namespace UnitTest EXPECT_TRUE(archive->ClosePack(realNameBuf)); } - TEST_F(ArchiveTestFixture, IResourceList_Add_EmptyFileName_DoesNotCrash) + TEST_F(ArchiveTestFixture, IResourceList_Add_EmptyFileName_DoesNotInsert) { AZ::IO::IResourceList* reslist = AZ::Interface::Get()->GetResourceList(AZ::IO::IArchive::RFOM_EngineStartup); ASSERT_NE(nullptr, reslist); reslist->Clear(); reslist->Add(""); - EXPECT_STREQ(reslist->GetFirst(), ""); - reslist->Clear(); + EXPECT_EQ(nullptr, reslist->GetFirst()); } - TEST_F(ArchiveTestFixture, IResourceList_Add_RegularFileName_NormalizesAppropriately) + TEST_F(ArchiveTestFixture, IResourceList_Add_RegularFileName_ResolvesAppropriately) { AZ::IO::IResourceList* reslist = AZ::Interface::Get()->GetResourceList(AZ::IO::IArchive::RFOM_EngineStartup); ASSERT_NE(nullptr, reslist); + AZ::IO::FileIOBase* ioBase = AZ::IO::FileIOBase::GetInstance(); + ASSERT_NE(nullptr, ioBase); + AZ::IO::FixedMaxPath resolvedTestPath; + EXPECT_TRUE(ioBase->ResolvePath(resolvedTestPath, "blah/blah/abcde")); + reslist->Clear(); reslist->Add("blah\\blah/AbCDE"); - - // it normalizes the string, so the slashes flip and everything is lowercased. - EXPECT_STREQ(reslist->GetFirst(), "blah/blah/abcde"); + AZ::IO::FixedMaxPath resolvedAddedPath; + EXPECT_TRUE(ioBase->ResolvePath(resolvedAddedPath, reslist->GetFirst())); + EXPECT_EQ(resolvedTestPath, resolvedAddedPath); reslist->Clear(); } - TEST_F(ArchiveTestFixture, IResourceList_Add_ReallyShortFileName_NormalizesAppropriately) + TEST_F(ArchiveTestFixture, IResourceList_Add_ReallyShortFileName_ResolvesAppropriately) { AZ::IO::IResourceList* reslist = AZ::Interface::Get()->GetResourceList(AZ::IO::IArchive::RFOM_EngineStartup); ASSERT_NE(nullptr, reslist); + AZ::IO::FileIOBase* ioBase = AZ::IO::FileIOBase::GetInstance(); + ASSERT_NE(nullptr, ioBase); + AZ::IO::FixedMaxPath resolvedTestPath; + EXPECT_TRUE(ioBase->ResolvePath(resolvedTestPath, "a")); + reslist->Clear(); reslist->Add("A"); - - // it normalizes the string, so the slashes flip and everything is lowercased. - EXPECT_STREQ(reslist->GetFirst(), "a"); + AZ::IO::FixedMaxPath resolvedAddedPath; + EXPECT_TRUE(ioBase->ResolvePath(resolvedAddedPath, reslist->GetFirst())); + EXPECT_EQ(resolvedTestPath, resolvedAddedPath); reslist->Clear(); } @@ -848,7 +861,7 @@ namespace UnitTest AZ::IO::FileIOBase* ioBase = AZ::IO::FileIOBase::GetInstance(); ASSERT_NE(nullptr, ioBase); - const char *assetsPath = ioBase->GetAlias("@assets@"); + const char* assetsPath = ioBase->GetAlias("@assets@"); ASSERT_NE(nullptr, assetsPath); auto stringToAdd = AZ::IO::Path(assetsPath) / "textures" / "test.dds"; @@ -864,195 +877,4 @@ namespace UnitTest EXPECT_EQ(resolvedAddedPath, resolvedResourcePath); reslist->Clear(); } - - - class ArchiveUnitTestsWithAllocators - : public ScopedAllocatorSetupFixture - { - protected: - - void SetUp() override - { - m_localFileIO = aznew AZ::IO::LocalFileIO(); - AZ::IO::FileIOBase::SetDirectInstance(m_localFileIO); - m_localFileIO->SetAlias(m_firstAlias.c_str(), m_firstAliasPath.c_str()); - m_localFileIO->SetAlias(m_secondAlias.c_str(), m_secondAliasPath.c_str()); - } - - void TearDown() override - { - AZ::IO::FileIOBase::SetDirectInstance(nullptr); - delete m_localFileIO; - m_localFileIO = nullptr; - } - - AZ::IO::FileIOBase* m_localFileIO = nullptr; - AZStd::string m_firstAlias = "@devassets@"; - AZStd::string m_firstAliasPath = "devassets_absolutepath"; - AZStd::string m_secondAlias = "@assets@"; - AZStd::string m_secondAliasPath = "assets_absolutepath"; - }; - - // ConvertAbsolutePathToAliasedPath tests are built to verify existing behavior doesn't change. - // It's a legacy function and the actual intended behavior is unknown, so these are black box unit tests. - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_NullString_ReturnsSuccess) - { - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath(nullptr); - EXPECT_TRUE(conversionResult); - EXPECT_TRUE(conversionResult->empty()); - } - - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_NoAliasInSource_ReturnsSource) - { - AZStd::string sourceString("NoAlias"); - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath(sourceString.c_str()); - EXPECT_TRUE(conversionResult); - // ConvertAbsolutePathToAliasedPath returns sourceString if there is no alias in the source. - EXPECT_STREQ(sourceString.c_str(), conversionResult->c_str()); - } - - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_NullAliasToLookFor_ReturnsSource) - { - AZStd::string sourceString("NoAlias"); - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath(sourceString.c_str(), nullptr); - EXPECT_TRUE(conversionResult); - EXPECT_STREQ(sourceString.c_str(), conversionResult->c_str()); - } - - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_NullAliasToReplaceWith_ReturnsSource) - { - AZStd::string sourceString("NoAlias"); - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath(sourceString.c_str(), "@SomeAlias", nullptr); - EXPECT_TRUE(conversionResult); - EXPECT_STREQ(sourceString.c_str(), conversionResult->c_str()); - } - - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_NullAliases_ReturnsSource) - { - AZStd::string sourceString("NoAlias"); - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath(sourceString.c_str(), nullptr, nullptr); - EXPECT_TRUE(conversionResult); - EXPECT_STREQ(sourceString.c_str(), conversionResult->c_str()); - } - - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_AbsPathInSource_ReturnsReplacedAlias) - { - // ConvertAbsolutePathToAliasedPath only replaces data if GetDirectInstance is valid. - EXPECT_TRUE(AZ::IO::FileIOBase::GetDirectInstance() != nullptr); - - const char* fullPath = AZ::IO::FileIOBase::GetDirectInstance()->GetAlias(m_firstAlias.c_str()); - AZStd::string sourceString = AZStd::string::format("%s" AZ_CORRECT_FILESYSTEM_SEPARATOR_STRING "SomeStringWithAlias", fullPath); - AZStd::string expectedResult = AZStd::string::format("%s" AZ_CORRECT_FILESYSTEM_SEPARATOR_STRING "somestringwithalias", m_secondAlias.c_str()); - - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath( - sourceString.c_str(), - m_firstAlias.c_str(), // find any instance of FirstAlias in sourceString - m_secondAlias.c_str()); // replace it with SecondAlias - EXPECT_TRUE(conversionResult); - EXPECT_STREQ(conversionResult->c_str(), expectedResult.c_str()); - } - - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_AliasInSource_ReturnsReplacedAlias) - { - // ConvertAbsolutePathToAliasedPath only replaces data if GetDirectInstance is valid. - EXPECT_TRUE(AZ::IO::FileIOBase::GetDirectInstance() != nullptr); - - AZStd::string sourceString = AZStd::string::format("%s" AZ_CORRECT_FILESYSTEM_SEPARATOR_STRING "SomeStringWithAlias", m_firstAlias.c_str()); - AZStd::string expectedResult = AZStd::string::format("%s" AZ_CORRECT_FILESYSTEM_SEPARATOR_STRING "somestringwithalias", m_secondAlias.c_str()); - - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath( - sourceString.c_str(), - m_firstAlias.c_str(), // find any instance of FirstAlias in sourceString - m_secondAlias.c_str()); // replace it with SecondAlias - EXPECT_TRUE(conversionResult); - EXPECT_STREQ(conversionResult->c_str(), expectedResult.c_str()); - } - -#if AZ_TRAIT_OS_USE_WINDOWS_FILE_PATHS - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_AbsPathInSource_DOSSlashInSource_ReturnsReplacedAlias) - { - // ConvertAbsolutePathToAliasedPath only replaces data if GetDirectInstance is valid. - EXPECT_TRUE(AZ::IO::FileIOBase::GetDirectInstance() != nullptr); - - const char* fullPath = AZ::IO::FileIOBase::GetDirectInstance()->GetAlias(m_firstAlias.c_str()); - AZStd::string sourceString = AZStd::string::format("%s" AZ_WRONG_DATABASE_SEPARATOR_STRING "SomeStringWithAlias", fullPath); - AZStd::string expectedResult = AZStd::string::format("%s" AZ_CORRECT_FILESYSTEM_SEPARATOR_STRING "somestringwithalias", m_secondAlias.c_str()); - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath( - sourceString.c_str(), - m_firstAlias.c_str(), // find any instance of FirstAlias in sourceString - m_secondAlias.c_str()); // replace it with SecondAlias - EXPECT_TRUE(conversionResult); - EXPECT_STREQ(conversionResult->c_str(), expectedResult.c_str()); - } -#endif // AZ_TRAIT_OS_USE_WINDOWS_FILE_PATHS - - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_AbsPathInSource_UNIXSlashInSource_ReturnsReplacedAlias) - { - // ConvertAbsolutePathToAliasedPath only replaces data if GetDirectInstance is valid. - EXPECT_TRUE(AZ::IO::FileIOBase::GetDirectInstance() != nullptr); - - const char* fullPath = AZ::IO::FileIOBase::GetDirectInstance()->GetAlias(m_firstAlias.c_str()); - AZStd::string sourceString = AZStd::string::format("%s" AZ_CORRECT_DATABASE_SEPARATOR_STRING "SomeStringWithAlias", fullPath); - AZStd::string expectedResult = AZStd::string::format("%s" AZ_CORRECT_DATABASE_SEPARATOR_STRING "somestringwithalias", m_secondAlias.c_str()); - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath( - sourceString.c_str(), - m_firstAlias.c_str(), // find any instance of FirstAlias in sourceString - m_secondAlias.c_str()); // replace it with SecondAlias - EXPECT_TRUE(conversionResult); - EXPECT_STREQ(conversionResult->c_str(), expectedResult.c_str()); - } - - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_AliasInSource_DOSSlashInSource_ReturnsReplacedAlias) - { - // ConvertAbsolutePathToAliasedPath only replaces data if GetDirectInstance is valid. - EXPECT_TRUE(AZ::IO::FileIOBase::GetDirectInstance() != nullptr); - - AZStd::string sourceString = AZStd::string::format("%s" AZ_WRONG_DATABASE_SEPARATOR_STRING "SomeStringWithAlias", m_firstAlias.c_str()); - AZStd::string expectedResult = AZStd::string::format("%s" AZ_WRONG_DATABASE_SEPARATOR_STRING "somestringwithalias", m_secondAlias.c_str()); - - // sourceString is now (firstAlias)SomeStringWithAlias - - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath( - sourceString.c_str(), - m_firstAlias.c_str(), // find any instance of FirstAlias in sourceString - m_secondAlias.c_str()); // replace it with SecondAlias - EXPECT_TRUE(conversionResult); - EXPECT_STREQ(conversionResult->c_str(), expectedResult.c_str()); - } - - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_AliasInSource_UNIXSlashInSource_ReturnsReplacedAlias) - { - // ConvertAbsolutePathToAliasedPath only replaces data if GetDirectInstance is valid. - EXPECT_TRUE(AZ::IO::FileIOBase::GetDirectInstance() != nullptr); - - AZStd::string sourceString = AZStd::string::format("%s" AZ_CORRECT_DATABASE_SEPARATOR_STRING "SomeStringWithAlias", m_firstAlias.c_str()); - AZStd::string expectedResult = AZStd::string::format("%s" AZ_CORRECT_DATABASE_SEPARATOR_STRING "somestringwithalias", m_secondAlias.c_str()); - - // sourceString is now (firstAlias)SomeStringWithAlias - - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath( - sourceString.c_str(), - m_firstAlias.c_str(), // find any instance of FirstAlias in sourceString - m_secondAlias.c_str()); // replace it with SecondAlias - EXPECT_TRUE(conversionResult); - EXPECT_STREQ(conversionResult->c_str(), expectedResult.c_str()); - } - - TEST_F(ArchiveUnitTestsWithAllocators, ConvertAbsolutePathToAliasedPath_SourceLongerThanMaxPath_ReturnsFailure) - { - const int longPathArraySize = AZ::IO::MaxPathLength + 2; - char longPath[longPathArraySize]; - memset(longPath, 'a', sizeof(char) * longPathArraySize); - longPath[longPathArraySize - 1] = '\0'; - AZ_TEST_START_TRACE_SUPPRESSION; - auto conversionResult = AZ::IO::ArchiveInternal::ConvertAbsolutePathToAliasedPath(longPath); - AZ_TEST_STOP_TRACE_SUPPRESSION(1); - EXPECT_FALSE(conversionResult); - } - - class ArchivePathCompareTestFixture - : public ScopedAllocatorSetupFixture - , public ::testing::WithParamInterface> - { - }; } diff --git a/Code/Framework/AzGameFramework/AzGameFramework/Application/GameApplication.cpp b/Code/Framework/AzGameFramework/AzGameFramework/Application/GameApplication.cpp index c42720311c..f759857505 100644 --- a/Code/Framework/AzGameFramework/AzGameFramework/Application/GameApplication.cpp +++ b/Code/Framework/AzGameFramework/AzGameFramework/Application/GameApplication.cpp @@ -9,12 +9,10 @@ #include "GameApplication.h" #include #include -#include -#include -#include -#include +#include + +#include #include -#include #include namespace AzGameFramework @@ -26,6 +24,28 @@ namespace AzGameFramework GameApplication::GameApplication(int argc, char** argv) : Application(&argc, &argv) { + // In the Launcher Applications the Settings Registry + // can read from the FileIOBase instance if available + m_settingsRegistry->SetUseFileIO(true); + + // Attempt to mount the engine pak from the Executable Directory + // at the Assets alias, otherwise to attempting to mount the engine pak + // from the Cache folder + AZ::IO::FixedMaxPath enginePakPath = AZ::Utils::GetExecutableDirectory(); + enginePakPath /= "Engine.pak"; + if (m_archiveFileIO->Exists(enginePakPath.c_str())) + { + m_archive->OpenPack("@assets@", enginePakPath.Native()); + } + else if (enginePakPath.clear(); m_settingsRegistry->Get(enginePakPath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_CacheRootFolder)) + { + // fall back to checking if there is an Engine.pak in the Asset Cache + enginePakPath /= "Engine.pak"; + if (m_archiveFileIO->Exists(enginePakPath.c_str())) + { + m_archive->OpenPack("@assets@", enginePakPath.Native()); + } + } } GameApplication::~GameApplication() @@ -50,8 +70,8 @@ namespace AzGameFramework AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_CommandLine(registry, m_commandLine, false); AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_ProjectUserRegistry(registry, AZ_TRAIT_OS_PLATFORM_CODENAME, specializations, &scratchBuffer); AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_CommandLine(registry, m_commandLine, false); - AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddRuntimeFilePaths(registry); #endif + AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddRuntimeFilePaths(registry); AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_TargetBuildDependencyRegistry(registry, AZ_TRAIT_OS_PLATFORM_CODENAME, specializations, &scratchBuffer); diff --git a/Code/Legacy/CryCommon/CryFile.h b/Code/Legacy/CryCommon/CryFile.h index 56d7097344..8850b90932 100644 --- a/Code/Legacy/CryCommon/CryFile.h +++ b/Code/Legacy/CryCommon/CryFile.h @@ -9,11 +9,11 @@ // Description : File wrapper. #pragma once -#include -#include -#include -#include +#include +#include #include +#include +#include ////////////////////////////////////////////////////////////////////////// #define CRYFILE_MAX_PATH 260 @@ -28,7 +28,7 @@ public: CCryFile(const char* filename, const char* mode); ~CCryFile(); - bool Open(const char* filename, const char* mode, int nOpenFlagsEx = 0); + bool Open(const char* filename, const char* mode); void Close(); // Summary: @@ -58,15 +58,7 @@ public: // Description: // Retrieves the filename of the selected file. - const char* GetFilename() const { return m_filename; }; - - // Description: - // Retrieves the filename after adjustment to the real relative to engine root path. - // Example: - // Original filename "textures/red.dds" adjusted filename will look like "game/textures/red.dds" - // Return: - // Adjusted filename, this is a pointer to a static string, copy return value if you want to keep it. - const char* GetAdjustedFilename() const; + const char* GetFilename() const { return m_filename.c_str(); }; // Summary: // Checks if file is opened from Archive file. @@ -74,12 +66,11 @@ public: // Summary: // Gets path of archive this file is in. - const char* GetPakPath() const; + AZ::IO::PathView GetPakPath() const; private: - char m_filename[CRYFILE_MAX_PATH]; + AZ::IO::FixedMaxPath m_filename; AZ::IO::HandleType m_fileHandle; - AZ::IO::IArchive* m_pIArchive; }; // Summary: @@ -87,14 +78,12 @@ private: inline CCryFile::CCryFile() { m_fileHandle = AZ::IO::InvalidHandle; - m_pIArchive = gEnv ? gEnv->pCryPak : NULL; } ////////////////////////////////////////////////////////////////////////// inline CCryFile::CCryFile(const char* filename, const char* mode) { m_fileHandle = AZ::IO::InvalidHandle; - m_pIArchive = gEnv ? gEnv->pCryPak : NULL; Open(filename, mode); } @@ -109,22 +98,17 @@ inline CCryFile::~CCryFile() // For nOpenFlagsEx see IArchive::EFOpenFlags // See also: // IArchive::EFOpenFlags -inline bool CCryFile::Open(const char* filename, const char* mode, int nOpenFlagsEx) +inline bool CCryFile::Open(const char* filename, const char* mode) { - char tempfilename[CRYFILE_MAX_PATH] = ""; - azstrcpy(tempfilename, CRYFILE_MAX_PATH, filename); - + m_filename = filename; #if !defined (_RELEASE) - if (gEnv && gEnv->IsEditor() && gEnv->pConsole) + if (auto console = AZ::Interface::Get(); console != nullptr) { - ICVar* const pCvar = gEnv->pConsole->GetCVar("ed_lowercasepaths"); - if (pCvar) + if (bool lowercasePaths{}; console->GetCvarValue("ed_lowercasepaths", lowercasePaths) == AZ::GetValueResult::Success) { - const int lowercasePaths = pCvar->GetIVal(); if (lowercasePaths) { - const AZStd::string lowerString = PathUtil::ToLower(tempfilename); - azstrcpy(tempfilename, CRYFILE_MAX_PATH, lowerString.c_str()); + AZStd::to_lower(m_filename.Native().begin(), m_filename.Native().end()); } } } @@ -133,16 +117,8 @@ inline bool CCryFile::Open(const char* filename, const char* mode, int nOpenFlag { Close(); } - azstrcpy(m_filename, CRYFILE_MAX_PATH, tempfilename); - if (m_pIArchive) - { - m_fileHandle = m_pIArchive->FOpen(tempfilename, mode, nOpenFlagsEx); - } - else - { - AZ::IO::FileIOBase::GetInstance()->Open(tempfilename, AZ::IO::GetOpenModeFromStringMode(mode), m_fileHandle); - } + AZ::IO::FileIOBase::GetInstance()->Open(m_filename.c_str(), AZ::IO::GetOpenModeFromStringMode(mode), m_fileHandle); return m_fileHandle != AZ::IO::InvalidHandle; } @@ -152,27 +128,17 @@ inline void CCryFile::Close() { if (m_fileHandle != AZ::IO::InvalidHandle) { - if (m_pIArchive) - { - m_pIArchive->FClose(m_fileHandle); - } - else - { - AZ::IO::FileIOBase::GetInstance()->Close(m_fileHandle); - } + AZ::IO::FileIOBase::GetInstance()->Close(m_fileHandle); + m_fileHandle = AZ::IO::InvalidHandle; - m_filename[0] = 0; + m_filename.clear(); } } ////////////////////////////////////////////////////////////////////////// inline size_t CCryFile::Write(const void* lpBuf, size_t nSize) { - assert(m_fileHandle != AZ::IO::InvalidHandle); - if (m_pIArchive) - { - return m_pIArchive->FWrite(lpBuf, 1, nSize, m_fileHandle); - } + AZ_Assert(m_fileHandle != AZ::IO::InvalidHandle, "File Handle is invalid. Cannot Write"); if (AZ::IO::FileIOBase::GetInstance()->Write(m_fileHandle, lpBuf, nSize)) { @@ -185,11 +151,7 @@ inline size_t CCryFile::Write(const void* lpBuf, size_t nSize) ////////////////////////////////////////////////////////////////////////// inline size_t CCryFile::ReadRaw(void* lpBuf, size_t nSize) { - assert(m_fileHandle != AZ::IO::InvalidHandle); - if (m_pIArchive) - { - return m_pIArchive->FReadRaw(lpBuf, 1, nSize, m_fileHandle); - } + AZ_Assert(m_fileHandle != AZ::IO::InvalidHandle, "File Handle is invalid. Cannot Read"); AZ::u64 bytesRead = 0; AZ::IO::FileIOBase::GetInstance()->Read(m_fileHandle, lpBuf, nSize, false, &bytesRead); @@ -200,11 +162,7 @@ inline size_t CCryFile::ReadRaw(void* lpBuf, size_t nSize) ////////////////////////////////////////////////////////////////////////// inline size_t CCryFile::GetLength() { - assert(m_fileHandle != AZ::IO::InvalidHandle); - if (m_pIArchive) - { - return m_pIArchive->FGetSize(m_fileHandle); - } + AZ_Assert(m_fileHandle != AZ::IO::InvalidHandle, "File Handle is invalid. Cannot query file length"); //long curr = ftell(m_file); AZ::u64 size = 0; AZ::IO::FileIOBase::GetInstance()->Size(m_fileHandle, size); @@ -214,11 +172,7 @@ inline size_t CCryFile::GetLength() ////////////////////////////////////////////////////////////////////////// inline size_t CCryFile::Seek(size_t seek, int mode) { - assert(m_fileHandle != AZ::IO::InvalidHandle); - if (m_pIArchive) - { - return m_pIArchive->FSeek(m_fileHandle, long(seek), mode); - } + AZ_Assert(m_fileHandle != AZ::IO::InvalidHandle, "File Handle is invalid. Cannot seek in unopen file"); if (AZ::IO::FileIOBase::GetInstance()->Seek(m_fileHandle, seek, AZ::IO::GetSeekTypeFromFSeekMode(mode))) { @@ -230,45 +184,25 @@ inline size_t CCryFile::Seek(size_t seek, int mode) ////////////////////////////////////////////////////////////////////////// inline bool CCryFile::IsInPak() const { - if (m_fileHandle != AZ::IO::InvalidHandle && m_pIArchive) + if (auto archive = AZ::Interface::Get(); + m_fileHandle != AZ::IO::InvalidHandle && archive != nullptr) { - return m_pIArchive->GetFileArchivePath(m_fileHandle) != NULL; + return !archive->GetFileArchivePath(m_fileHandle).empty(); } return false; } ////////////////////////////////////////////////////////////////////////// -inline const char* CCryFile::GetPakPath() const +inline AZ::IO::PathView CCryFile::GetPakPath() const { - if (m_fileHandle != AZ::IO::InvalidHandle && m_pIArchive) + if (auto archive = AZ::Interface::Get(); + m_fileHandle != AZ::IO::InvalidHandle && archive != nullptr) { - const char* sPath = m_pIArchive->GetFileArchivePath(m_fileHandle); - if (sPath != NULL) + if (AZ::IO::PathView sPath(archive->GetFileArchivePath(m_fileHandle)); sPath.empty()) { return sPath; } } - return ""; + return {}; } - -////////////////////////////////////////////////////////////////////////// -inline const char* CCryFile::GetAdjustedFilename() const -{ - static char szAdjustedFile[AZ::IO::IArchive::MaxPath]; - assert(m_pIArchive); - if (!m_pIArchive) - { - return ""; - } - - // Gets mod path to file. - const char* gameUrl = m_pIArchive->AdjustFileName(m_filename, szAdjustedFile, AZ::IO::IArchive::MaxPath, 0); - - // Returns standard path otherwise. - if (gameUrl != &szAdjustedFile[0]) - { - azstrcpy(szAdjustedFile, AZ::IO::IArchive::MaxPath, gameUrl); - } - return szAdjustedFile; -} diff --git a/Code/Legacy/CryCommon/CryPath.h b/Code/Legacy/CryCommon/CryPath.h index 046006f738..197c9b2d56 100644 --- a/Code/Legacy/CryCommon/CryPath.h +++ b/Code/Legacy/CryCommon/CryPath.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "platform.h" diff --git a/Code/Legacy/CryCommon/Mocks/ICryPakMock.h b/Code/Legacy/CryCommon/Mocks/ICryPakMock.h index ab3b3875af..50cae45991 100644 --- a/Code/Legacy/CryCommon/Mocks/ICryPakMock.h +++ b/Code/Legacy/CryCommon/Mocks/ICryPakMock.h @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -20,35 +21,23 @@ struct CryPakMock MOCK_METHOD5(AdjustFileName, const char*(AZStd::string_view src, char* dst, size_t dstSize, uint32_t nFlags, bool skipMods)); MOCK_METHOD1(Init, bool(AZStd::string_view szBasePath)); MOCK_METHOD0(Release, void()); - MOCK_CONST_METHOD1(IsInstalledToHDD, bool(AZStd::string_view acFilePath)); - MOCK_METHOD5(OpenPack, bool(AZStd::string_view, uint32_t, AZStd::intrusive_ptr, AZStd::fixed_string*, bool)); - MOCK_METHOD6(OpenPack, bool(AZStd::string_view, AZStd::string_view, uint32_t, AZStd::intrusive_ptr, AZStd::fixed_string*, bool)); - MOCK_METHOD3(OpenPacks, bool(AZStd::string_view pWildcard, uint32_t nFlags, AZStd::vector>* pFullPaths)); - MOCK_METHOD4(OpenPacks, bool(AZStd::string_view pBindingRoot, AZStd::string_view pWildcard, uint32_t nFlags, AZStd::vector>* pFullPaths)); - MOCK_METHOD2(ClosePack, bool(AZStd::string_view pName, uint32_t nFlags)); - MOCK_METHOD2(ClosePacks, bool(AZStd::string_view pWildcard, uint32_t nFlags)); + MOCK_METHOD4(OpenPack, bool(AZStd::string_view, AZStd::intrusive_ptr, AZ::IO::FixedMaxPathString*, bool)); + MOCK_METHOD5(OpenPack, bool(AZStd::string_view, AZStd::string_view, AZStd::intrusive_ptr, AZ::IO::FixedMaxPathString*, bool)); + MOCK_METHOD2(OpenPacks, bool(AZStd::string_view pWildcard, AZStd::vector* pFullPaths)); + MOCK_METHOD3(OpenPacks, bool(AZStd::string_view pBindingRoot, AZStd::string_view pWildcard, AZStd::vector* pFullPaths)); + MOCK_METHOD1(ClosePack, bool(AZStd::string_view pName)); + MOCK_METHOD1(ClosePacks, bool(AZStd::string_view pWildcard)); MOCK_METHOD1(FindPacks, bool(AZStd::string_view pWildcardIn)); - MOCK_METHOD3(SetPacksAccessible, bool(bool bAccessible, AZStd::string_view pWildcard, uint32_t nFlags)); - MOCK_METHOD3(SetPackAccessible, bool(bool bAccessible, AZStd::string_view pName, uint32_t nFlags)); - MOCK_METHOD3(LoadPakToMemory, bool(AZStd::string_view pName, EInMemoryArchiveLocation eLoadToMemory, AZStd::intrusive_ptr pMemoryBlock)); - MOCK_METHOD2(LoadPaksToMemory, void(int nMaxPakSize, bool bLoadToMemory)); - MOCK_METHOD1(GetMod, const char*(int index)); - MOCK_METHOD1(ParseAliases, void(AZStd::string_view szCommandLine)); - MOCK_METHOD3(SetAlias, void(AZStd::string_view szName, AZStd::string_view szAlias, bool bAdd)); - MOCK_METHOD2(GetAlias, const char*(AZStd::string_view szName, bool bReturnSame)); - MOCK_METHOD0(Lock, void()); - MOCK_METHOD0(Unlock, void()); + MOCK_METHOD2(SetPacksAccessible, bool(bool bAccessible, AZStd::string_view pWildcard)); + MOCK_METHOD2(SetPackAccessible, bool(bool bAccessible, AZStd::string_view pName)); MOCK_METHOD1(SetLocalizationFolder, void(AZStd::string_view sLocalizationFolder)); MOCK_CONST_METHOD0(GetLocalizationFolder, const char*()); MOCK_CONST_METHOD0(GetLocalizationRoot, const char*()); - MOCK_METHOD3(FOpen, AZ::IO::HandleType(AZStd::string_view pName, const char* mode, uint32_t nFlags)); - MOCK_METHOD4(FReadRaw, size_t(void* data, size_t length, size_t elems, AZ::IO::HandleType handle)); - MOCK_METHOD3(FReadRawAll, size_t(void* data, size_t nFileSize, AZ::IO::HandleType handle)); - MOCK_METHOD2(FGetCachedFileData, void*(AZ::IO::HandleType handle, size_t & nFileSize)); - MOCK_METHOD4(FWrite, size_t(const void* data, size_t length, size_t elems, AZ::IO::HandleType handle)); - MOCK_METHOD3(FGets, char*(char*, int, AZ::IO::HandleType)); - MOCK_METHOD1(Getc, int(AZ::IO::HandleType)); + MOCK_METHOD2(FOpen, AZ::IO::HandleType(AZStd::string_view pName, const char* mode)); + MOCK_METHOD2(FGetCachedFileData, void*(AZ::IO::HandleType handle, size_t& nFileSize)); + MOCK_METHOD3(FRead, size_t(void* data, size_t bytesToRead, AZ::IO::HandleType handle)); + MOCK_METHOD3(FWrite, size_t(const void* data, size_t bytesToWrite, AZ::IO::HandleType handle)); MOCK_METHOD1(FGetSize, size_t(AZ::IO::HandleType f)); MOCK_METHOD2(FGetSize, size_t(AZStd::string_view pName, bool bAllowUseFileSystem)); MOCK_METHOD1(IsInPak, bool(AZ::IO::HandleType handle)); @@ -70,9 +59,8 @@ struct CryPakMock MOCK_METHOD2(IsFileExist, bool(AZStd::string_view sFilename, EFileSearchLocation)); MOCK_METHOD1(IsFolder, bool(AZStd::string_view sPath)); MOCK_METHOD1(GetFileSizeOnDisk, AZ::IO::IArchive::SignedFileSize(AZStd::string_view filename)); - MOCK_METHOD1(MakeDir, bool(AZStd::string_view szPath)); MOCK_METHOD4(OpenArchive, AZStd::intrusive_ptr (AZStd::string_view szPath, AZStd::string_view bindRoot, uint32_t nFlags, AZStd::intrusive_ptr pData)); - MOCK_METHOD1(GetFileArchivePath, const char* (AZ::IO::HandleType f)); + MOCK_METHOD1(GetFileArchivePath, AZ::IO::PathView (AZ::IO::HandleType f)); MOCK_METHOD5(RawCompress, int(const void* pUncompressed, size_t* pDestSize, void* pCompressed, size_t nSrcSize, int nLevel)); MOCK_METHOD4(RawUncompress, int(void* pUncompressed, size_t* pDestSize, const void* pCompressed, size_t nSrcSize)); MOCK_METHOD1(RecordFileOpen, void(ERecordFileOpenList eList)); @@ -80,24 +68,15 @@ struct CryPakMock MOCK_METHOD1(GetResourceList, AZ::IO::IResourceList * (ERecordFileOpenList eList)); MOCK_METHOD2(SetResourceList, void(ERecordFileOpenList eList, AZ::IO::IResourceList * pResourceList)); MOCK_METHOD0(GetRecordFileOpenList, AZ::IO::IArchive::ERecordFileOpenList()); - MOCK_METHOD2(ComputeCRC, uint32_t(AZStd::string_view szPath, uint32_t nFileOpenFlags)); - MOCK_METHOD4(ComputeMD5, bool(AZStd::string_view szPath, uint8_t* md5, uint32_t nFileOpenFlags, bool useDirectAccess)); MOCK_METHOD1(RegisterFileAccessSink, void(AZ::IO::IArchiveFileAccessSink * pSink)); MOCK_METHOD1(UnregisterFileAccessSink, void(AZ::IO::IArchiveFileAccessSink * pSink)); MOCK_METHOD1(DisableRuntimeFileAccess, void(bool status)); MOCK_METHOD2(DisableRuntimeFileAccess, bool(bool status, AZStd::thread_id threadId)); - MOCK_METHOD2(CheckFileAccessDisabled, bool(AZStd::string_view name, const char* mode)); - MOCK_METHOD1(SetRenderThreadId, void(AZStd::thread_id renderThreadId)); MOCK_CONST_METHOD0(GetPakPriority, AZ::IO::ArchiveLocationPriority()); MOCK_CONST_METHOD1(GetFileOffsetOnMedia, uint64_t(AZStd::string_view szName)); MOCK_CONST_METHOD1(GetFileMediaType, EStreamSourceMediaType(AZStd::string_view szName)); MOCK_METHOD0(GetLevelPackOpenEvent, auto()->LevelPackOpenEvent*); MOCK_METHOD0(GetLevelPackCloseEvent, auto()->LevelPackCloseEvent*); - // Implementations required for variadic functions - virtual int FPrintf([[maybe_unused]] AZ::IO::HandleType handle, [[maybe_unused]] const char* format, ...) PRINTF_PARAMS(3, 4) - { - return 0; - } }; diff --git a/Code/Legacy/CrySystem/ConsoleBatchFile.cpp b/Code/Legacy/CrySystem/ConsoleBatchFile.cpp index d3a0ccf4de..bdc8d6de7d 100644 --- a/Code/Legacy/CrySystem/ConsoleBatchFile.cpp +++ b/Code/Legacy/CrySystem/ConsoleBatchFile.cpp @@ -91,15 +91,15 @@ bool CConsoleBatchFile::ExecuteConfigFile(const char* sFilename) AZStd::string filenameLog; AZStd::string sfn = PathUtil::GetFile(filename); - if (file.Open(filename.c_str(), "rb", AZ::IO::IArchive::FOPEN_HINT_QUIET | AZ::IO::IArchive::FOPEN_ONDISK)) + if (file.Open(filename.c_str(), "rb")) { filenameLog = AZStd::string("game/") + sfn; } - else if (file.Open((AZStd::string("config/") + sfn).c_str(), "rb", AZ::IO::IArchive::FOPEN_HINT_QUIET | AZ::IO::IArchive::FOPEN_ONDISK)) + else if (file.Open((AZStd::string("config/") + sfn).c_str(), "rb")) { filenameLog = AZStd::string("game/config/") + sfn; } - else if (file.Open((AZStd::string("./") + sfn).c_str(), "rb", AZ::IO::IArchive::FOPEN_HINT_QUIET | AZ::IO::IArchive::FOPEN_ONDISK)) + else if (file.Open((AZStd::string("./") + sfn).c_str(), "rb")) { filenameLog = AZStd::string("./") + sfn; } diff --git a/Code/Legacy/CrySystem/LevelSystem/LevelSystem.cpp b/Code/Legacy/CrySystem/LevelSystem/LevelSystem.cpp index c5d364a025..2e61760d8c 100644 --- a/Code/Legacy/CrySystem/LevelSystem/LevelSystem.cpp +++ b/Code/Legacy/CrySystem/LevelSystem/LevelSystem.cpp @@ -50,12 +50,11 @@ bool CLevelInfo::OpenLevelPak() return false; } - AZStd::string levelpak(m_levelPath); - levelpak += "/level.pak"; - AZStd::fixed_string fullLevelPakPath; - bool bOk = gEnv->pCryPak->OpenPack( - levelpak.c_str(), m_isPak ? AZ::IO::IArchive::FLAGS_LEVEL_PAK_INSIDE_PAK : (unsigned)0, NULL, &fullLevelPakPath, false); - m_levelPakFullPath.assign(fullLevelPakPath.c_str()); + AZ::IO::Path levelPak(m_levelPath); + levelPak /= "level.pak"; + AZ::IO::FixedMaxPathString fullLevelPakPath; + bool bOk = gEnv->pCryPak->OpenPack(levelPak.Native(), nullptr, &fullLevelPakPath, false); + m_levelPakFullPath.assign(fullLevelPakPath.c_str(), fullLevelPakPath.size()); return bOk; } @@ -74,7 +73,7 @@ void CLevelInfo::CloseLevelPak() if (!m_levelPakFullPath.empty()) { - gEnv->pCryPak->ClosePack(m_levelPakFullPath.c_str(), AZ::IO::IArchive::FLAGS_PATH_REAL); + gEnv->pCryPak->ClosePack(m_levelPakFullPath.c_str()); m_levelPakFullPath.clear(); } } @@ -324,8 +323,7 @@ void CLevelSystem::ScanFolder(const char* subfolder, bool modFolder) // Open all the available paks found in the levels folder for (auto iter = pakList.begin(); iter != pakList.end(); iter++) { - AZStd::fixed_string fullLevelPakPath; - gEnv->pCryPak->OpenPack(iter->c_str(), (unsigned)0, nullptr, &fullLevelPakPath, false); + gEnv->pCryPak->OpenPack(iter->c_str(), nullptr, nullptr, false); } // Levels in bundles now take priority over levels outside of bundles. diff --git a/Code/Legacy/CrySystem/SystemCFG.cpp b/Code/Legacy/CrySystem/SystemCFG.cpp index 802e577386..c97345a518 100644 --- a/Code/Legacy/CrySystem/SystemCFG.cpp +++ b/Code/Legacy/CrySystem/SystemCFG.cpp @@ -272,14 +272,12 @@ static bool ParseSystemConfig(const AZStd::string& strSysConfigFilePath, ILoadCo CCryFile file; AZStd::string filenameLog; { - int flags = AZ::IO::IArchive::FOPEN_HINT_QUIET | AZ::IO::IArchive::FOPEN_ONDISK; - if (filename[0] == '@') { // this is used when theres a very specific file to read, like @user@/game.cfg which is read // IN ADDITION to the one in the game folder, and afterwards to override values in it. // if the file is missing and its already prefixed with an alias, there is no need to look any further. - if (!(file.Open(filename.c_str(), "rb", flags))) + if (!(file.Open(filename.c_str(), "rb"))) { if (warnIfMissing) { @@ -293,11 +291,11 @@ static bool ParseSystemConfig(const AZStd::string& strSysConfigFilePath, ILoadCo // otherwise, if the file isn't prefixed with an alias, then its likely one of the convenience mappings // to either root or assets/config. this is done so that code can just request a simple file name and get its data if ( - !(file.Open(filename.c_str(), "rb", flags)) && - !(file.Open((AZStd::string("@root@/") + filename).c_str(), "rb", flags)) && - !(file.Open((AZStd::string("@assets@/") + filename).c_str(), "rb", flags)) && - !(file.Open((AZStd::string("@assets@/config/") + filename).c_str(), "rb", flags)) && - !(file.Open((AZStd::string("@assets@/config/spec/") + filename).c_str(), "rb", flags)) + !(file.Open(filename.c_str(), "rb")) && + !(file.Open((AZStd::string("@root@/") + filename).c_str(), "rb")) && + !(file.Open((AZStd::string("@assets@/") + filename).c_str(), "rb")) && + !(file.Open((AZStd::string("@assets@/config/") + filename).c_str(), "rb")) && + !(file.Open((AZStd::string("@assets@/config/spec/") + filename).c_str(), "rb")) ) { if (warnIfMissing) @@ -308,7 +306,9 @@ static bool ParseSystemConfig(const AZStd::string& strSysConfigFilePath, ILoadCo } } - filenameLog = file.GetAdjustedFilename(); + AZ::IO::FixedMaxPath resolvedFilePath; + AZ::IO::FileIOBase::GetInstance()->ResolvePath(resolvedFilePath, file.GetFilename()); + filenameLog = resolvedFilePath.String(); } INDENT_LOG_DURING_SCOPE(); diff --git a/Code/Legacy/CrySystem/SystemInit.cpp b/Code/Legacy/CrySystem/SystemInit.cpp index 5205b19e19..2c70717e8f 100644 --- a/Code/Legacy/CrySystem/SystemInit.cpp +++ b/Code/Legacy/CrySystem/SystemInit.cpp @@ -588,25 +588,6 @@ bool CSystem::InitFileSystem() m_env.pCryPak->RecordFileOpen(AZ::IO::IArchive::RFOM_EngineStartup); } - //init crypak - if (m_env.pCryPak->Init("")) - { -#if !defined(_RELEASE) - const ICmdLineArg* pakalias = m_pCmdLine->FindArg(eCLAT_Pre, "pakalias"); -#else - const ICmdLineArg* pakalias = nullptr; -#endif // !defined(_RELEASE) - if (pakalias && strlen(pakalias->GetValue()) > 0) - { - m_env.pCryPak->ParseAliases(pakalias->GetValue()); - } - } - else - { - AZ_Assert(false, "Failed to initialize CryPak."); - return false; - } - // Now that file systems are init, we will clear any events that have arrived // during file system init, so that systems do not reload assets that were already compiled in the // critical compilation section. @@ -844,9 +825,6 @@ void CSystem::OpenBasicPaks() #endif //AZ_PLATFORM_ANDROID InlineInitializationProcessing("CSystem::OpenBasicPaks OpenPacks( Engine... )"); - - // Load paks required for game init to mem - gEnv->pCryPak->LoadPakToMemory("Engine.pak", AZ::IO::IArchive::eInMemoryPakLocale_GPU); } ////////////////////////////////////////////////////////////////////////// @@ -891,8 +869,6 @@ void CSystem::OpenLanguageAudioPak([[maybe_unused]] const char* sLanguage) // Initialize languages. - int nPakFlags = 0; - // Omit the trailing slash! AZStd::string sLocalizationFolder(AZStd::string().assign(PathUtil::GetLocalizationFolder(), 0, PathUtil::GetLocalizationFolder().size() - 1)); @@ -904,7 +880,7 @@ void CSystem::OpenLanguageAudioPak([[maybe_unused]] const char* sLanguage) // load localized pak with crc32 filenames on consoles to save memory. AZStd::string sLocalizedPath = "loc.pak"; - if (!m_env.pCryPak->OpenPacks(sLocalizationFolder.c_str(), sLocalizedPath.c_str(), nPakFlags)) + if (!m_env.pCryPak->OpenPacks(sLocalizationFolder.c_str(), sLocalizedPath.c_str())) { // make sure the localized language is found - not really necessary, for TC AZ_Error(AZ_TRACE_SYSTEM_WINDOW, false, "Localized language content(%s) not available or modified from the original installation.", sLanguage); diff --git a/Code/Legacy/CrySystem/WindowsErrorReporting.cpp b/Code/Legacy/CrySystem/WindowsErrorReporting.cpp index d467923c74..cde5d4f8c6 100644 --- a/Code/Legacy/CrySystem/WindowsErrorReporting.cpp +++ b/Code/Legacy/CrySystem/WindowsErrorReporting.cpp @@ -101,8 +101,11 @@ LONG WINAPI CryEngineExceptionFilterWER(struct _EXCEPTION_POINTERS* pExceptionPo { if (g_cvars.sys_WER > 1) { - char szScratch [_MAX_PATH]; - const char* szDumpPath = gEnv->pCryPak->AdjustFileName("@log@/CE2Dump.dmp", szScratch, AZ_ARRAY_SIZE(szScratch), 0); + AZ::IO::FixedMaxPath dumpPath{ "@log@/CE2Dump.dmp" }; + if (auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); fileIoBase != nullptr) + { + dumpPath = fileIoBase->ResolvePath(dumpPath, "@log@/CE2Dump.dmp"); + } MINIDUMP_TYPE mdumpValue = (MINIDUMP_TYPE)(MiniDumpNormal); if (g_cvars.sys_WER > 1) @@ -110,7 +113,7 @@ LONG WINAPI CryEngineExceptionFilterWER(struct _EXCEPTION_POINTERS* pExceptionPo mdumpValue = (MINIDUMP_TYPE)(g_cvars.sys_WER - 2); } - return CryEngineExceptionFilterMiniDump(pExceptionPointers, szDumpPath, mdumpValue); + return CryEngineExceptionFilterMiniDump(pExceptionPointers, dumpPath.c_str(), mdumpValue); } LONG lRet = EXCEPTION_CONTINUE_SEARCH; diff --git a/Code/Legacy/CrySystem/XConsoleVariable.h b/Code/Legacy/CrySystem/XConsoleVariable.h index 3ff71f9f39..d62685c94c 100644 --- a/Code/Legacy/CrySystem/XConsoleVariable.h +++ b/Code/Legacy/CrySystem/XConsoleVariable.h @@ -257,12 +257,12 @@ public: void Set(const char* s) override; void Set(float f) override { - stack_string s = stack_string::format("%g", f); + AZStd::fixed_string<32> s = AZStd::fixed_string<32>::format("%g", f); Set(s.c_str()); } void Set(int i) override { - stack_string s = stack_string::format("%d", i); + AZStd::fixed_string<32> s = AZStd::fixed_string<32>::format("%d", i); Set(s.c_str()); } int GetType() override { return CVAR_STRING; } diff --git a/Code/Legacy/CrySystem/XML/XmlUtils.cpp b/Code/Legacy/CrySystem/XML/XmlUtils.cpp index 111831ed41..3e6a49f266 100644 --- a/Code/Legacy/CrySystem/XML/XmlUtils.cpp +++ b/Code/Legacy/CrySystem/XML/XmlUtils.cpp @@ -169,7 +169,7 @@ public: { if (m_fileHandle != AZ::IO::InvalidHandle) { - gEnv->pCryPak->FWrite(pData, size, 1, m_fileHandle); + gEnv->pCryPak->FWrite(pData, size, m_fileHandle); } } private: diff --git a/Code/Legacy/CrySystem/XML/xml.cpp b/Code/Legacy/CrySystem/XML/xml.cpp index 826e74b758..65a2cb6ffd 100644 --- a/Code/Legacy/CrySystem/XML/xml.cpp +++ b/Code/Legacy/CrySystem/XML/xml.cpp @@ -948,10 +948,12 @@ void CXmlNode::AddToXmlString(XmlString& xml, int level, AZ::IO::HandleType file { if (fileHandle != AZ::IO::InvalidHandle && chunkSize > 0) { + auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); + AZ_Assert(fileIoBase != nullptr, "FileIOBase is expected to be initialized for CXmlNode"); size_t len = xml.length(); if (len >= chunkSize) { - gEnv->pCryPak->FWrite(xml.c_str(), len, 1, fileHandle); + fileIoBase->Write(fileHandle, xml.c_str(), len); xml.assign (""); // should not free memory and does not! } } @@ -1258,7 +1260,8 @@ bool CXmlNode::saveToFile([[maybe_unused]] const char* fileName, size_t chunkSiz XmlString xml; xml.assign (""); xml.reserve(chunkSize * 2); // we reserve double memory, as writing in chunks is not really writing in fixed blocks but a bit fuzzy - auto pCryPak = gEnv->pCryPak; + auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); + AZ_Assert(fileIoBase != nullptr, "FileIOBase is expected to be initialized for CXmlNode"); if (fileHandle == AZ::IO::InvalidHandle) { return false; @@ -1267,7 +1270,7 @@ bool CXmlNode::saveToFile([[maybe_unused]] const char* fileName, size_t chunkSiz size_t len = xml.length(); if (len > 0) { - pCryPak->FWrite(xml.c_str(), len, 1, fileHandle); + fileIoBase->Write(fileHandle, xml.c_str(), len); } xml.clear(); // xml.resize(0) would not reclaim memory return true; @@ -1639,10 +1642,18 @@ XmlNodeRef XmlParserImp::ParseFile(const char* filename, XmlString& errorString, CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "%s", str); return 0; } - adjustedFilename = xmlFile.GetAdjustedFilename(); - AZStd::replace(adjustedFilename.begin(), adjustedFilename.end(), '\\', '/'); - pakPath = xmlFile.GetPakPath(); - AZStd::replace(pakPath.begin(), pakPath.end(), '\\', '/'); + + AZ::IO::FixedMaxPath resolvedPath(AZ::IO::PosixPathSeparator); + auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); + AZ_Assert(fileIoBase != nullptr, "FileIOBase is expected to be initialized for CXmlNode"); + if (fileIoBase->ResolvePath(resolvedPath, xmlFile.GetFilename())) + { + adjustedFilename = resolvedPath.MakePreferred().Native(); + } + if (fileIoBase->ResolvePath(resolvedPath, xmlFile.GetPakPath())) + { + pakPath = resolvedPath.MakePreferred().Native(); + } } XMLBinary::XMLBinaryReader reader; diff --git a/Code/Tools/AssetProcessor/native/InternalBuilders/SettingsRegistryBuilder.cpp b/Code/Tools/AssetProcessor/native/InternalBuilders/SettingsRegistryBuilder.cpp index 7c88c4324c..e6d9894116 100644 --- a/Code/Tools/AssetProcessor/native/InternalBuilders/SettingsRegistryBuilder.cpp +++ b/Code/Tools/AssetProcessor/native/InternalBuilders/SettingsRegistryBuilder.cpp @@ -191,10 +191,51 @@ namespace AssetProcessor response.m_createJobOutputs.push_back(AZStd::move(job)); } - AZ::IO::Path settingsRegistryWildcard = AZ::SettingsRegistryInterface::RegistryFolder; + AZ::IO::Path settingsRegistryWildcard = AZStd::string_view(AZ::Utils::GetEnginePath()); + settingsRegistryWildcard /= AZ::SettingsRegistryInterface::RegistryFolder; settingsRegistryWildcard /= "*.setreg"; response.m_sourceFileDependencyList.emplace_back(AZStd::move(settingsRegistryWildcard.Native()), AZ::Uuid::CreateNull(), AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType::Wildcards); + + auto projectPath = AZ::IO::Path(AZStd::string_view(AZ::Utils::GetProjectPath())); + response.m_sourceFileDependencyList.emplace_back( + AZStd::move((projectPath / AZ::SettingsRegistryInterface::RegistryFolder / "*.setreg").Native()), + AZ::Uuid::CreateNull(), + AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType::Wildcards); + response.m_sourceFileDependencyList.emplace_back( + AZStd::move((projectPath / AZ::SettingsRegistryInterface::DevUserRegistryFolder / "*.setreg").Native()), + AZ::Uuid::CreateNull(), + AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType::Wildcards); + + if (auto settingsRegistry = AZ::Interface::Get(); settingsRegistry != nullptr) + { + AZStd::vector gemInfos; + if (AzFramework::GetGemsInfo(gemInfos, *settingsRegistry)) + { + // Gather unique list of Settings Registry wildcard directories + AZStd::vector gemSettingsRegistryWildcards; + for (const AzFramework::GemInfo& gemInfo : gemInfos) + { + for (const AZ::IO::Path& absoluteSourcePath : gemInfo.m_absoluteSourcePaths) + { + auto gemSettingsRegistryWildcard = absoluteSourcePath / AZ::SettingsRegistryInterface::RegistryFolder / "*.setreg"; + if (auto foundIt = AZStd::find(gemSettingsRegistryWildcards.begin(), gemSettingsRegistryWildcards.end(), gemSettingsRegistryWildcard); + foundIt == gemSettingsRegistryWildcards.end()) + { + gemSettingsRegistryWildcards.emplace_back(gemSettingsRegistryWildcard); + } + } + } + + // Add to the Source File Dependency list + for (AZ::IO::Path& gemSettingsRegistryWildcard : gemSettingsRegistryWildcards) + { + response.m_sourceFileDependencyList.emplace_back( + AZStd::move(gemSettingsRegistryWildcard.Native()), AZ::Uuid::CreateNull(), + AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType::Wildcards); + } + } + } response.m_result = AssetBuilderSDK::CreateJobsResultCode::Success; } diff --git a/Gems/AtomLyIntegration/AtomFont/Code/Include/AtomLyIntegration/AtomFont/FFont.h b/Gems/AtomLyIntegration/AtomFont/Code/Include/AtomLyIntegration/AtomFont/FFont.h index 83f53ffec2..2cc8a67cfa 100644 --- a/Gems/AtomLyIntegration/AtomFont/Code/Include/AtomLyIntegration/AtomFont/FFont.h +++ b/Gems/AtomLyIntegration/AtomFont/Code/Include/AtomLyIntegration/AtomFont/FFont.h @@ -283,7 +283,7 @@ namespace AZ FontTexture* m_fontTexture = nullptr; size_t m_fontBufferSize = 0; - unsigned char* m_fontBuffer = nullptr; + AZStd::unique_ptr m_fontBuffer; AZ::Data::Instance m_fontStreamingImage; AZ::RHI::Ptr m_fontImage; diff --git a/Gems/AtomLyIntegration/AtomFont/Code/Source/FFont.cpp b/Gems/AtomLyIntegration/AtomFont/Code/Source/FFont.cpp index 5e96af0b1b..c5268b427f 100644 --- a/Gems/AtomLyIntegration/AtomFont/Code/Source/FFont.cpp +++ b/Gems/AtomLyIntegration/AtomFont/Code/Source/FFont.cpp @@ -13,8 +13,6 @@ #if !defined(USE_NULLFONT_ALWAYS) -#include - #include #include #include @@ -124,17 +122,10 @@ bool AZ::FFont::Load(const char* fontFilePath, unsigned int width, unsigned int Free(); - auto pPak = gEnv->pCryPak; + auto fileIoBase = AZ::IO::FileIOBase::GetInstance(); - AZStd::string fullFile; - if (pPak->IsAbsPath(fontFilePath)) - { - fullFile = fontFilePath; - } - else - { - fullFile = m_curPath + fontFilePath; - } + AZ::IO::Path fullFile(m_curPath); + fullFile /= fontFilePath; int smoothMethodFlag = (flags & TTFFLAG_SMOOTH_MASK) >> TTFFLAG_SMOOTH_SHIFT; AZ::FontSmoothMethod smoothMethod = AZ::FontSmoothMethod::None; @@ -161,42 +152,41 @@ bool AZ::FFont::Load(const char* fontFilePath, unsigned int width, unsigned int } - AZ::IO::HandleType fileHandle = pPak->FOpen(fullFile.c_str(), "rb"); + AZ::IO::HandleType fileHandle = AZ::IO::InvalidHandle; + fileIoBase->Open(fullFile.c_str(), AZ::IO::GetOpenModeFromStringMode("rb"), fileHandle); if (fileHandle == AZ::IO::InvalidHandle) { return false; } - size_t fileSize = pPak->FGetSize(fileHandle); + AZ::u64 fileSize{}; + fileIoBase->Size(fileHandle, fileSize); if (!fileSize) { - pPak->FClose(fileHandle); + fileIoBase->Close(fileHandle); return false; } - unsigned char* buffer = new unsigned char[fileSize]; - if (!pPak->FReadRaw(buffer, fileSize, 1, fileHandle)) + auto buffer = AZStd::make_unique(fileSize); + if (!fileIoBase->Read(fileHandle, buffer.get(), fileSize)) { - pPak->FClose(fileHandle); - delete [] buffer; + fileIoBase->Close(fileHandle); return false; } - pPak->FClose(fileHandle); + fileIoBase->Close(fileHandle); if (!m_fontTexture) { m_fontTexture = new FontTexture(); } - - if (!m_fontTexture || !m_fontTexture->CreateFromMemory(buffer, (int)fileSize, width, height, smoothMethod, smoothAmount, widthNumSlots, heightNumSlots, sizeRatio)) + if (!m_fontTexture || !m_fontTexture->CreateFromMemory(buffer.get(), (int)fileSize, width, height, smoothMethod, smoothAmount, widthNumSlots, heightNumSlots, sizeRatio)) { - delete [] buffer; return false; } m_monospacedFont = m_fontTexture->GetMonospaced(); - m_fontBuffer = buffer; + m_fontBuffer = AZStd::move(buffer); m_fontBufferSize = fileSize; m_fontTexDirty = false; m_sizeRatio = sizeRatio; @@ -213,10 +203,9 @@ void AZ::FFont::Free() m_fontImageVersion = 0; delete m_fontTexture; - m_fontTexture = 0; + m_fontTexture = nullptr; - delete[] m_fontBuffer; - m_fontBuffer = 0; + m_fontBuffer.reset(); m_fontBufferSize = 0; } diff --git a/Gems/AtomLyIntegration/CommonFeatures/Code/CMakeLists.txt b/Gems/AtomLyIntegration/CommonFeatures/Code/CMakeLists.txt index df5ab134fa..bf28ed3f14 100644 --- a/Gems/AtomLyIntegration/CommonFeatures/Code/CMakeLists.txt +++ b/Gems/AtomLyIntegration/CommonFeatures/Code/CMakeLists.txt @@ -115,6 +115,7 @@ if(PAL_TRAIT_BUILD_HOST_TOOLS) ly_create_alias(NAME AtomLyIntegration_CommonFeatures.Builders NAMESPACE Gem TARGETS Gem::AtomLyIntegration_CommonFeatures.Editor + Gem::Atom_Feature_Common.Builders Gem::Atom_RPI.Builders Gem::GradientSignal.Builders )