diff --git a/Code/CryEngine/CryCommon/Mocks/ICryPakMock.h b/Code/CryEngine/CryCommon/Mocks/ICryPakMock.h index 2d451793bc..072a798985 100644 --- a/Code/CryEngine/CryCommon/Mocks/ICryPakMock.h +++ b/Code/CryEngine/CryCommon/Mocks/ICryPakMock.h @@ -67,7 +67,7 @@ struct CryPakMock MOCK_METHOD1(PoolMalloc, void*(size_t size)); MOCK_METHOD1(PoolFree, void(void* p)); MOCK_METHOD3(PoolAllocMemoryBlock, AZStd::intrusive_ptr (size_t nSize, const char* sUsage, size_t nAlign)); - MOCK_METHOD3(FindFirst, AZ::IO::ArchiveFileIterator(AZStd::string_view pDir, uint32_t nFlags, bool bAllOwUseFileSystem)); + MOCK_METHOD2(FindFirst, AZ::IO::ArchiveFileIterator(AZStd::string_view pDir, AZ::IO::IArchive::EFileSearchType)); MOCK_METHOD1(FindNext, AZ::IO::ArchiveFileIterator(AZ::IO::ArchiveFileIterator handle)); MOCK_METHOD1(FindClose, bool(AZ::IO::ArchiveFileIterator)); MOCK_METHOD1(GetModificationTime, AZ::IO::IArchive::FileTime(AZ::IO::HandleType f)); diff --git a/Code/CryEngine/CrySystem/LevelSystem/LevelSystem.cpp b/Code/CryEngine/CrySystem/LevelSystem/LevelSystem.cpp index c36b9bcee7..08caeeeb70 100644 --- a/Code/CryEngine/CrySystem/LevelSystem/LevelSystem.cpp +++ b/Code/CryEngine/CrySystem/LevelSystem/LevelSystem.cpp @@ -306,8 +306,7 @@ void CLevelSystem::ScanFolder(const char* subfolder, bool modFolder) AZStd::unordered_set pakList; - bool allowFileSystem = true; - AZ::IO::ArchiveFileIterator handle = pPak->FindFirst(search.c_str(), 0, allowFileSystem); + AZ::IO::ArchiveFileIterator handle = pPak->FindFirst(search.c_str(), AZ::IO::IArchive::eFileSearchType_AllowOnDiskOnly); if (handle) { @@ -320,7 +319,7 @@ void CLevelSystem::ScanFolder(const char* subfolder, bool modFolder) { if (AZ::StringFunc::Equal(handle.m_filename.data(), LevelPakName)) { - // level folder contain pak files like 'level.pak' + // level folder contain pak files like 'level.pak' // which we only want to load during level loading. continue; } @@ -351,7 +350,7 @@ void CLevelSystem::ScanFolder(const char* subfolder, bool modFolder) PopulateLevels(search, folder, pPak, modFolder, false); // Load levels outside of the bundles to maintain backward compatibility. PopulateLevels(search, folder, pPak, modFolder, true); - + } void CLevelSystem::PopulateLevels( @@ -360,7 +359,7 @@ void CLevelSystem::PopulateLevels( { // allow this find first to actually touch the file system // (causes small overhead but with minimal amount of levels this should only be around 150ms on actual DVD Emu) - AZ::IO::ArchiveFileIterator handle = pPak->FindFirst(searchPattern.c_str(), 0, fromFileSystemOnly); + AZ::IO::ArchiveFileIterator handle = pPak->FindFirst(searchPattern.c_str(), AZ::IO::IArchive::eFileSearchType_AllowOnDiskOnly); if (handle) { @@ -973,7 +972,7 @@ void CLevelSystem::UnloadLevel() m_lastLevelName.clear(); SAFE_RELEASE(m_pCurrentLevel); - + // Force Lua garbage collection (may no longer be needed now the legacy renderer has been removed). // Normally the GC step is triggered at the end of this method (by the ESYSTEM_EVENT_LEVEL_POST_UNLOAD event). EBUS_EVENT(AZ::ScriptSystemRequestBus, GarbageCollect); diff --git a/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp b/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp index 04573eb2e5..5cb2006447 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp @@ -1290,7 +1290,7 @@ namespace AZ::IO ////////////////////////////////////////////////////////////////////////// - AZ::IO::ArchiveFileIterator Archive::FindFirst(AZStd::string_view pDir, [[maybe_unused]] uint32_t nPathFlags, bool bAllowUseFileSystem) + AZ::IO::ArchiveFileIterator Archive::FindFirst(AZStd::string_view pDir, EFileSearchType searchType) { auto szFullPath = AZ::IO::FileIOBase::GetDirectInstance()->ResolvePath(pDir); if (!szFullPath) @@ -1299,8 +1299,26 @@ namespace AZ::IO return {}; } + bool bScanZips{}; + bool bAllowUseFileSystem{}; + switch (searchType) + { + case IArchive::eFileSearchType_AllowInZipsOnly: + bAllowUseFileSystem = false; + bScanZips = true; + break; + case IArchive::eFileSearchType_AllowOnDiskAndInZips: + bAllowUseFileSystem = true; + bScanZips = true; + break; + case IArchive::eFileSearchType_AllowOnDiskOnly: + bAllowUseFileSystem = true; + bScanZips = false; + break; + } + AZStd::intrusive_ptr pFindData = new AZ::IO::FindData(); - pFindData->Scan(this, szFullPath->Native(), bAllowUseFileSystem); + pFindData->Scan(this, szFullPath->Native(), bAllowUseFileSystem, bScanZips); return pFindData->Fetch(); } @@ -1676,7 +1694,7 @@ namespace AZ::IO return true; } - if (AZ::IO::ArchiveFileIterator fileIterator = FindFirst(pWildcardIn, 0, true); fileIterator) + if (AZ::IO::ArchiveFileIterator fileIterator = FindFirst(pWildcardIn, IArchive::eFileSearchType_AllowOnDiskOnly); fileIterator) { AZStd::vector files; do diff --git a/Code/Framework/AzFramework/AzFramework/Archive/Archive.h b/Code/Framework/AzFramework/AzFramework/Archive/Archive.h index 997b3e3d2c..329beb4291 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/Archive.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/Archive.h @@ -234,7 +234,7 @@ namespace AZ::IO uint64_t FTell(AZ::IO::HandleType handle) override; int FFlush(AZ::IO::HandleType handle) override; int FClose(AZ::IO::HandleType handle) override; - AZ::IO::ArchiveFileIterator FindFirst(AZStd::string_view pDir, uint32_t nPathFlags = 0, bool bAllOwUseFileSystem = false) override; + AZ::IO::ArchiveFileIterator FindFirst(AZStd::string_view pDir, EFileSearchType searchType = eFileSearchType_AllowInZipsOnly) override; AZ::IO::ArchiveFileIterator FindNext(AZ::IO::ArchiveFileIterator fileIterator) override; bool FindClose(AZ::IO::ArchiveFileIterator fileIterator) override; int FEof(AZ::IO::HandleType handle) override; diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.cpp index 1794ae90e7..05da5f16eb 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.cpp @@ -77,7 +77,7 @@ namespace AZ::IO return m_findData && m_lastFetchValid; } - void FindData::Scan(IArchive* archive, AZStd::string_view szDir, bool bAllowUseFS) + void FindData::Scan(IArchive* archive, AZStd::string_view szDir, bool bAllowUseFS, bool bScanZips) { // get the priority into local variable to avoid it changing in the course of // this function execution @@ -87,12 +87,18 @@ namespace AZ::IO { // first, find the file system files ScanFS(archive, szDir); - ScanZips(archive, szDir); + if (bScanZips) + { + ScanZips(archive, szDir); + } } else { // first, find the zip files - ScanZips(archive, szDir); + if (bScanZips) + { + ScanZips(archive, szDir); + } if (bAllowUseFS || nVarPakPriority != ArchiveLocationPriority::ePakPriorityPakOnly) { ScanFS(archive, szDir); @@ -111,30 +117,31 @@ namespace AZ::IO } AZ::IO::FileIOBase::GetDirectInstance()->FindFiles(searchDirectory.c_str(), pattern.c_str(), [&](const char* filePath) -> bool { - AZ::IO::FileDesc fileDesc; - AZStd::string filePathEntry{filePath}; + AZ::IO::ArchiveFileIterator fileIterator; + fileIterator.m_filename = AZ::IO::PathView(filePath).Filename().Native(); + fileIterator.m_fileDesc.nAttrib = {}; if (AZ::IO::FileIOBase::GetDirectInstance()->IsDirectory(filePath)) { - fileDesc.nAttrib = fileDesc.nAttrib | AZ::IO::FileDesc::Attribute::Subdirectory; + fileIterator.m_fileDesc.nAttrib = fileIterator.m_fileDesc.nAttrib | AZ::IO::FileDesc::Attribute::Subdirectory; + m_fileStack.emplace_back(AZStd::move(fileIterator)); } else { if (AZ::IO::FileIOBase::GetDirectInstance()->IsReadOnly(filePath)) { - fileDesc.nAttrib = fileDesc.nAttrib | AZ::IO::FileDesc::Attribute::ReadOnly; + fileIterator.m_fileDesc.nAttrib = fileIterator.m_fileDesc.nAttrib | AZ::IO::FileDesc::Attribute::ReadOnly; } AZ::u64 fileSize = 0; AZ::IO::FileIOBase::GetDirectInstance()->Size(filePath, fileSize); - fileDesc.nSize = fileSize; - fileDesc.tWrite = AZ::IO::FileIOBase::GetDirectInstance()->ModificationTime(filePath); + fileIterator.m_fileDesc.nSize = fileSize; + fileIterator.m_fileDesc.tWrite = AZ::IO::FileIOBase::GetDirectInstance()->ModificationTime(filePath); // These times are not supported by our file interface - fileDesc.tAccess = fileDesc.tWrite; - fileDesc.tCreate = fileDesc.tWrite; + fileIterator.m_fileDesc.tAccess = fileIterator.m_fileDesc.tWrite; + fileIterator.m_fileDesc.tCreate = fileIterator.m_fileDesc.tWrite; + m_fileStack.emplace_back(AZStd::move(fileIterator)); } - [[maybe_unused]] auto result = m_mapFiles.emplace(AZStd::move(filePathEntry), fileDesc); - AZ_Assert(result.second, "Failed to insert FindData entry for filePath %s", filePath); return true; }); } @@ -164,7 +171,7 @@ namespace AZ::IO fileDesc.nAttrib = AZ::IO::FileDesc::Attribute::ReadOnly | AZ::IO::FileDesc::Attribute::Archive; fileDesc.nSize = fileEntry->desc.lSizeUncompressed; fileDesc.tWrite = fileEntry->GetModificationTime(); - m_mapFiles.emplace(fname, fileDesc); + m_fileStack.emplace_back(AZ::IO::ArchiveFileIterator{ this, fname, fileDesc }); } ZipDir::FindDir findDirectoryEntry(zipCache); @@ -177,7 +184,7 @@ namespace AZ::IO } AZ::IO::FileDesc fileDesc; fileDesc.nAttrib = AZ::IO::FileDesc::Attribute::ReadOnly | AZ::IO::FileDesc::Attribute::Archive | AZ::IO::FileDesc::Attribute::Subdirectory; - m_mapFiles.emplace(fname, fileDesc); + m_fileStack.emplace_back(AZ::IO::ArchiveFileIterator{ this, fname, fileDesc }); } }; @@ -246,7 +253,7 @@ namespace AZ::IO if (!bindRootIter->empty() && AZStd::wildcard_match(sourcePathRemainder.Native(), bindRootIter->Native())) { AZ::IO::FileDesc fileDesc{ AZ::IO::FileDesc::Attribute::ReadOnly | AZ::IO::FileDesc::Attribute::Archive | AZ::IO::FileDesc::Attribute::Subdirectory }; - m_mapFiles.emplace(bindRootIter->Native(), fileDesc); + m_fileStack.emplace_back(AZ::IO::ArchiveFileIterator{ this, bindRootIter->Native(), fileDesc }); } } else @@ -262,22 +269,19 @@ namespace AZ::IO AZ::IO::ArchiveFileIterator FindData::Fetch() { - AZ::IO::ArchiveFileIterator fileIterator; - fileIterator.m_findData = this; - if (m_mapFiles.empty()) + if (m_fileStack.empty()) { - return fileIterator; + AZ::IO::ArchiveFileIterator emptyFileIterator; + emptyFileIterator.m_lastFetchValid = false; + emptyFileIterator.m_findData = this; + return emptyFileIterator; } - auto pakFileIter = m_mapFiles.begin(); - AZStd::string fullFilePath; - AZ::StringFunc::Path::GetFullFileName(pakFileIter->first.c_str(), fullFilePath); - fileIterator.m_filename = AZStd::move(fullFilePath); - fileIterator.m_fileDesc = pakFileIter->second; - fileIterator.m_lastFetchValid = true; - // Remove Fetched item from the FindData map so that the iteration continues - m_mapFiles.erase(pakFileIter); + AZ::IO::ArchiveFileIterator fileIterator{ m_fileStack.back() }; + fileIterator.m_lastFetchValid = true; + fileIterator.m_findData = this; + m_fileStack.pop_back(); return fileIterator; } } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.h b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.h index d5e23779fc..a07e98c81f 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/ArchiveFindData.h @@ -15,7 +15,6 @@ #include #include - namespace AZ::IO { struct IArchive; @@ -74,13 +73,13 @@ namespace AZ::IO AZ_CLASS_ALLOCATOR(FindData, AZ::SystemAllocator, 0); FindData() = default; AZ::IO::ArchiveFileIterator Fetch(); - void Scan(IArchive* archive, AZStd::string_view path, bool bAllowUseFS = false); + void Scan(IArchive* archive, AZStd::string_view path, bool bAllowUseFS = false, bool bScanZips = true); protected: void ScanFS(IArchive* archive, AZStd::string_view path); void ScanZips(IArchive* archive, AZStd::string_view path); - using FileMap = AZStd::map; - FileMap m_mapFiles; + using FileStack = AZStd::vector; + FileStack m_fileStack; }; } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/IArchive.h b/Code/Framework/AzFramework/AzFramework/Archive/IArchive.h index ce8403b033..08bd87c334 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/IArchive.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/IArchive.h @@ -197,6 +197,13 @@ namespace AZ::IO eInMemoryPakLocale_PAK, }; + enum EFileSearchType + { + eFileSearchType_AllowInZipsOnly = 0, + eFileSearchType_AllowOnDiskAndInZips, + eFileSearchType_AllowOnDiskOnly + }; + using SignedFileSize = int64_t; virtual ~IArchive() = default; @@ -315,7 +322,7 @@ namespace AZ::IO // Arguments: // nFlags is a combination of EPathResolutionRules flags. - virtual ArchiveFileIterator FindFirst(AZStd::string_view pDir, uint32_t nFlags = 0, bool bAllowUseFileSystem = false) = 0; + virtual ArchiveFileIterator FindFirst(AZStd::string_view pDir, EFileSearchType searchType = eFileSearchType_AllowInZipsOnly) = 0; virtual ArchiveFileIterator FindNext(AZ::IO::ArchiveFileIterator handle) = 0; virtual bool FindClose(AZ::IO::ArchiveFileIterator handle) = 0; //returns file modification time diff --git a/Code/Sandbox/Editor/CryEditDoc.cpp b/Code/Sandbox/Editor/CryEditDoc.cpp index 5b2a4b9a83..77d5794ab3 100644 --- a/Code/Sandbox/Editor/CryEditDoc.cpp +++ b/Code/Sandbox/Editor/CryEditDoc.cpp @@ -1139,7 +1139,7 @@ bool CCryEditDoc::SaveLevel(const QString& filename) const QString oldLevelPattern = QDir(oldLevelFolder).absoluteFilePath("*.*"); const QString oldLevelName = Path::GetFile(GetLevelPathName()); const QString oldLevelXml = Path::ReplaceExtension(oldLevelName, "xml"); - AZ::IO::ArchiveFileIterator findHandle = pIPak->FindFirst(oldLevelPattern.toUtf8().data(), 0, true); + AZ::IO::ArchiveFileIterator findHandle = pIPak->FindFirst(oldLevelPattern.toUtf8().data(), AZ::IO::IArchive::eFileSearchType_AllowOnDiskAndInZips); if (findHandle) { do diff --git a/Gems/AudioSystem/Code/Tests/AudioSystemEditorTest.cpp b/Gems/AudioSystem/Code/Tests/AudioSystemEditorTest.cpp index e086246e29..d7f9b31e25 100644 --- a/Gems/AudioSystem/Code/Tests/AudioSystemEditorTest.cpp +++ b/Gems/AudioSystem/Code/Tests/AudioSystemEditorTest.cpp @@ -36,14 +36,14 @@ namespace CustomMocks : m_levelName(levelName) {} - AZ::IO::ArchiveFileIterator FindFirst([[maybe_unused]] AZStd::string_view dir, [[maybe_unused]] unsigned int flags, [[maybe_unused]] bool allowUseFileSystem) override + AZ::IO::ArchiveFileIterator FindFirst([[maybe_unused]] AZStd::string_view dir, AZ::IO::IArchive::EFileSearchType) override { AZ::IO::FileDesc fileDesc; fileDesc.nSize = sizeof(AZ::IO::FileDesc); // Add a filename and file description reference to the TestFindData map to make sure the file iterator is valid - AZStd::intrusive_ptr findData = new TestFindData{}; - findData->m_mapFiles.emplace(m_levelName, fileDesc); - return findData->Fetch(); + m_findData = new TestFindData(); + m_findData->m_fileStack.emplace_back(AZ::IO::ArchiveFileIterator{ static_cast(m_findData.get()), m_levelName, fileDesc }); + return m_findData->Fetch(); } AZ::IO::ArchiveFileIterator FindNext(AZ::IO::ArchiveFileIterator iter) override @@ -54,13 +54,14 @@ namespace CustomMocks // public: for easy resetting... AZStd::string m_levelName; - // Add an inherited FindData class to control the adding of a mapfile which indicates that a FileIterator is valid struct TestFindData : AZ::IO::FindData { - using AZ::IO::FindData::m_mapFiles; + using AZ::IO::FindData::m_fileStack; }; + + AZStd::intrusive_ptr m_findData; }; } // namespace CustomMocks