birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for configuring default Archive File Search Mode through a Cache Var (#5668)

Browse Source 
* Renamed ArchiveLocationPriority enum to FileSearchPriority and made it a proper enum class

Added an ArchiveVars.cpp which checks the a new define: `LY_ARCHIVE_FILE_SEARCH_MODE_DEFAULT`
That define represents the default value to use for the Archive system search mode

Moved the FileSearchLocation enum to the ArchiveVars.h header

Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com>

* Updated the AssetBundleComponent to use AZ::IO::Path for level dirs

Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com>

* Added a LY_ARCHIVE_FILE_SEARCH_MODE cache variable

The Cache Variable default value is to Archive File Search Mode to PakOnly
in Release. This can be overridden using a value for all configurations
by specifying a number of 0, 1 or 2.
Alternatively a generator expression can be used to set the Archive File
Search Mode in specific configurations.
For example to set the FileSearchMode to 1 in profile and 2 in release
the following LY_ARCHIVE_FILE_SEARCH_MODE value can be used
`$<$<CONFIG:profile>:1>$<$<CONFIG:release>:2>`

Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com>

* Updated AssetBundler(Batch) VS Debugger arguments to populate the
project-path optoin if a single project is configured.

Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com>

* Add support for serializing older versions of the AssetBundleManifest

This is done by attaching the "ObjectStreamWriteElementOverride"
attribute to the AssetBundleManifest reflection.
That attribute contains a function which outputs an older serialized
version of the AssetBundleManifest based on the `m_bundleVersion` member
value.

Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com>

* AZStd::variant Serialization fix

The AttributeData<T> type is no longer suitable for storing the
ObjectStreamWriterOverrideCB function

Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com>
This commit is contained in:
lumberyard-employee-dm
2021-11-18 10:22:08 -06:00
committed by GitHub
parent 489877b82a
commit 9d22c98c26
31 changed files with 396 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Legacy/CryCommon/Mocks/ICryPakMock.h
+4 -3
View File
@@ -11,6 +11,7 @@
#include <AzTest/AzTest.h>
#include <AzCore/IO/FileIO.h>
#include <AzCore/IO/Path/Path.h>
#include <AzFramework/Archive/ArchiveVars.h>
#include <AzFramework/Archive/INestedArchive.h>
#include <AzFramework/Archive/IArchive.h>
#include <CryCommon/platform.h>
@@ -52,11 +53,11 @@ struct CryPakMock
MOCK_METHOD1(PoolMalloc, void*(size_t size));
MOCK_METHOD1(PoolFree, void(void* p));
MOCK_METHOD3(PoolAllocMemoryBlock, AZStd::intrusive_ptr<AZ::IO::MemoryBlock> (size_t nSize, const char* sUsage, size_t nAlign));
MOCK_METHOD2(FindFirst, AZ::IO::ArchiveFileIterator(AZStd::string_view pDir, AZ::IO::IArchive::EFileSearchType));
MOCK_METHOD2(FindFirst, AZ::IO::ArchiveFileIterator(AZStd::string_view pDir, AZ::IO::FileSearchLocation));
MOCK_METHOD1(FindNext, AZ::IO::ArchiveFileIterator(AZ::IO::ArchiveFileIterator handle));
MOCK_METHOD1(FindClose, bool(AZ::IO::ArchiveFileIterator));
MOCK_METHOD1(GetModificationTime, AZ::IO::IArchive::FileTime(AZ::IO::HandleType f));
MOCK_METHOD2(IsFileExist, bool(AZStd::string_view sFilename, EFileSearchLocation));
MOCK_METHOD2(IsFileExist, bool(AZStd::string_view sFilename, AZ::IO::FileSearchLocation));
MOCK_METHOD1(IsFolder, bool(AZStd::string_view sPath));
MOCK_METHOD1(GetFileSizeOnDisk, AZ::IO::IArchive::SignedFileSize(AZStd::string_view filename));
MOCK_METHOD4(OpenArchive, AZStd::intrusive_ptr<AZ::IO::INestedArchive> (AZStd::string_view szPath, AZStd::string_view bindRoot, uint32_t nFlags, AZStd::intrusive_ptr<AZ::IO::MemoryBlock> pData));
@@ -72,7 +73,7 @@ struct CryPakMock
MOCK_METHOD1(UnregisterFileAccessSink, void(AZ::IO::IArchiveFileAccessSink * pSink));
MOCK_METHOD1(DisableRuntimeFileAccess, void(bool status));
MOCK_METHOD2(DisableRuntimeFileAccess, bool(bool status, AZStd::thread_id threadId));
MOCK_CONST_METHOD0(GetPakPriority, AZ::IO::ArchiveLocationPriority());
MOCK_CONST_METHOD0(GetPakPriority, AZ::IO::FileSearchPriority());
MOCK_CONST_METHOD1(GetFileOffsetOnMedia, uint64_t(AZStd::string_view szName));
MOCK_CONST_METHOD1(GetFileMediaType, EStreamSourceMediaType(AZStd::string_view szName));
MOCK_METHOD0(GetLevelPackOpenEvent, auto()->LevelPackOpenEvent*);
Code/Legacy/CrySystem/LevelSystem/LevelSystem.cpp
+77 -80
View File
@@ -203,24 +203,22 @@ CLevelSystem::CLevelSystem(ISystem* pSystem, const char* levelsFolder)
{
return;
}
auto pPak = gEnv->pCryPak;
auto archive = AZ::Interface<AZ::IO::IArchive>::Get();
if (AZ::IO::IArchive::LevelPackOpenEvent* levelPakOpenEvent = pPak->GetLevelPackOpenEvent())
if (AZ::IO::IArchive::LevelPackOpenEvent* levelPakOpenEvent = archive->GetLevelPackOpenEvent())
{
m_levelPackOpenHandler = AZ::IO::IArchive::LevelPackOpenEvent::Handler([this](const AZStd::vector<AZStd::string>& levelDirs)
m_levelPackOpenHandler = AZ::IO::IArchive::LevelPackOpenEvent::Handler([this](const AZStd::vector<AZ::IO::Path>& levelDirs)
{
for (AZStd::string dir : levelDirs)
for (AZ::IO::Path levelDir : levelDirs)
{
AZ::StringFunc::Path::StripComponent(dir, true);
AZStd::string searchPattern = dir + AZ_FILESYSTEM_SEPARATOR_WILDCARD;
bool modFolder = false;
PopulateLevels(searchPattern, dir, gEnv->pCryPak, modFolder, false);
PopulateLevels((levelDir / "*").Native(), levelDir.Native(), AZ::Interface<AZ::IO::IArchive>::Get(), modFolder, false);
}
});
m_levelPackOpenHandler.Connect(*levelPakOpenEvent);
}
if (AZ::IO::IArchive::LevelPackCloseEvent* levelPakCloseEvent = pPak->GetLevelPackCloseEvent())
if (AZ::IO::IArchive::LevelPackCloseEvent* levelPakCloseEvent = archive->GetLevelPackCloseEvent())
{
m_levelPackCloseHandler = AZ::IO::IArchive::LevelPackCloseEvent::Handler([this](AZStd::string_view)
{
@@ -288,7 +286,7 @@ void CLevelSystem::ScanFolder(const char* subfolder, bool modFolder)
AZStd::unordered_set<AZStd::string> pakList;
AZ::IO::ArchiveFileIterator handle = pPak->FindFirst(search.c_str(), AZ::IO::IArchive::eFileSearchType_AllowOnDiskOnly);
AZ::IO::ArchiveFileIterator handle = pPak->FindFirst(search.c_str(), AZ::IO::FileSearchLocation::OnDisk);
if (handle)
{
@@ -335,86 +333,85 @@ void CLevelSystem::ScanFolder(const char* subfolder, bool modFolder)
}
void CLevelSystem::PopulateLevels(
AZStd::string searchPattern, AZStd::string& folder, AZ::IO::IArchive* pPak, bool& modFolder, bool fromFileSystemOnly)
AZStd::string searchPattern, const AZStd::string& folder, AZ::IO::IArchive* pPak, bool& modFolder, bool fromFileSystemOnly)
{
// allow this find first to actually touch the file system
// (causes small overhead but with minimal amount of levels this should only be around 150ms on actual DVD Emu)
AZ::IO::ArchiveFileIterator handle = pPak->FindFirst(searchPattern.c_str(),
fromFileSystemOnly ? AZ::IO::FileSearchLocation::OnDisk : AZ::IO::FileSearchLocation::InPak);
if (handle)
{
// allow this find first to actually touch the file system
// (causes small overhead but with minimal amount of levels this should only be around 150ms on actual DVD Emu)
AZ::IO::ArchiveFileIterator handle = pPak->FindFirst(searchPattern.c_str(), AZ::IO::IArchive::eFileSearchType_AllowOnDiskOnly);
if (handle)
do
{
do
if ((handle.m_fileDesc.nAttrib & AZ::IO::FileDesc::Attribute::Subdirectory) != AZ::IO::FileDesc::Attribute::Subdirectory ||
handle.m_filename == "." || handle.m_filename == "..")
{
if ((handle.m_fileDesc.nAttrib & AZ::IO::FileDesc::Attribute::Subdirectory) != AZ::IO::FileDesc::Attribute::Subdirectory ||
handle.m_filename == "." || handle.m_filename == "..")
{
continue;
}
continue;
}
AZStd::string levelFolder;
if (fromFileSystemOnly)
{
levelFolder =
(folder.empty() ? "" : (folder + "/")) + AZStd::string(handle.m_filename.data(), handle.m_filename.size());
}
else
{
AZStd::string levelName(AZ::IO::PathView(handle.m_filename).Filename().Native());
levelFolder = (folder.empty() ? "" : (folder + "/")) + levelName;
}
AZStd::string levelFolder;
if (fromFileSystemOnly)
{
levelFolder =
(folder.empty() ? "" : (folder + "/")) + AZStd::string(handle.m_filename.data(), handle.m_filename.size());
}
else
{
AZStd::string levelName(AZ::IO::PathView(handle.m_filename).Filename().Native());
levelFolder = (folder.empty() ? "" : (folder + "/")) + levelName;
}
AZStd::string levelPath;
if (AZ::StringFunc::StartsWith(levelFolder.c_str(), m_levelsFolder.c_str()))
AZStd::string levelPath;
if (AZ::StringFunc::StartsWith(levelFolder.c_str(), m_levelsFolder.c_str()))
{
levelPath = levelFolder;
}
else
{
levelPath = m_levelsFolder + "/" + levelFolder;
}
const AZStd::string levelPakName = levelPath + "/" + LevelPakName;
const AZStd::string levelInfoName = levelPath + "/levelinfo.xml";
if (!pPak->IsFileExist(
levelPakName.c_str(),
fromFileSystemOnly ? AZ::IO::FileSearchLocation::OnDisk : AZ::IO::FileSearchLocation::InPak) &&
!pPak->IsFileExist(
levelInfoName.c_str(),
fromFileSystemOnly ? AZ::IO::FileSearchLocation::OnDisk : AZ::IO::FileSearchLocation::InPak))
{
ScanFolder(levelFolder.c_str(), modFolder);
continue;
}
// With the level.pak workflow, levelPath and levelName will point to a directory.
// levelPath: levels/mylevel
// levelName: mylevel
CLevelInfo levelInfo;
levelInfo.m_levelPath = levelPath;
levelInfo.m_levelName = levelFolder;
levelInfo.m_isPak = !fromFileSystemOnly;
CLevelInfo* pExistingInfo = GetLevelInfoInternal(levelInfo.m_levelName);
// Don't add the level if it is already in the list
if (pExistingInfo == NULL)
{
m_levelInfos.push_back(levelInfo);
}
else
{
// Levels in bundles take priority over levels outside bundles.
if (!pExistingInfo->m_isPak && levelInfo.m_isPak)
{
levelPath = levelFolder;
}
else
{
levelPath = m_levelsFolder + "/" + levelFolder;
*pExistingInfo = levelInfo;
}
}
} while (handle = pPak->FindNext(handle));
const AZStd::string levelPakName = levelPath + "/" + LevelPakName;
const AZStd::string levelInfoName = levelPath + "/levelinfo.xml";
if (!pPak->IsFileExist(
levelPakName.c_str(),
fromFileSystemOnly ? AZ::IO::IArchive::eFileLocation_OnDisk : AZ::IO::IArchive::eFileLocation_InPak) &&
!pPak->IsFileExist(
levelInfoName.c_str(),
fromFileSystemOnly ? AZ::IO::IArchive::eFileLocation_OnDisk : AZ::IO::IArchive::eFileLocation_InPak))
{
ScanFolder(levelFolder.c_str(), modFolder);
continue;
}
// With the level.pak workflow, levelPath and levelName will point to a directory.
// levelPath: levels/mylevel
// levelName: mylevel
CLevelInfo levelInfo;
levelInfo.m_levelPath = levelPath;
levelInfo.m_levelName = levelFolder;
levelInfo.m_isPak = !fromFileSystemOnly;
CLevelInfo* pExistingInfo = GetLevelInfoInternal(levelInfo.m_levelName);
// Don't add the level if it is already in the list
if (pExistingInfo == NULL)
{
m_levelInfos.push_back(levelInfo);
}
else
{
// Levels in bundles take priority over levels outside bundles.
if (!pExistingInfo->m_isPak && levelInfo.m_isPak)
{
*pExistingInfo = levelInfo;
}
}
} while (handle = pPak->FindNext(handle));
pPak->FindClose(handle);
}
pPak->FindClose(handle);
}
}
Code/Legacy/CrySystem/LevelSystem/LevelSystem.h
+1 -1
View File
@@ -116,7 +116,7 @@ private:
void ScanFolder(const char* subfolder, bool modFolder);
void PopulateLevels(
AZStd::string searchPattern, AZStd::string& folder, AZ::IO::IArchive* pPak, bool& modFolder, bool fromFileSystemOnly);
AZStd::string searchPattern, const AZStd::string& folder, AZ::IO::IArchive* pPak, bool& modFolder, bool fromFileSystemOnly);
void PrepareNextLevel(const char* levelName);
ILevel* LoadLevelInternal(const char* _levelName);
Code/Legacy/CrySystem/SystemInit.cpp
-38
View File
@@ -170,14 +170,6 @@ void CryEngineSignalHandler(int signal)
extern HMODULE gDLLHandle;
#endif
namespace
{
#if defined(AZ_PLATFORM_WINDOWS)
// on windows, we lock our cache using a lockfile. On other platforms this is not necessary since devices like ios, android, consoles cannot
// run more than one game process that uses the same folder anyway.
HANDLE g_cacheLock = INVALID_HANDLE_VALUE;
#endif
}
//static int g_sysSpecChanged = false;
@@ -339,9 +331,6 @@ bool CSystem::InitFileSystem()
m_pUserCallback->OnInitProgress("Initializing File System...");
}
// get the DirectInstance FileIOBase which should be the AZ::LocalFileIO
m_env.pFileIO = AZ::IO::FileIOBase::GetDirectInstance();
m_env.pCryPak = AZ::Interface<AZ::IO::IArchive>::Get();
m_env.pFileIO = AZ::IO::FileIOBase::GetInstance();
AZ_Assert(m_env.pCryPak, "CryPak has not been initialized on AZ::Interface");
@@ -365,33 +354,6 @@ bool CSystem::InitFileSystem()
void CSystem::ShutdownFileSystem()
{
#if defined(AZ_PLATFORM_WINDOWS)
if (g_cacheLock != INVALID_HANDLE_VALUE)
{
CloseHandle(g_cacheLock);
g_cacheLock = INVALID_HANDLE_VALUE;
}
#endif
using namespace AZ::IO;
FileIOBase* directInstance = FileIOBase::GetDirectInstance();
FileIOBase* pakInstance = FileIOBase::GetInstance();
if (directInstance == m_env.pFileIO)
{
// we only mess with file io if we own the instance that we installed.
// if we dont' own the instance, then we never configured fileIO and we should not alter it.
delete directInstance;
FileIOBase::SetDirectInstance(nullptr);
if (pakInstance != directInstance)
{
delete pakInstance;
FileIOBase::SetInstance(nullptr);
}
}
m_env.pFileIO = nullptr;
}