birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Red code legacy MemoryManager, StreamEngine, ResourceManager, ImageHandler, AsyncPakManager, and more (#758)

Browse Source 
Remove from CryCommon and CrySystem:
- MemoryManager and all related classes/files
- StreamEngine, ResourceManager, ImageHandler, and AsyncPakManager
- Various other related interfaces/files/classes etc. that are all now unused
This commit is contained in:
bosnichd
2021-05-14 08:53:56 -06:00
committed by GitHub
parent d0daab4fe0
commit a9d986c10f
116 changed files with 37 additions and 19717 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/CryEngine/CrySystem/AsyncPakManager.cpp
-516
View File
@@ -1,516 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Manage async pak files
#include "CrySystem_precompiled.h"
#include "AsyncPakManager.h"
#include "System.h"
#include "IStreamEngine.h"
#include <AzFramework/Archive/Archive.h>
#include <AzFramework/API/ApplicationAPI.h>
#include "ResourceManager.h"
#define MEGA_BYTE 1024* 1024
//////////////////////////////////////////////////////////////////////////
string& CAsyncPakManager::SAsyncPak::GetStatus(string& status) const
{
switch (eState)
{
case STATE_UNLOADED:
status = "Unloaded";
break;
case STATE_REQUESTED:
status = "Requested";
break;
case STATE_REQUESTUNLOAD:
status = "RequestUnload";
break;
case STATE_LOADED:
status = "Loaded";
break;
default:
status = "Unknown";
break;
}
return status;
}
//////////////////////////////////////////////////////////////////////////
CAsyncPakManager::CAsyncPakManager()
{
m_nTotalOpenLayerPakSize = 0;
m_bRequestLayerUpdate = false;
}
CAsyncPakManager::~CAsyncPakManager()
{
Clear();
}
//////////////////////////////////////////////////////////////////////////
void CAsyncPakManager::Clear()
{
//float startTime = gEnv->pTimer->GetAsyncCurTime();
for (TPakMap::iterator it = m_paks.begin();
it != m_paks.end(); ++it)
{
SAsyncPak& layerPak = it->second;
if (layerPak.bStreaming)
{
// wait until finished
layerPak.pReadStream->Abort();
}
ReleaseData(&layerPak);
}
m_paks.clear();
m_bRequestLayerUpdate = false;
assert(m_nTotalOpenLayerPakSize == 0);
m_nTotalOpenLayerPakSize = 0;
//printf("CAsyncPakManager::Clear() %0.4f secs\n", gEnv->pTimer->GetAsyncCurTime() - startTime);
}
void CAsyncPakManager::UnloadLevelLoadPaks()
{
for (TPakMap::iterator it = m_paks.begin();
it != m_paks.end(); ++it)
{
SAsyncPak& layerPak = it->second;
if (layerPak.eLifeTime == SAsyncPak::LIFETIME_LOAD_ONLY)
{
if (layerPak.bStreaming)
{
// wait until finished
layerPak.pReadStream->Abort();
}
ReleaseData(&layerPak);
}
}
}
//////////////////////////////////////////////////////////////////////////
void CAsyncPakManager::ParseLayerPaks(const string& levelCachePath)
{
string layerPath = levelCachePath + "/"; // "/layers/";
string search = layerPath + "*";
auto pPak = gEnv->pCryPak;
// allow this find first to actually touch the file system
AZ::IO::ArchiveFileIterator fileIterator= pPak->FindFirst(search.c_str(), 0, true);
if (fileIterator)
{
do
{
if ((fileIterator.m_fileDesc.nAttrib & AZ::IO::FileDesc::Attribute::Subdirectory) == AZ::IO::FileDesc::Attribute::Subdirectory || fileIterator.m_filename == "." || fileIterator.m_filename == "..")
{
continue;
}
string pakName(fileIterator.m_filename.data(), fileIterator.m_filename.size());
size_t findPos = pakName.find_last_of('.');
if (findPos == string::npos)
{
continue;
}
string extension = pakName.substr(findPos + 1, pakName.size());
if (extension != "pak")
{
continue;
}
SAsyncPak layerPak;
layerPak.layername = pakName.substr(0, findPos);
layerPak.filename = layerPath + pakName;
layerPak.nSize = pPak->FGetSize(layerPak.filename.c_str(), true); // allow to go to disc for this access
layerPak.bClosePakOnRelease = true;
m_paks[layerPak.layername] = layerPak;
} while (fileIterator = pPak->FindNext(fileIterator));
pPak->FindClose(fileIterator);
}
}
//////////////////////////////////////////////////////////////////////////
void CAsyncPakManager::StartStreaming(SAsyncPak* pLayerPak)
{
StreamReadParams params;
params.dwUserData = (DWORD_PTR) pLayerPak;
params.nSize = 0;
params.pBuffer = NULL;
params.nFlags = IStreamEngine::FLAGS_FILE_ON_DISK;
params.ePriority = estpIdle;
pLayerPak->pReadStream = gEnv->pSystem->GetStreamEngine()->StartRead(eStreamTaskTypePak, pLayerPak->filename.c_str(), this, &params);
if (pLayerPak->pReadStream)
{
pLayerPak->bStreaming = true;
}
else
{
pLayerPak->eState = SAsyncPak::STATE_UNLOADED;
pLayerPak->pData.reset();
}
}
void CAsyncPakManager::ReleaseData(SAsyncPak* pLayerPak)
{
if (pLayerPak->eState == SAsyncPak::STATE_LOADED)
{
if (pLayerPak->bClosePakOnRelease)
{
gEnv->pCryPak->ClosePack(pLayerPak->filename.c_str(), 0);
//printf("Unload pak from mem: %s\n", pLayerPak->filename.c_str());
}
else
{
gEnv->pCryPak->LoadPakToMemory(pLayerPak->filename.c_str(), AZ::IO::IArchive::eInMemoryPakLocale_Unload);
//printf("Close pak: %s\n", pLayerPak->filename.c_str());
}
m_nTotalOpenLayerPakSize -= pLayerPak->nSize;
}
if (pLayerPak->pData)
{
assert(pLayerPak->pData->use_count() == 1);
}
assert((!pLayerPak->pData) || (pLayerPak->pData && pLayerPak->pData->use_count() == 1));
pLayerPak->pData.reset();
pLayerPak->eState = SAsyncPak::STATE_UNLOADED;
m_bRequestLayerUpdate = true;
}
//////////////////////////////////////////////////////////////////////////
bool CAsyncPakManager::LoadLayerPak(const char* sLayerName)
{
// only load layer paks from valid files
TPakMap::iterator findResult = m_paks.find(sLayerName);
if (findResult != m_paks.end())
{
return LoadPak(findResult->second);
}
return false;
}
bool CAsyncPakManager::LoadPakToMemAsync(const char* pPath, bool bLevelLoadOnly)
{
//check if pak reference exists
TPakMap::iterator findResult = m_paks.find(pPath);
if (findResult != m_paks.end())
{
return LoadPak(findResult->second);
}
else
{
char szFullPathBuf[AZ::IO::IArchive::MaxPath];
const char* szFullPath = gEnv->pCryPak->AdjustFileName(pPath, szFullPathBuf, AZ_ARRAY_SIZE(szFullPathBuf), AZ::IO::IArchive::FOPEN_HINT_QUIET | AZ::IO::IArchive::FLAGS_PATH_REAL);
// Check if the pak file actually exists before trying to load
if (!gEnv->pCryPak->IsFileExist(szFullPath, AZ::IO::IArchive::eFileLocation_Any))
{
// Cached file does not exist
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "Level cache pak file %s does not exist", szFullPath);
return false;
}
SAsyncPak layerPak;
layerPak.layername = pPath;
layerPak.filename = szFullPathBuf;
layerPak.nSize = 0;
layerPak.eLifeTime = bLevelLoadOnly ? SAsyncPak::LIFETIME_LOAD_ONLY : SAsyncPak::LIFETIME_LEVEL_COMPLETE;
m_paks[layerPak.layername] = layerPak;
return LoadPak(m_paks[layerPak.layername]);
}
return false;
}
bool CAsyncPakManager::LoadPak(SAsyncPak& layerPak)
{
layerPak.nRequestCount++;
if (layerPak.eState == SAsyncPak::STATE_LOADED || layerPak.bStreaming ||
layerPak.eState == SAsyncPak::STATE_REQUESTED)
{
return true;
}
layerPak.eState = SAsyncPak::STATE_REQUESTED;
//printf("Streaming level pak: %s\n", layerPak.layername.c_str());
StartStreaming(&layerPak);
return false;
}
//////////////////////////////////////////////////////////////////////////
void CAsyncPakManager::UnloadLayerPak(const char* sLayerName)
{
TPakMap::iterator findResult = m_paks.find(sLayerName);
if (findResult == m_paks.end())
{
return;
}
SAsyncPak& layerPak = findResult->second;
layerPak.nRequestCount--;
assert(layerPak.nRequestCount >= 0);
if (layerPak.nRequestCount > 0)
{
return;
}
if (layerPak.bStreaming)
{
if (layerPak.pReadStream)
{
layerPak.pReadStream->Abort();
}
layerPak.eState = SAsyncPak::STATE_REQUESTUNLOAD;
return;
}
if (layerPak.eState == SAsyncPak::STATE_LOADED)
{
ReleaseData(&layerPak);
m_bRequestLayerUpdate = true;
}
if (layerPak.eState == SAsyncPak::STATE_REQUESTED)
{
layerPak.eState = SAsyncPak::STATE_UNLOADED;
}
}
//////////////////////////////////////////////////////////////////////////
void CAsyncPakManager::GetLayerPakStats(
SLayerPakStats& stats, bool bCollectAllStats) const
{
stats.m_MaxSize = (g_cvars.archiveVars.nTotalInMemoryPakSizeLimit * MEGA_BYTE);
stats.m_UsedSize = m_nTotalOpenLayerPakSize;
for (TPakMap::const_iterator it = m_paks.begin(); it != m_paks.end(); ++it)
{
const SAsyncPak& layerPak = it->second;
if (bCollectAllStats || layerPak.eState != SAsyncPak::STATE_UNLOADED)
{
SLayerPakStats::SEntry entry;
entry.name = it->first;
entry.nSize = layerPak.nSize;
entry.bStreaming = layerPak.bStreaming;
layerPak.GetStatus(entry.status);
stats.m_entries.push_back(entry);
}
}
}
//////////////////////////////////////////////////////////////////////////
void CAsyncPakManager::StreamAsyncOnComplete(
IReadStream* pStream, unsigned nError)
{
if (nError != 0)
{
return;
}
SAsyncPak* pLayerPak = (SAsyncPak*) pStream->GetUserData();
//Check is pak is already open, if so, just assign mem
if (gEnv->pCryPak->LoadPakToMemory(pLayerPak->filename.c_str(), AZ::IO::IArchive::eInMemoryPakLocale_GPU, pLayerPak->pData))
{
pLayerPak->bPakAlreadyOpen = true;
}
else
{
bool usePrefabSystemForLevels = false;
AzFramework::ApplicationRequests::Bus::BroadcastResult(
usePrefabSystemForLevels,
&AzFramework::ApplicationRequests::IsPrefabSystemForLevelsEnabled);
if (usePrefabSystemForLevels)
{
gEnv->pCryPak->OpenPack(
"@assets@", {pLayerPak->filename.c_str(), pLayerPak->filename.size()}, AZ::IO::IArchive::FLAGS_FILENAMES_AS_CRC32, NULL);
}
else
{
//
// ugly hack - depending on the pak file pak may need special root info / open flags
//
if (pLayerPak->layername.find("level.pak") != string::npos)
{
gEnv->pCryPak->OpenPack(
{pLayerPak->filename.c_str(), pLayerPak->filename.size()}, AZ::IO::IArchive::FLAGS_FILENAMES_AS_CRC32, NULL);
}
else if (pLayerPak->layername.find("levelshadercache.pak") != string::npos)
{
gEnv->pCryPak->OpenPack(
"@assets@", {pLayerPak->filename.c_str(), pLayerPak->filename.size()}, AZ::IO::IArchive::FLAGS_PATH_REAL, NULL);
}
else
{
gEnv->pCryPak->OpenPack(
"@assets@", {pLayerPak->filename.c_str(), pLayerPak->filename.size()}, AZ::IO::IArchive::FLAGS_FILENAMES_AS_CRC32,
NULL);
}
}
gEnv->pCryPak->LoadPakToMemory(pLayerPak->filename.c_str(), AZ::IO::IArchive::eInMemoryPakLocale_GPU, pLayerPak->pData);
}
pLayerPak->eState = SAsyncPak::STATE_LOADED;
//printf("Finished streaming level pak: %s\n", pLayerPak->layername.c_str());
}
//////////////////////////////////////////////////////////////////////////
void CAsyncPakManager::StreamOnComplete(
IReadStream* pStream, unsigned nError)
{
SAsyncPak* pLayerPak = (SAsyncPak*) pStream->GetUserData();
if (nError != 0)
{
ReleaseData(pLayerPak);
}
pLayerPak->bStreaming = false;
pLayerPak->pReadStream = NULL;
m_bRequestLayerUpdate = true;
}
void* CAsyncPakManager::StreamOnNeedStorage(IReadStream* pStream, unsigned nSize, bool& bAbortOnFailToAlloc)
{
SAsyncPak* pAsyncPak = (SAsyncPak*)pStream->GetUserData();
pAsyncPak->nSize = nSize;
if ((m_nTotalOpenLayerPakSize + nSize) > (size_t)(g_cvars.archiveVars.nTotalInMemoryPakSizeLimit * MEGA_BYTE))
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "Not enough space to load in memory layer pak %s (Current: %" PRISIZE_T " Required: %d)",
pAsyncPak->filename.c_str(), m_nTotalOpenLayerPakSize, nSize);
//printf("Not enough space to load in memory layer pak %s (Current: %d Required: %d)\n", pAsyncPak->filename.c_str(), m_nTotalOpenLayerPakSize, nSize);
pAsyncPak->eState = SAsyncPak::STATE_UNLOADED;
pAsyncPak->bStreaming = false;
pAsyncPak->pReadStream = NULL;
bAbortOnFailToAlloc = true;
return NULL;
}
if (nSize)
{
auto pCryPak = static_cast<AZ::IO::Archive*>(gEnv->pCryPak);
// allocate the data
const char* szUsage = "In Memory Zip File";
pAsyncPak->pData = pCryPak->PoolAllocMemoryBlock(nSize, szUsage, alignof(uint8_t));
m_nTotalOpenLayerPakSize += nSize;
return pAsyncPak->pData->m_address.get();
}
return NULL;
}
//////////////////////////////////////////////////////////////////////////
void CAsyncPakManager::Update()
{
if (!m_bRequestLayerUpdate)
{
return;
}
m_bRequestLayerUpdate = false;
for (TPakMap::iterator it = m_paks.begin();
it != m_paks.end(); ++it)
{
SAsyncPak& layerPak = it->second;
if (!layerPak.bStreaming)
{
if (layerPak.eState == SAsyncPak::STATE_REQUESTUNLOAD)
{
// done streaming and not interested in it anymore, then release it again
ReleaseData(&layerPak);
}
else if (layerPak.eState == SAsyncPak::STATE_REQUESTED &&
(m_nTotalOpenLayerPakSize + layerPak.nSize <= ((size_t)g_cvars.archiveVars.nTotalInMemoryPakSizeLimit * MEGA_BYTE)))
{
// do we have enough memory now to start streaming the pak
StartStreaming(&layerPak);
}
}
}
}
// Abort streaming jobs and prevent any more requests
// Paks which are loaded remain, they will be cleaned up as usual
void CAsyncPakManager::CancelPendingJobs()
{
for (TPakMap::iterator it = m_paks.begin(); it != m_paks.end(); ++it)
{
SAsyncPak& layerPak = it->second;
if (layerPak.bStreaming)
{
layerPak.pReadStream->Abort();
ReleaseData(&layerPak);
//printf("Pak %s Aborted\n", layerPak.filename.c_str());
}
else if (layerPak.eState == SAsyncPak::STATE_REQUESTED)
{
layerPak.eState = SAsyncPak::STATE_UNLOADED;
ReleaseData(&layerPak);
//printf("Pak %s Cancelled\n", layerPak.filename.c_str());
}
}
}
//////////////////////////////////////////////////////////////////////////
Code/CryEngine/CrySystem/AsyncPakManager.h
-116
View File
@@ -1,116 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Manage async pak files
#ifndef CRYINCLUDE_CRYSYSTEM_ASYNCPAKMANAGER_H
#define CRYINCLUDE_CRYSYSTEM_ASYNCPAKMANAGER_H
#pragma once
#include <AzCore/std/smart_ptr/intrusive_ptr.h>
#include <IResourceManager.h>
#include <IStreamEngine.h>
namespace AZ::IO
{
struct MemoryBlock;
}
class CAsyncPakManager
: public IStreamCallback
{
protected:
struct SAsyncPak
{
enum EState
{
STATE_UNLOADED,
STATE_REQUESTED,
STATE_REQUESTUNLOAD,
STATE_LOADED,
};
enum ELifeTime
{
LIFETIME_LOAD_ONLY,
LIFETIME_LEVEL_COMPLETE,
LIFETIME_PERMANENT
};
SAsyncPak()
: nRequestCount(0)
, eState(STATE_UNLOADED)
, eLifeTime(LIFETIME_LOAD_ONLY)
, nSize(0)
, pData(0)
, bStreaming(false)
, bPakAlreadyOpen(false)
, bClosePakOnRelease(false)
, pReadStream(0) {}
string& GetStatus(string&) const;
string layername;
string filename;
size_t nSize;
AZStd::intrusive_ptr<AZ::IO::MemoryBlock> pData;
EState eState;
ELifeTime eLifeTime;
bool bStreaming;
bool bPakAlreadyOpen;
bool bClosePakOnRelease;
int nRequestCount;
IReadStreamPtr pReadStream;
};
typedef std::map<string, SAsyncPak> TPakMap;
public:
CAsyncPakManager();
~CAsyncPakManager();
void ParseLayerPaks(const string& levelCachePath);
bool LoadPakToMemAsync(const char* pPath, bool bLevelLoadOnly);
void UnloadLevelLoadPaks();
bool LoadLayerPak(const char* sLayerName);
void UnloadLayerPak(const char* sLayerName);
void CancelPendingJobs();
void GetLayerPakStats(SLayerPakStats& stats, bool bCollectAllStats) const;
void Clear();
void Update();
protected:
bool LoadPak(SAsyncPak& layerPak);
void StartStreaming(SAsyncPak* pLayerPak);
void ReleaseData(SAsyncPak* pLayerPak);
//////////////////////////////////////////////////////////////////////////
// IStreamCallback interface implementation.
//////////////////////////////////////////////////////////////////////////
virtual void StreamAsyncOnComplete (IReadStream* pStream, unsigned nError);
virtual void StreamOnComplete (IReadStream* pStream, unsigned nError);
virtual void* StreamOnNeedStorage(IReadStream* pStream, unsigned nSize, bool& bAbortOnFailToAlloc);
//////////////////////////////////////////////////////////////////////////
TPakMap m_paks;
size_t m_nTotalOpenLayerPakSize;
bool m_bRequestLayerUpdate;
};
#endif // CRYINCLUDE_CRYSYSTEM_ASYNCPAKMANAGER_H
Code/CryEngine/CrySystem/CrashHandler.rc
-114
View File
@@ -1,114 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "winres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
/////////////////////////////////////////////////////////////////////////////
//
// DESIGNINFO
//
#ifdef APSTUDIO_INVOKED
GUIDELINES DESIGNINFO
BEGIN
IDD_CRITICAL_ERROR, DIALOG
BEGIN
LEFTMARGIN, 5
RIGHTMARGIN, 260
BOTTOMMARGIN, 223
END
IDD_EXCEPTION, DIALOG
BEGIN
END
IDD_CONFIRM_SAVE_LEVEL, DIALOG
BEGIN
LEFTMARGIN, 7
RIGHTMARGIN, 239
TOPMARGIN, 7
BOTTOMMARGIN, 96
END
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Dialog
//
IDD_CRITICAL_ERROR DIALOGEX 0, 0, 267, 230
STYLE DS_SETFONT | DS_MODALFRAME | DS_SETFOREGROUND | DS_CENTER | WS_POPUP | WS_VISIBLE | WS_CAPTION
CAPTION "Critical Exception"
FONT 8, "MS Sans Serif", 0, 0, 0x0
BEGIN
DEFPUSHBUTTON "&Abort",IDB_EXIT,100,207,58,14
EDITTEXT IDC_CALLSTACK,10,95,245,102,ES_MULTILINE | ES_AUTOVSCROLL | WS_VSCROLL | WS_HSCROLL
EDITTEXT IDC_EXCEPTION_CODE,10,25,50,12,ES_AUTOHSCROLL | ES_READONLY
LTEXT "Call Stack Trace",IDC_STATIC,13,85,54,8
LTEXT "Code",IDC_STATIC,10,15,18,8
LTEXT "Address:",IDC_STATIC,66,15,28,8
EDITTEXT IDC_EXCEPTION_ADDRESS,65,25,75,12,ES_AUTOHSCROLL | ES_READONLY
LTEXT "Description",IDC_STATIC,10,40,36,8
GROUPBOX "Exception Info",IDC_STATIC,5,5,255,200
EDITTEXT IDC_EXCEPTION_MODULE,145,25,110,12,ES_AUTOHSCROLL | ES_READONLY
LTEXT "Module",IDC_STATIC,145,15,24,8
EDITTEXT IDC_EXCEPTION_DESC,10,50,245,30,ES_MULTILINE | ES_AUTOHSCROLL | ES_READONLY
PUSHBUTTON "&Ignore",IDB_IGNORE,160,207,59,14,WS_DISABLED
END
IDD_EXCEPTION DIALOG 0, 0, 138, 52
STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_POPUP | WS_VISIBLE | WS_CAPTION | WS_SYSMENU
CAPTION "Exception"
FONT 8, "MS Sans Serif"
BEGIN
LTEXT "Exception Intercepted\r\nRetrieving Info...",IDC_STATIC,33,18,71,19
END
IDD_CONFIRM_SAVE_LEVEL DIALOGEX 0, 0, 280, 123
STYLE DS_SYSMODAL | DS_SETFONT | DS_MODALFRAME | DS_SETFOREGROUND | DS_FIXEDSYS | DS_CENTER | WS_POPUP | WS_VISIBLE | WS_CAPTION
EXSTYLE WS_EX_TOPMOST
CAPTION "Engine, Game or Editor Crash"
FONT 8, "MS Shell Dlg", 400, 0, 0x1
BEGIN
PUSHBUTTON "Save",IDB_CONFIRM_SAVE,4,96,68,20
PUSHBUTTON "Cancel",IDB_DONT_SAVE,206,96,68,20
LTEXT "Open 3D Engine has encountered an error and needs to close.\n\nA backup has been saved to the '_savebackup' subfolder.\n\nIf you are unable to save your file, you can recover by copying the contents of the _savebackup folder over the broken files.",IDC_STATIC,60,8,210,61
LTEXT "Attempt save?",IDC_STATIC,60,72,180,21
CONTROL 128,IDC_STATIC,"Static",SS_BITMAP | SS_CENTERIMAGE | SS_REALSIZEIMAGE,8,8,48,40
END
/////////////////////////////////////////////////////////////////////////////
//
// Bitmap
//
IDB_CRASH_FACE BITMAP "crash_face.bmp"
#endif // English (United States) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED
Code/CryEngine/CrySystem/CustomMemoryHeap.cpp
-155
View File
@@ -1,155 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySystem_precompiled.h"
#include "CustomMemoryHeap.h"
#if defined(AZ_RESTRICTED_PLATFORM)
#undef AZ_RESTRICTED_SECTION
#define CUSTOMMEMORYHEAP_CPP_SECTION_1 1
#define CUSTOMMEMORYHEAP_CPP_SECTION_2 2
#define CUSTOMMEMORYHEAP_CPP_SECTION_3 3
#endif
#if defined(AZ_RESTRICTED_PLATFORM)
#define AZ_RESTRICTED_SECTION CUSTOMMEMORYHEAP_CPP_SECTION_1
#include AZ_RESTRICTED_FILE(CustomMemoryHeap_cpp)
#endif
//////////////////////////////////////////////////////////////////////////
CCustomMemoryHeapBlock::CCustomMemoryHeapBlock(CCustomMemoryHeap* pHeap)
: m_pHeap(pHeap)
, m_pData(0)
, m_nSize(0)
, m_nGPUHandle(0)
{
}
//////////////////////////////////////////////////////////////////////////
CCustomMemoryHeapBlock::~CCustomMemoryHeapBlock()
{
m_pHeap->DeallocateBlock(this);
}
//////////////////////////////////////////////////////////////////////////
void* CCustomMemoryHeapBlock::GetData()
{
return m_pData;
}
//////////////////////////////////////////////////////////////////////////
void CCustomMemoryHeapBlock::CopyMemoryRegion(void* pOutputBuffer, size_t nOffset, size_t nSize)
{
assert(nOffset + nSize <= m_nSize);
if (nOffset + nSize <= m_nSize)
{
memcpy(pOutputBuffer, (uint8*)m_pData + nOffset, nSize);
}
else
{
CryFatalError("Bad CopyMemoryRegion range");
}
}
//////////////////////////////////////////////////////////////////////////
ICustomMemoryBlock* CCustomMemoryHeap::AllocateBlock(size_t const nAllocateSize, char const* const sUsage, size_t const nAlignment /* = 16 */)
{
CCustomMemoryHeapBlock* pBlock = new CCustomMemoryHeapBlock(this);
pBlock->m_sUsage = sUsage;
pBlock->m_nSize = nAllocateSize;
switch (m_eAllocPolicy)
{
case IMemoryManager::eapDefaultAllocator:
{
size_t allocated = 0;
pBlock->m_pData = CryMalloc(nAllocateSize, allocated, nAlignment);
break;
}
case IMemoryManager::eapPageMapped:
pBlock->m_pData = CryMemory::AllocPages(nAllocateSize);
break;
case IMemoryManager::eapCustomAlignment:
#if defined(DEBUG)
if (nAlignment == 0)
{
CryFatalError("CCustomMemoryHeap: trying to allocate memory via eapCustomAlignment with an alignment of zero!");
}
#endif
pBlock->m_pData = CryModuleMemalign(nAllocateSize, nAlignment);
break;
#if defined(AZ_RESTRICTED_PLATFORM)
#define AZ_RESTRICTED_SECTION CUSTOMMEMORYHEAP_CPP_SECTION_2
#include AZ_RESTRICTED_FILE(CustomMemoryHeap_cpp)
#endif
default:
CryFatalError("CCustomMemoryHeap: unknown allocation policy during AllocateBlock!");
break;
}
CryInterlockedAdd(&m_nAllocatedSize, nAllocateSize);
return pBlock;
}
void CCustomMemoryHeap::DeallocateBlock(CCustomMemoryHeapBlock* pBlock)
{
switch (m_eAllocPolicy)
{
case IMemoryManager::eapDefaultAllocator:
CryFree(pBlock->m_pData, 0);
break;
case IMemoryManager::eapPageMapped:
CryMemory::FreePages(pBlock->m_pData, pBlock->GetSize());
break;
case IMemoryManager::eapCustomAlignment:
CryModuleMemalignFree(pBlock->m_pData);
break;
#if defined(AZ_RESTRICTED_PLATFORM)
#define AZ_RESTRICTED_SECTION CUSTOMMEMORYHEAP_CPP_SECTION_3
#include AZ_RESTRICTED_FILE(CustomMemoryHeap_cpp)
#endif
default:
CryFatalError("CCustomMemoryHeap: unknown allocation policy during DeallocateBlock!");
break;
}
int nAllocateSize = (int)pBlock->m_nSize;
CryInterlockedAdd(&m_nAllocatedSize, -nAllocateSize);
}
//////////////////////////////////////////////////////////////////////////
void CCustomMemoryHeap::GetMemoryUsage(ICrySizer* pSizer)
{
pSizer->AddObject(this, m_nAllocatedSize);
}
//////////////////////////////////////////////////////////////////////////
size_t CCustomMemoryHeap::GetAllocated()
{
return m_nAllocatedSize;
}
//////////////////////////////////////////////////////////////////////////
CCustomMemoryHeap::CCustomMemoryHeap(IMemoryManager::EAllocPolicy const eAllocPolicy)
{
m_nAllocatedSize = 0;
m_eAllocPolicy = eAllocPolicy;
m_nTraceHeapHandle = 0;
}
//////////////////////////////////////////////////////////////////////////
CCustomMemoryHeap::~CCustomMemoryHeap()
{
}
Code/CryEngine/CrySystem/DllMain.cpp
-44
View File
@@ -62,46 +62,6 @@ AZ_POP_DISABLE_WARNING
}
#endif
//////////////////////////////////////////////////////////////////////////
struct CSystemEventListner_System
: public ISystemEventListener
{
public:
virtual void OnSystemEvent(ESystemEvent event, [[maybe_unused]] UINT_PTR wparam, [[maybe_unused]] UINT_PTR lparam)
{
switch (event)
{
case ESYSTEM_EVENT_LEVEL_LOAD_START:
case ESYSTEM_EVENT_LEVEL_LOAD_END:
{
CryCleanup();
break;
}
case ESYSTEM_EVENT_LEVEL_POST_UNLOAD:
{
CryCleanup();
STLALLOCATOR_CLEANUP;
break;
}
}
}
};
static CSystemEventListner_System g_system_event_listener_system;
static AZ::EnvironmentVariable<IMemoryManager*> s_cryMemoryManager;
// Force the CryMemoryManager into the AZ::Environment for exposure to other DLLs
void ExportCryMemoryManager()
{
IMemoryManager* cryMemoryManager = nullptr;
CryGetIMemoryManagerInterface((void**)&cryMemoryManager);
AZ_Assert(cryMemoryManager, "Unable to resolve CryMemoryManager");
s_cryMemoryManager = AZ::Environment::CreateVariable<IMemoryManager*>("CryIMemoryManagerInterface", cryMemoryManager);
}
extern "C"
{
CRYSYSTEM_API ISystem* CreateSystemInterface(const SSystemInitParams& startupParams)
@@ -113,8 +73,6 @@ CRYSYSTEM_API ISystem* CreateSystemInterface(const SSystemInitParams& startupPar
// Environment should have been attached via InjectEnvironment
AZ_Assert(AZ::Environment::IsReady(), "Environment is not attached, must be attached before CreateSystemInterface can be called");
ExportCryMemoryManager();
pSystem = new CSystem(startupParams.pSharedEnvironment);
ModuleInitISystem(pSystem, "CrySystem");
@@ -146,8 +104,6 @@ CRYSYSTEM_API ISystem* CreateSystemInterface(const SSystemInitParams& startupPar
return 0;
}
pSystem->GetISystemEventDispatcher()->RegisterListener(&g_system_event_listener_system);
return pSystem;
}
};
Code/CryEngine/CrySystem/GeneralMemoryHeap.cpp
-188
View File
@@ -1,188 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySystem_precompiled.h"
#include "GeneralMemoryHeap.h"
#include <AzCore/Memory/AllocatorBase.h>
#include <AzCore/Memory/HphaSchema.h>
class GeneralMemoryHeapAllocator
: public AZ::SimpleSchemaAllocator<AZ::HphaSchema>
{
using Base = AZ::SimpleSchemaAllocator<AZ::HphaSchema>;
public:
static const size_t DEFAULT_ALIGNMENT = sizeof(void*);
GeneralMemoryHeapAllocator(const char* desc)
: Base("GeneralMemoryHeapAllocator", desc)
{
}
void Reserve(size_t size)
{
// Allocate a block, then free it, forcing it into the page tree/cache
void* block = m_schema->Allocate(size, GeneralMemoryHeapAllocator::DEFAULT_ALIGNMENT, 0, "GeneralMemoryHeapAllocator Reserve", __FILE__, __LINE__);
m_schema->DeAllocate(block);
}
};
CGeneralMemoryHeap::CGeneralMemoryHeap([[maybe_unused]] UINT_PTR base, [[maybe_unused]] size_t upperLimit, size_t reserveSize, const char* sUsage)
: m_refCount(0)
, m_block(nullptr)
, m_blockSize(0)
{
AZ::HphaSchema::Descriptor desc;
desc.m_subAllocator = &AZ::AllocatorInstance<AZ::LegacyAllocator>::Get();
m_allocator.reset(new AZ::AllocatorWrapper<GeneralMemoryHeapAllocator>);
m_allocator->Create(desc, sUsage);
if (reserveSize)
{
(*m_allocator)->Reserve(reserveSize);
}
}
CGeneralMemoryHeap::CGeneralMemoryHeap(void* base, size_t size, const char* sUsage)
: m_refCount(0)
, m_block(base)
, m_blockSize(size)
{
AZ::HphaSchema::Descriptor desc;
desc.m_fixedMemoryBlock = base;
desc.m_fixedMemoryBlockByteSize = size;
desc.m_fixedMemoryBlockAlignment = GeneralMemoryHeapAllocator::DEFAULT_ALIGNMENT;
m_allocator.reset(new AZ::AllocatorWrapper<GeneralMemoryHeapAllocator>);
m_allocator->Create(desc, sUsage);
}
CGeneralMemoryHeap::~CGeneralMemoryHeap()
{
}
bool CGeneralMemoryHeap::Cleanup()
{
(*m_allocator)->GarbageCollect();
return true;
}
int CGeneralMemoryHeap::AddRef()
{
return m_refCount.fetch_add(1);
}
int CGeneralMemoryHeap::Release()
{
int nRef = m_refCount.fetch_sub(1);
if (nRef <= 1)
{
delete this;
}
return nRef;
}
void CGeneralMemoryHeap::RecordAlloc(void* ptr, size_t size)
{
if (m_block == nullptr)
{
m_allocs.emplace(ptr, size);
}
}
void CGeneralMemoryHeap::RecordFree(void* ptr, size_t size)
{
if (m_block == nullptr)
{
m_allocs.erase(Alloc(ptr, size));
}
}
bool CGeneralMemoryHeap::IsInAddressRange(void* ptr) const
{
if (m_block)
{
return (static_cast<char*>(ptr) - static_cast<char*>(m_block)) <= m_blockSize;
}
auto it = m_allocs.find(Alloc(ptr));
return it != m_allocs.end();
}
void* CGeneralMemoryHeap::Calloc(size_t numElements, size_t size, const char* sUsage)
{
void* ptr = (*m_allocator)->Allocate(numElements * size, GeneralMemoryHeapAllocator::DEFAULT_ALIGNMENT, 0, sUsage, __FILE__, __LINE__);
memset(ptr, 0, numElements * size);
RecordAlloc(ptr, numElements * size);
return ptr;
}
void* CGeneralMemoryHeap::Malloc(size_t size, const char* sUsage)
{
void* ptr = (*m_allocator)->Allocate(size, GeneralMemoryHeapAllocator::DEFAULT_ALIGNMENT, 0, sUsage, __FILE__, __LINE__);
RecordAlloc(ptr, size);
return ptr;
}
size_t CGeneralMemoryHeap::Free(void* ptr)
{
// The client code using these heaps tend to use a guesswork algorithm to freeing
// which involves handing the pointer to every known heap until it frees, so
// it's necessary to validate that the ptr belongs to this heap before attempting to free
if (IsInAddressRange(ptr))
{
size_t size = (*m_allocator)->AllocationSize(ptr);
RecordFree(ptr, size);
(*m_allocator)->DeAllocate(ptr);
return size;
}
return 0;
}
void* CGeneralMemoryHeap::Realloc(void* ptr, size_t size, const char* /*sUsage*/)
{
RecordFree(ptr, (*m_allocator)->AllocationSize(ptr));
void* newPtr = (*m_allocator)->ReAllocate(ptr, size, GeneralMemoryHeapAllocator::DEFAULT_ALIGNMENT);
RecordAlloc(newPtr, size);
return newPtr;
}
void* CGeneralMemoryHeap::ReallocAlign(void* ptr, size_t size, size_t alignment, const char* /*sUsage*/)
{
RecordFree(ptr, (*m_allocator)->AllocationSize(ptr));
void* newPtr = (*m_allocator)->ReAllocate(ptr, size, alignment);
RecordAlloc(newPtr, size);
return newPtr;
}
void* CGeneralMemoryHeap::Memalign(size_t boundary, size_t size, const char* sUsage)
{
void* ptr = (*m_allocator)->Allocate(size, boundary, 0, sUsage, __FILE__, __LINE__);
RecordAlloc(ptr, size);
return ptr;
}
size_t CGeneralMemoryHeap::UsableSize(void* ptr) const
{
// The client code using these heaps tend to use a guesswork algorithm to determine
// which heap owns the pointer. Calls to UsableSize() are a part of this guesswork.
// The overrun detector doesn't play nicely on AllocationSize() lookups for pointers that
// don't belong to the heap, so validate that we're in the correct address range before trying
// to look up the size.
return IsInAddressRange(ptr) ? (*m_allocator)->AllocationSize(ptr) : 0;
}
AZ::IAllocator* CGeneralMemoryHeap::GetAllocator() const
{
return m_allocator->Get();
}
Code/CryEngine/CrySystem/GeneralMemoryHeap.h
-94
View File
@@ -1,94 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_CRYSYSTEM_GENERALMEMORYHEAP_H
#define CRYINCLUDE_CRYSYSTEM_GENERALMEMORYHEAP_H
#pragma once
#include "IMemory.h"
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <AzCore/std/parallel/atomic.h>
#include <AzCore/std/containers/set.h>
class GeneralMemoryHeapAllocator;
class CGeneralMemoryHeap
: public IGeneralMemoryHeap
{
struct Alloc
{
void* m_base;
size_t m_size;
Alloc(void* base = nullptr, size_t size = 0)
: m_base(base)
, m_size(size)
{}
bool operator==(const Alloc& rhs) const
{
// size doesn't matter
return m_base == rhs.m_base;
}
bool operator<(const Alloc& rhs) const
{
// this will cause allocs to be sorted by address
return m_base < rhs.m_base;
}
};
public:
// Create a heap that will map/unmap pages in the range [baseAddress, baseAddress + upperLimit).
CGeneralMemoryHeap(UINT_PTR baseAddress, size_t upperLimit, size_t reserveSize, const char* sUsage);
// Create a heap that will assumes all memory in the range [base, base + size) is already mapped.
CGeneralMemoryHeap(void* base, size_t size, const char* sUsage);
~CGeneralMemoryHeap();
public: // IGeneralMemoryHeap Members
bool Cleanup();
int AddRef();
int Release();
bool IsInAddressRange(void* ptr) const;
void* Calloc(size_t nmemb, size_t size, const char* sUsage = NULL);
void* Malloc(size_t sz, const char* sUsage = NULL);
size_t Free(void* ptr);
void* Realloc(void* ptr, size_t sz, const char* sUsage = NULL);
void* ReallocAlign(void* ptr, size_t size, size_t alignment, const char* sUsage = NULL);
void* Memalign(size_t boundary, size_t size, const char* sUsage = NULL);
size_t UsableSize(void* ptr) const;
AZ::IAllocator* GetAllocator() const override;
private:
CGeneralMemoryHeap(const CGeneralMemoryHeap&) = delete;
CGeneralMemoryHeap& operator = (const CGeneralMemoryHeap&) = delete;
void RecordAlloc(void* ptr, size_t size);
void RecordFree(void* ptr, size_t size);
private:
AZStd::unique_ptr<AZ::AllocatorWrapper<GeneralMemoryHeapAllocator>> m_allocator;
AZStd::atomic_int m_refCount;
void* m_block;
size_t m_blockSize;
AZStd::set<Alloc, AZStd::less<Alloc>, AZ::AZStdAlloc<AZ::LegacyAllocator>> m_allocs;
};
#endif // CRYINCLUDE_CRYSYSTEM_GENERALMEMORYHEAP_H
Code/CryEngine/CrySystem/ImageHandler.cpp
-280
View File
@@ -1,280 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include "CrySystem_precompiled.h"
#include <AzCore/IO/SystemFile.h>
#include "ImageHandler.h"
#include <numeric>
#include "ScopeGuard.h"
#include "Algorithm.h"
#include "System.h"
#if defined(AZ_RESTRICTED_PLATFORM)
#include AZ_RESTRICTED_FILE(ImageHandler_cpp)
#endif
#if !(defined(ANDROID) || defined(IOS) || defined(LINUX)) && AZ_LEGACY_CRYSYSTEM_TRAIT_IMAGEHANDLER_TIFFIO // Rally US1050 - Compile libtiff for Android and IOS
#include <tiffio.h>
static_assert(sizeof(thandle_t) >= sizeof(AZ::IO::HandleType), "Platform defines thandle_t to be smaller than required");
#endif
namespace
{
class Image
: public IImageHandler::IImage
{
public:
Image(std::vector<unsigned char>&& data, int width, int height)
{
CRY_ASSERT(data.size() == width * height * ImageHandler::c_BytesPerPixel);
m_data = std::move(data);
m_width = width;
m_height = height;
}
private:
virtual const std::vector<unsigned char>& GetData() const override { return m_data; }
virtual int GetWidth() const override { return m_width; }
virtual int GetHeight() const override { return m_height; }
unsigned int m_width;
unsigned int m_height;
std::vector<unsigned char> m_data;
};
#if AZ_LEGACY_CRYSYSTEM_TRAIT_IMAGEHANDLER_TIFFIO
struct TiffIO
{
static tsize_t Read(thandle_t handle, tdata_t buffer, tsize_t size)
{
AZ::u64 bytesRead = 0;
AZ::IO::FileIOBase::GetDirectInstance()->Read(static_cast<AZ::IO::HandleType>(reinterpret_cast<AZ::u64>(handle)), buffer, size, false, &bytesRead);
return static_cast<tsize_t>(bytesRead);
};
static tsize_t Write(thandle_t handle, tdata_t buffer, tsize_t size)
{
AZ::u64 sizeWritten;
if (AZ::IO::FileIOBase::GetDirectInstance()->Write(static_cast<AZ::IO::HandleType>(reinterpret_cast<AZ::u64>(handle)), buffer, size, &sizeWritten))
{
return static_cast<tsize_t>(sizeWritten);
}
else
{
return 0;
}
};
static int Close(thandle_t handle)
{
AZ::IO::FileIOBase::GetDirectInstance()->Close(static_cast<AZ::IO::HandleType>(reinterpret_cast<AZ::u64>(handle)));
return 0;
};
static toff_t Seek(thandle_t handle, toff_t pos, int mode)
{
if (AZ::IO::FileIOBase::GetDirectInstance()->Seek(static_cast<AZ::IO::HandleType>(reinterpret_cast<AZ::u64>(handle)), static_cast<uint64_t>(pos), AZ::IO::GetSeekTypeFromFSeekMode(mode)))
{
if (mode == SEEK_SET)
{
return pos;
}
else
{
AZ::u64 offsetFromBegin;
if (AZ::IO::FileIOBase::GetDirectInstance()->Tell(static_cast<AZ::IO::HandleType>(reinterpret_cast<AZ::u64>(handle)), offsetFromBegin))
{
return static_cast<tsize_t>(offsetFromBegin);
}
else
{
return -1;
}
}
}
return -1;
};
static toff_t Size(thandle_t handle)
{
AZ::u64 fileSize = 0;
AZ::IO::FileIOBase::GetDirectInstance()->Size(static_cast<AZ::IO::HandleType>(reinterpret_cast<AZ::u64>(handle)), fileSize);
return static_cast<tsize_t>(fileSize);
};
static int Map(thandle_t, tdata_t*, toff_t*)
{
return 0;
};
static void Unmap(thandle_t, tdata_t, toff_t)
{
return;
};
};
#endif
}
std::unique_ptr<IImageHandler::IImage> ImageHandler::CreateImage(std::vector<unsigned char>&& data, int width, int height) const
{
return std::make_unique<Image>(std::move(data), width, height);
}
std::unique_ptr<IImageHandler::IImage> ImageHandler::LoadImage([[maybe_unused]] const char* filename) const
{
#if AZ_LEGACY_CRYSYSTEM_TRAIT_IMAGEHANDLER_TIFFIO
AZ::IO::HandleType fileHandle;
AZ::IO::FileIOBase::GetDirectInstance()->Open(filename, AZ::IO::GetOpenModeFromStringMode("rb"), fileHandle);
if (fileHandle == AZ::IO::InvalidHandle)
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_ERROR, "Failed to open image file %s", filename);
return nullptr;
}
auto tifHandle = std17::unique_resource_checked(TIFFClientOpen(filename, "rb", reinterpret_cast<thandle_t>(static_cast<AZ::u64>(fileHandle)), TiffIO::Read, TiffIO::Write, TiffIO::Seek, TiffIO::Close, &TiffIO::Size, TiffIO::Map, TiffIO::Unmap), (TIFF*)nullptr, TIFFClose);
if (!tifHandle)
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_ERROR, "Failed to load image %s", filename);
return nullptr;
}
int width = 0;
int height = 0;
TIFFGetField(tifHandle, TIFFTAG_IMAGEWIDTH, &width);
TIFFGetField(tifHandle, TIFFTAG_IMAGELENGTH, &height);
std::vector<unsigned char> data(4 * width * height);
if (!TIFFReadRGBAImageOriented(tifHandle, width, height, reinterpret_cast<uint32*>(data.data()), ORIENTATION_TOPLEFT))
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_ERROR, "Failed to load image %s", filename);
return nullptr;
}
//strip alpha
int every4th = 0;
data.erase(std::remove_if(begin(data), end(data), [&](unsigned char)
{
return (every4th++ & 3) == 3;
}), end(data));
return std::make_unique<Image>(std::move(data), width, height);
#else
CRY_ASSERT(0); // UNIMPLEMENTED
return nullptr;
#endif
}
bool ImageHandler::SaveImage([[maybe_unused]] IImageHandler::IImage* image, [[maybe_unused]] const char* filename) const
{
#if AZ_LEGACY_CRYSYSTEM_TRAIT_IMAGEHANDLER_TIFFIO
AZ::IO::HandleType fileHandle;
AZ::IO::FileIOBase::GetDirectInstance()->Open(filename, AZ::IO::GetOpenModeFromStringMode("wb"), fileHandle);
if (fileHandle == AZ::IO::InvalidHandle)
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_ERROR, "Failed to open image file for write %s", filename);
return false;
}
auto tifHandle = std17::unique_resource_checked(TIFFClientOpen(filename, "wb", reinterpret_cast<thandle_t>(static_cast<AZ::u64>(fileHandle)), TiffIO::Read, TiffIO::Write, TiffIO::Seek, TiffIO::Close, &TiffIO::Size, TiffIO::Map, TiffIO::Unmap), (TIFF*)nullptr, TIFFClose);
if (!tifHandle)
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_ERROR, "Failed to save image %s", filename);
return false;
}
TIFFSetField(tifHandle, TIFFTAG_IMAGEWIDTH, image->GetWidth());
TIFFSetField(tifHandle, TIFFTAG_IMAGELENGTH, image->GetHeight());
TIFFSetField(tifHandle, TIFFTAG_SAMPLESPERPIXEL, c_BytesPerPixel);
TIFFSetField(tifHandle, TIFFTAG_BITSPERSAMPLE, 8);
TIFFSetField(tifHandle, TIFFTAG_ORIENTATION, ORIENTATION_TOPLEFT);
TIFFSetField(tifHandle, TIFFTAG_PLANARCONFIG, PLANARCONFIG_CONTIG);
TIFFSetField(tifHandle, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_RGB);
TIFFSetField(tifHandle, TIFFTAG_COMPRESSION, COMPRESSION_LZW);
tsize_t bytesPerLine = c_BytesPerPixel * image->GetWidth();
std::vector<unsigned char> lineBuffer;
if (TIFFScanlineSize(tifHandle) != bytesPerLine)
{
lineBuffer.resize(bytesPerLine);
}
else
{
lineBuffer.resize(TIFFScanlineSize(tifHandle));
}
TIFFSetField(tifHandle, TIFFTAG_ROWSPERSTRIP, TIFFDefaultStripSize(tifHandle, image->GetWidth() * c_BytesPerPixel));
auto srcData = image->GetData().data();
for (uint32 row = 0; row < image->GetHeight(); row++)
{
memcpy(lineBuffer.data(), &srcData[row * bytesPerLine], bytesPerLine);
if (TIFFWriteScanline(tifHandle, lineBuffer.data(), row, 0) < 0)
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "Failed to write part of image %s", filename);
return false;
}
}
return true;
#else
CRY_ASSERT(0); // UNIMPLEMENTED
return false;
#endif
}
std::unique_ptr<IImageHandler::IImage> ImageHandler::CreateDiffImage(IImageHandler::IImage* image1, IImageHandler::IImage* image2) const
{
if (!image1 || !image2)
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "Could not create diff image, null arguments");
return nullptr;
}
if (image1->GetWidth() != image2->GetWidth() || image1->GetHeight() != image2->GetHeight())
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "Could not create diff image, 2 images were not the same size");
return nullptr;
}
CRY_ASSERT(image1->GetData().size() == image1->GetWidth() * image1->GetHeight() * ImageHandler::c_BytesPerPixel);
CRY_ASSERT(image2->GetData().size() == image2->GetWidth() * image2->GetHeight() * ImageHandler::c_BytesPerPixel);
std::vector<unsigned char> resultRGBData;
auto iter1 = image1->GetData().data();
auto iter2 = image2->GetData().data();
for (int i = 0; i < image1->GetWidth() * image1->GetHeight() * c_BytesPerPixel; ++i)
{
resultRGBData.push_back(static_cast<unsigned char>(abs(static_cast<int>(iter1[i]) - static_cast<int>(iter2[i]))));
}
return std::make_unique<Image>(std::move(resultRGBData), image1->GetWidth(), image1->GetHeight());
}
float ImageHandler::CalculatePSNR(IImageHandler::IImage* diffIimage) const
{
if (!diffIimage)
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "Could not create diff image, null arguments");
return 0;
}
CRY_ASSERT(diffIimage->GetData().size() == diffIimage->GetWidth() * diffIimage->GetHeight() * ImageHandler::c_BytesPerPixel);
auto mse = std17::accumulate(diffIimage->GetData(), 0.0, [](double result, unsigned char value) -> double { return result += (double)value * (double)value; });
mse /= (c_BytesPerPixel * diffIimage->GetWidth() * diffIimage->GetHeight());
if (mse <= 0)
{
return std::numeric_limits<float>::max();
}
// see http://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio for a derivation of this formula and source for magic numbers
return static_cast<float>(20 * log10(255) - 10 * log10(mse));
}
Code/CryEngine/CrySystem/ImageHandler.h
-30
View File
@@ -1,30 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#ifndef CRYINCLUDE_CRYSYSTEM_IMAGEHANDLER_H
#define CRYINCLUDE_CRYSYSTEM_IMAGEHANDLER_H
#include "IImageHandler.h"
class ImageHandler
: public IImageHandler
{
public:
static const int c_BytesPerPixel = 3; //This only deals with RGB data for now, no alpha
private:
virtual std::unique_ptr<IImageHandler::IImage> CreateImage(std::vector<unsigned char>&& rgbData, int width, int height) const override;
virtual std::unique_ptr<IImageHandler::IImage> LoadImage(const char* filename) const override;
virtual bool SaveImage(IImageHandler::IImage* image, const char* filename) const override;
virtual std::unique_ptr<IImageHandler::IImage> CreateDiffImage(IImageHandler::IImage* image1, IImageHandler::IImage* image2) const override;
virtual float CalculatePSNR(IImageHandler::IImage* diffIimage) const override;
};
#endif // CRYINCLUDE_CRYSYSTEM_IMAGEHANDLER_H
Code/CryEngine/CrySystem/LevelSystem/LevelSystem.cpp
-6
View File
@@ -17,7 +17,6 @@
#include "LevelSystem.h"
#include <IAudioSystem.h>
#include "IMovieSystem.h"
#include <IResourceManager.h>
#include <ILocalizationManager.h>
#include "CryPath.h"
#include <Pak/CryPakUtils.h>
@@ -778,9 +777,6 @@ void CLevelSystem::PrepareNextLevel(const char* levelName)
// switched to level heap, so now imm start the loading screen (renderer will be reinitialized in the levelheap)
gEnv->pSystem->GetISystemEventDispatcher()->OnSystemEvent(ESYSTEM_EVENT_LEVEL_LOAD_START_LOADINGSCREEN, 0, 0);
gEnv->pSystem->SetSystemGlobalState(ESYSTEM_GLOBAL_STATE_LEVEL_LOAD_START_PREPARE);
// Inform resource manager about loading of the new level.
GetISystem()->GetIResourceManager()->PrepareLevel(pLevelInfo->GetPath(), pLevelInfo->GetName());
}
for (AZStd::vector<ILevelSystemListener*>::const_iterator it = m_listeners.begin(); it != m_listeners.end(); ++it)
@@ -987,8 +983,6 @@ void CLevelSystem::UnloadLevel()
m_lastLevelName.clear();
GetISystem()->GetIResourceManager()->UnloadLevel();
SAFE_RELEASE(m_pCurrentLevel);
// Force Lua garbage collection (may no longer be needed now the legacy renderer has been removed).
Code/CryEngine/CrySystem/LevelSystem/SpawnableLevelSystem.cpp
-3
View File
@@ -14,7 +14,6 @@
#include "SpawnableLevelSystem.h"
#include <IAudioSystem.h>
#include "IMovieSystem.h"
#include <IResourceManager.h>
#include <LoadScreenBus.h>
@@ -566,8 +565,6 @@ namespace LegacyLevelSystem
m_lastLevelName.clear();
GetISystem()->GetIResourceManager()->UnloadLevel();
// Force Lua garbage collection (may no longer be needed now the legacy renderer has been removed).
// Normally the GC step is triggered at the end of this method (by the ESYSTEM_EVENT_LEVEL_POST_UNLOAD event).
EBUS_EVENT(AZ::ScriptSystemRequestBus, GarbageCollect);
Code/CryEngine/CrySystem/Log.cpp
-17
View File
@@ -20,7 +20,6 @@
//this should not be included here
#include <IConsole.h>
#include <ISystem.h>
#include <IStreamEngine.h>
#include "System.h"
#include "CryPath.h" // PathUtil::ReplaceExtension()
#include <Pak/CryPakUtils.h>
@@ -1439,22 +1438,6 @@ void CLog::UpdateLoadingScreen(const char* szFormat, ...)
va_end(args);
}
#endif
if (CryGetCurrentThreadId() == m_nMainThreadId)
{
#ifndef LINUX
// Take this opportunity to update streaming engine.
if (IStreamEngine* pStreamEngine = GetISystem()->GetStreamEngine())
{
const float curTime = m_pSystem->GetITimer()->GetAsyncCurTime();
if (curTime - m_fLastLoadingUpdateTime > .1f) // not frequent than once in 100ms
{
m_fLastLoadingUpdateTime = curTime;
pStreamEngine->Update();
}
}
#endif
}
}
//////////////////////////////////////////////////////////////////////////
Code/CryEngine/CrySystem/MTSafeAllocator.cpp
-163
View File
@@ -1,163 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySystem_precompiled.h"
#include "MTSafeAllocator.h"
#include <IConsole.h>
extern CMTSafeHeap* g_pPakHeap;
// Uncomment this define to enable time tracing of the MTSAFE heap
#define MTSAFE_PROFILE 1
//#undef MTSAFE_PROFILE
namespace
{
class CSimpleTimer
{
LARGE_INTEGER& m_result;
LARGE_INTEGER m_start;
public:
CSimpleTimer(LARGE_INTEGER& li)
: m_result(li)
{ QueryPerformanceCounter(&m_start); }
~CSimpleTimer()
{
LARGE_INTEGER end;
QueryPerformanceCounter(&end);
m_result.QuadPart = end.QuadPart - m_start.QuadPart;
}
};
};
//////////////////////////////////////////////////////////////////////////
CMTSafeHeap::CMTSafeHeap()
: m_LiveTempAllocations()
, m_TotalAllocations()
, m_TempAllocationsFailed()
, m_TempAllocationsTime()
{
size_t allocated = 0;
m_pGeneralHeapStorage = (char*)CryMalloc(MTSAFE_GENERAL_HEAP_SIZE, allocated, MTSAFE_DEFAULT_ALIGNMENT);
m_pGeneralHeapStorageEnd = m_pGeneralHeapStorage + MTSAFE_GENERAL_HEAP_SIZE;
m_pGeneralHeap = CryGetIMemoryManager()->CreateGeneralMemoryHeap(m_pGeneralHeapStorage, MTSAFE_GENERAL_HEAP_SIZE, "MTSafeHeap");
}
//////////////////////////////////////////////////////////////////////////
CMTSafeHeap::~CMTSafeHeap()
{
SAFE_RELEASE(m_pGeneralHeap);
CryFree(m_pGeneralHeapStorage, MTSAFE_DEFAULT_ALIGNMENT);
}
//////////////////////////////////////////////////////////////////////////
size_t CMTSafeHeap::PersistentAllocSize(size_t nSize)
{
return nSize;
}
//////////////////////////////////////////////////////////////////////////
void* CMTSafeHeap::PersistentAlloc(size_t nSize)
{
size_t allocated = 0;
return CryMalloc(nSize, allocated, MTSAFE_DEFAULT_ALIGNMENT);
}
//////////////////////////////////////////////////////////////////////////
void CMTSafeHeap::FreePersistent(void* p)
{
CryFree(p, MTSAFE_DEFAULT_ALIGNMENT);
}
//////////////////////////////////////////////////////////////////////////
void* CMTSafeHeap::TempAlloc(size_t nSize, const char* szDbgSource, bool& bFallBackToMalloc, uint32 align)
{
# if MTSAFE_PROFILE
CSimpleTimer timer(m_TempAllocationsTime);
# endif
void* ptr = NULL;
if (align)
{
ptr = m_pGeneralHeap->Memalign(align, nSize, szDbgSource);
}
else
{
ptr = m_pGeneralHeap->Malloc(nSize, szDbgSource);
}
//explicit alignment not supported beyond this point, safer to return NULL
if (ptr || !bFallBackToMalloc)
{
bFallBackToMalloc = false;
return ptr;
}
bFallBackToMalloc = true;
# if MTSAFE_PROFILE
CryInterlockedAdd((volatile int*)&m_TempAllocationsFailed, (int)nSize);
# endif
return CryModuleMemalign(nSize, align > 0 ? align : MTSAFE_DEFAULT_ALIGNMENT);
}
//////////////////////////////////////////////////////////////////////////
void CMTSafeHeap::FreeTemporary(void* p)
{
# if MTSAFE_PROFILE
CSimpleTimer timer(m_TempAllocationsTime);
# endif
if (m_pGeneralHeap->IsInAddressRange(p))
{
m_pGeneralHeap->Free(p);
return;
}
// Fallback to free
CryModuleMemalignFree(p);
}
//////////////////////////////////////////////////////////////////////////
void* CMTSafeHeap::StaticAlloc([[maybe_unused]] void* pOpaque, unsigned nItems, unsigned nSize)
{
return g_pPakHeap->TempAlloc(nItems * nSize, "StaticAlloc");
}
//////////////////////////////////////////////////////////////////////////
void CMTSafeHeap::StaticFree ([[maybe_unused]] void* pOpaque, void* pAddress)
{
g_pPakHeap->FreeTemporary(pAddress);
}
//////////////////////////////////////////////////////////////////////////
void CMTSafeHeap::GetMemoryUsage(ICrySizer* pSizer)
{
SIZER_COMPONENT_NAME(pSizer, "FileSystem Pool");
}
void CMTSafeHeap::PrintStats()
{
# if MTSAFE_PROFILE
LARGE_INTEGER freq;
QueryPerformanceFrequency(&freq);
const double rFreq = 1. / static_cast<double>(freq.QuadPart);
CryLogAlways("mtsafe temporary pool failed for %" PRISIZE_T " bytes, time spent in allocations %3.08f seconds",
m_TempAllocationsFailed, static_cast<double>(m_TempAllocationsTime.QuadPart) * rFreq);
# endif
}
Code/CryEngine/CrySystem/MTSafeAllocator.h
-108
View File
@@ -1,108 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#pragma once
#if defined(LINUX)
# include "Linux_Win32Wrapper.h"
#endif
#include <ISystem.h>
////////////////////////////////////////////////////////////////////////////////
#if defined(AZ_RESTRICTED_PLATFORM)
#include AZ_RESTRICTED_FILE(MTSafeAllocator_h)
#endif
#if defined(AZ_RESTRICTED_SECTION_IMPLEMENTED)
#undef AZ_RESTRICTED_SECTION_IMPLEMENTED
#elif defined(MOBILE) // IOS/Android
# define MTSAFE_DEFAULT_ALIGNMENT 8
# define MTSAFE_GENERAL_HEAP_SIZE ((1U << 20) + (1U << 19))
#elif defined(WIN32) || defined(WIN64) || defined(LINUX) || defined(MAC)
# define MTSAFE_GENERAL_HEAP_SIZE (12U << 20)
# define MTSAFE_DEFAULT_ALIGNMENT 8
#else
# error Unknown target platform
#endif
class CMTSafeHeap
{
public:
// Constructor
CMTSafeHeap();
// Destructor
~CMTSafeHeap();
// Performs a persisistent (in other words, non-temporary) allocation.
void* PersistentAlloc(size_t nSize);
// Retrieves system memory allocation size for any call to PersistentAlloc.
// Required to not count virtual memory usage inside CrySizer
size_t PersistentAllocSize(size_t nSize);
// Frees memory allocation
void FreePersistent(void* p);
// Perform a allocation that is considered temporary and will be handled by
// the pool itself.
// Note: It is important that these temporary allocations are actually
// temporary and do not persist for a long persiod of time.
void* TempAlloc (size_t nSize, const char* szDbgSource, uint32 align = 0)
{
bool bFallbackToMalloc = true;
return TempAlloc(nSize, szDbgSource, bFallbackToMalloc, align);
}
void* TempAlloc (size_t nSize, const char* szDbgSource, bool& bFallBackToMalloc, uint32 align = 0);
bool IsInGeneralHeap(const void* p)
{
return m_pGeneralHeapStorage <= p && p < m_pGeneralHeapStorageEnd;
}
// Free a temporary allocaton.
void FreeTemporary(void* p);
// The number of live allocations allocation within the temporary pool
size_t NumAllocations() const { return m_LiveTempAllocations; }
// The memory usage of the mtsafe allocator
void GetMemoryUsage(ICrySizer* pSizer);
// zlib-compatible stubs
static void* StaticAlloc (void* pOpaque, unsigned nItems, unsigned nSize);
static void StaticFree (void* pOpaque, void* pAddress);
// Dump some statistics to the cry log
void PrintStats();
private:
friend class CSystem;
IGeneralMemoryHeap* m_pGeneralHeap;
char* m_pGeneralHeapStorage;
char* m_pGeneralHeapStorageEnd;
// The number of temporary allocations currently active within the pool
size_t m_LiveTempAllocations;
// The total number of allocations performed in the pool
size_t m_TotalAllocations;
// The total bytes that weren't temporarily allocated
size_t m_TempAllocationsFailed;
// The total number of temporary allocations that fell back to global system memory
LARGE_INTEGER m_TempAllocationsTime;
};
Code/CryEngine/CrySystem/MemoryAddressRange.cpp
-145
View File
@@ -1,145 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySystem_precompiled.h"
#include "MemoryAddressRange.h"
#include "System.h"
#if defined(APPLE) || defined(LINUX)
#include <sys/mman.h>
#endif
CMemoryAddressRange::CMemoryAddressRange(char* pBaseAddress, size_t nPageSize, size_t nPageCount, [[maybe_unused]] const char* sName)
: m_pBaseAddress(pBaseAddress)
, m_nPageSize(nPageSize)
, m_nPageCount(nPageCount)
{
}
void CMemoryAddressRange::Release()
{
delete this;
}
char* CMemoryAddressRange::GetBaseAddress() const
{
return m_pBaseAddress;
}
size_t CMemoryAddressRange::GetPageCount() const
{
return m_nPageCount;
}
size_t CMemoryAddressRange::GetPageSize() const
{
return m_nPageSize;
}
#if AZ_LEGACY_CRYSYSTEM_TRAIT_MEMADDRESSRANGE_WINDOWS_STYLE
void* CMemoryAddressRange::ReserveSpace(size_t capacity)
{
return VirtualAlloc(NULL, capacity, MEM_RESERVE, PAGE_READWRITE);
}
size_t CMemoryAddressRange::GetSystemPageSize()
{
SYSTEM_INFO si;
GetSystemInfo(&si);
return si.dwPageSize;
}
CMemoryAddressRange::CMemoryAddressRange(size_t capacity, [[maybe_unused]] const char* name)
{
m_nPageSize = GetSystemPageSize();
size_t algnCap = Align(capacity, m_nPageSize);
m_pBaseAddress = (char*)ReserveSpace(algnCap);
m_nPageCount = algnCap / m_nPageSize;
}
CMemoryAddressRange::~CMemoryAddressRange()
{
VirtualFree(m_pBaseAddress, 0, MEM_RELEASE);
}
void* CMemoryAddressRange::MapPage(size_t pageIdx)
{
void* pRet = VirtualAlloc(m_pBaseAddress + pageIdx * m_nPageSize, m_nPageSize, MEM_COMMIT, PAGE_READWRITE);
return pRet;
}
void CMemoryAddressRange::UnmapPage(size_t pageIdx)
{
char* pBase = m_pBaseAddress + pageIdx * m_nPageSize;
// Disable warning about only decommitting pages, and not releasing them
VirtualFree(pBase, m_nPageSize, MEM_DECOMMIT);
}
#elif defined(AZ_RESTRICTED_PLATFORM)
#include AZ_RESTRICTED_FILE(MemoryAddressRange_cpp)
#elif defined(APPLE) || defined(LINUX)
void* CMemoryAddressRange::ReserveSpace(size_t capacity)
{
return mmap(0, capacity, PROT_NONE, MAP_ANON | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
}
size_t CMemoryAddressRange::GetSystemPageSize()
{
return sysconf(_SC_PAGESIZE);
}
CMemoryAddressRange::CMemoryAddressRange(size_t capacity, const char* name)
{
m_nPageSize = GetSystemPageSize();
m_allocatedSpace = Align(capacity, m_nPageSize);
m_pBaseAddress = (char*)ReserveSpace(m_allocatedSpace);
assert(m_pBaseAddress != MAP_FAILED);
m_nPageCount = m_allocatedSpace / m_nPageSize;
}
CMemoryAddressRange::~CMemoryAddressRange()
{
int ret = munmap(m_pBaseAddress, m_allocatedSpace);
(void) ret;
assert(ret == 0);
}
void* CMemoryAddressRange::MapPage(size_t pageIdx)
{
// There is no equivalent to this function with mmap, this
// happens automatically in the OS. We just return the
// correct address.
void* pRet = NULL;
if (0 == mprotect(m_pBaseAddress + (pageIdx * m_nPageSize), m_nPageSize, PROT_READ | PROT_WRITE))
{
pRet = m_pBaseAddress + (pageIdx * m_nPageSize);
}
return pRet;
}
void CMemoryAddressRange::UnmapPage(size_t pageIdx)
{
char* pBase = m_pBaseAddress + pageIdx * m_nPageSize;
int ret = mprotect(pBase, m_nPageSize, PROT_NONE);
(void) ret;
assert(ret == 0);
}
#endif
Code/CryEngine/CrySystem/MemoryAddressRange.h
-61
View File
@@ -1,61 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_CRYSYSTEM_MEMORYADDRESSRANGE_H
#define CRYINCLUDE_CRYSYSTEM_MEMORYADDRESSRANGE_H
#pragma once
#include "IMemory.h"
class CMemoryAddressRange
: public IMemoryAddressRange
{
public:
static void* ReserveSpace(size_t sz);
static size_t GetSystemPageSize();
public:
CMemoryAddressRange(char* pBaseAddress, size_t nPageSize, size_t nPageCount, const char* sName);
CMemoryAddressRange(size_t capacity, const char* name);
~CMemoryAddressRange();
ILINE bool IsInRange(void* p) const
{
return m_pBaseAddress <= p && p < (m_pBaseAddress + m_nPageSize * m_nPageCount);
}
public:
void Release();
char* GetBaseAddress() const;
size_t GetPageCount() const;
size_t GetPageSize() const;
void* MapPage(size_t pageIdx);
void UnmapPage(size_t pageIdx);
private:
CMemoryAddressRange(const CMemoryAddressRange&);
CMemoryAddressRange& operator = (const CMemoryAddressRange&);
private:
char* m_pBaseAddress;
size_t m_nPageSize;
size_t m_nPageCount;
#if defined(APPLE) || defined(LINUX)
size_t m_allocatedSpace; // Required to unmap latter on
#endif
};
#endif // CRYINCLUDE_CRYSYSTEM_MEMORYADDRESSRANGE_H
Code/CryEngine/CrySystem/MemoryManager.cpp
-226
View File
@@ -1,226 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySystem_precompiled.h"
#include "MemoryManager.h"
#include "platform.h"
#include "CustomMemoryHeap.h"
#include "GeneralMemoryHeap.h"
#include "PageMappingHeap.h"
#if defined(AZ_RESTRICTED_PLATFORM)
#undef AZ_RESTRICTED_SECTION
#define MEMORYMANAGER_CPP_SECTION_1 1
#endif
#if defined(WIN32)
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <Psapi.h>
#endif
#if defined(APPLE)
#include <mach/mach.h> // task_info
#endif
#if defined(APPLE) || defined(LINUX)
#include <sys/types.h> // required by mman.h
#include <sys/mman.h> //mmap - virtual memory manager
#endif
#ifdef MEMMAN_STATIC
CCryMemoryManager g_memoryManager;
#endif
//////////////////////////////////////////////////////////////////////////
CCryMemoryManager* CCryMemoryManager::GetInstance()
{
#ifdef MEMMAN_STATIC
return &g_memoryManager;
#else
static CCryMemoryManager memman;
return &memman;
#endif
}
//////////////////////////////////////////////////////////////////////////
bool CCryMemoryManager::GetProcessMemInfo(SProcessMemInfo& minfo)
{
ZeroStruct(minfo);
#if defined(WIN32)
MEMORYSTATUSEX mem;
mem.dwLength = sizeof(mem);
GlobalMemoryStatusEx (&mem);
minfo.TotalPhysicalMemory = mem.ullTotalPhys;
minfo.FreePhysicalMemory = mem.ullAvailPhys;
//////////////////////////////////////////////////////////////////////////
typedef BOOL (WINAPI * GetProcessMemoryInfoProc)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD);
PROCESS_MEMORY_COUNTERS pc;
ZeroStruct(pc);
pc.cb = sizeof(pc);
static HMODULE hPSAPI = LoadLibraryA("psapi.dll");
if (hPSAPI)
{
static GetProcessMemoryInfoProc pGetProcessMemoryInfo = (GetProcessMemoryInfoProc)GetProcAddress(hPSAPI, "GetProcessMemoryInfo");
if (pGetProcessMemoryInfo)
{
if (pGetProcessMemoryInfo(GetCurrentProcess(), &pc, sizeof(pc)))
{
minfo.PageFaultCount = pc.PageFaultCount;
minfo.PeakWorkingSetSize = pc.PeakWorkingSetSize;
minfo.WorkingSetSize = pc.WorkingSetSize;
minfo.QuotaPeakPagedPoolUsage = pc.QuotaPeakPagedPoolUsage;
minfo.QuotaPagedPoolUsage = pc.QuotaPagedPoolUsage;
minfo.QuotaPeakNonPagedPoolUsage = pc.QuotaPeakNonPagedPoolUsage;
minfo.QuotaNonPagedPoolUsage = pc.QuotaNonPagedPoolUsage;
minfo.PagefileUsage = pc.PagefileUsage;
minfo.PeakPagefileUsage = pc.PeakPagefileUsage;
return true;
}
}
}
return false;
#else
#define AZ_RESTRICTED_SECTION_IMPLEMENTED
#if defined(AZ_RESTRICTED_PLATFORM)
#define AZ_RESTRICTED_SECTION MEMORYMANAGER_CPP_SECTION_1
#include AZ_RESTRICTED_FILE(MemoryManager_cpp)
#endif
bool retVal = true;
#if defined(AZ_RESTRICTED_SECTION_IMPLEMENTED)
#undef AZ_RESTRICTED_SECTION_IMPLEMENTED
#elif defined(LINUX)
MEMORYSTATUS MemoryStatus;
GlobalMemoryStatus(&MemoryStatus);
minfo.PagefileUsage = minfo.PeakPagefileUsage = MemoryStatus.dwTotalPhys - MemoryStatus.dwAvailPhys;
minfo.FreePhysicalMemory = MemoryStatus.dwAvailPhys;
minfo.TotalPhysicalMemory = MemoryStatus.dwTotalPhys;
#if defined(ANDROID)
// On Android, mallinfo() is an EXTREMELY time consuming operation. Nearly 80% CPU time will be spent
// on this operation once -memreplay is given. Since WorkingSetSize is only used for statistics and
// debugging purpose, it's simply ignored.
minfo.WorkingSetSize = 0;
#else
struct mallinfo meminfo = mallinfo();
minfo.WorkingSetSize = meminfo.usmblks + meminfo.uordblks;
#endif
#elif defined(APPLE)
MEMORYSTATUS MemoryStatus;
GlobalMemoryStatus(&MemoryStatus);
minfo.PagefileUsage = minfo.PeakPagefileUsage = MemoryStatus.dwTotalPhys - MemoryStatus.dwAvailPhys;
minfo.FreePhysicalMemory = MemoryStatus.dwAvailPhys;
minfo.TotalPhysicalMemory = MemoryStatus.dwTotalPhys;
// Retrieve WorkingSetSize from task_info
task_basic_info kTaskInfo;
mach_msg_type_number_t uInfoCount(sizeof(kTaskInfo) / sizeof(natural_t));
if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&kTaskInfo, &uInfoCount) != 0)
{
gEnv->pLog->LogError("task_info failed\n");
return false;
}
minfo.WorkingSetSize = kTaskInfo.resident_size;
#else
retVal = false;
#endif
return retVal;
#endif
}
//////////////////////////////////////////////////////////////////////////
CCryMemoryManager::HeapHandle CCryMemoryManager::TraceDefineHeap([[maybe_unused]] const char* heapName, [[maybe_unused]] size_t size, [[maybe_unused]] const void* pBase)
{
return 0;
}
//////////////////////////////////////////////////////////////////////////
void CCryMemoryManager::TraceHeapAlloc([[maybe_unused]] HeapHandle heap, [[maybe_unused]] void* mem, [[maybe_unused]] size_t size, [[maybe_unused]] size_t blockSize, [[maybe_unused]] const char* sUsage, [[maybe_unused]] const char* sNameHint)
{
}
//////////////////////////////////////////////////////////////////////////
void CCryMemoryManager::TraceHeapFree([[maybe_unused]] HeapHandle heap, [[maybe_unused]] void* mem, [[maybe_unused]] size_t blockSize)
{
}
//////////////////////////////////////////////////////////////////////////
void CCryMemoryManager::TraceHeapSetColor([[maybe_unused]] uint32 color)
{
}
//////////////////////////////////////////////////////////////////////////
void CCryMemoryManager::TraceHeapSetLabel([[maybe_unused]] const char* sLabel)
{
}
//////////////////////////////////////////////////////////////////////////
uint32 CCryMemoryManager::TraceHeapGetColor()
{
return 0;
}
//////////////////////////////////////////////////////////////////////////
ICustomMemoryHeap* const CCryMemoryManager::CreateCustomMemoryHeapInstance(IMemoryManager::EAllocPolicy const eAllocPolicy)
{
return new CCustomMemoryHeap(eAllocPolicy);
}
IGeneralMemoryHeap* CCryMemoryManager::CreateGeneralExpandingMemoryHeap(size_t upperLimit, size_t reserveSize, const char* sUsage)
{
return new CGeneralMemoryHeap(static_cast<UINT_PTR>(0), upperLimit, reserveSize, sUsage);
}
IGeneralMemoryHeap* CCryMemoryManager::CreateGeneralMemoryHeap(void* base, size_t sz, const char* sUsage)
{
return new CGeneralMemoryHeap(base, sz, sUsage);
}
IMemoryAddressRange* CCryMemoryManager::ReserveAddressRange(size_t capacity, const char* sName)
{
return new CMemoryAddressRange(capacity, sName);
}
IPageMappingHeap* CCryMemoryManager::CreatePageMappingHeap(size_t addressSpace, const char* sName)
{
return new CPageMappingHeap(addressSpace, sName);
}
extern "C"
{
CRYMEMORYMANAGER_API void CryGetIMemoryManagerInterface(void** pIMemoryManager)
{
// Static instance of the memory manager
*pIMemoryManager = CCryMemoryManager::GetInstance();
}
};
Code/CryEngine/CrySystem/MemoryManager.h
-52
View File
@@ -1,52 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_CRYSYSTEM_MEMORYMANAGER_H
#define CRYINCLUDE_CRYSYSTEM_MEMORYMANAGER_H
#pragma once
#include "ISystem.h"
//////////////////////////////////////////////////////////////////////////
// Class that implements IMemoryManager interface.
//////////////////////////////////////////////////////////////////////////
#ifndef MEMMAN_STATIC
class CCryMemoryManager
: public IMemoryManager
{
public:
// Singleton
static CCryMemoryManager* GetInstance();
//////////////////////////////////////////////////////////////////////////
virtual bool GetProcessMemInfo(SProcessMemInfo& minfo);
virtual HeapHandle TraceDefineHeap(const char* heapName, size_t size, const void* pBase);
virtual void TraceHeapAlloc(HeapHandle heap, void* mem, size_t size, size_t blockSize, const char* sUsage, const char* sNameHint = 0);
virtual void TraceHeapFree(HeapHandle heap, void* mem, size_t blockSize);
virtual void TraceHeapSetColor(uint32 color);
virtual uint32 TraceHeapGetColor();
virtual void TraceHeapSetLabel(const char* sLabel);
virtual ICustomMemoryHeap* const CreateCustomMemoryHeapInstance(IMemoryManager::EAllocPolicy const eAllocPolicy);
virtual IGeneralMemoryHeap* CreateGeneralExpandingMemoryHeap(size_t upperLimit, size_t reserveSize, const char* sUsage);
virtual IGeneralMemoryHeap* CreateGeneralMemoryHeap(void* base, size_t sz, const char* sUsage);
virtual IMemoryAddressRange* ReserveAddressRange(size_t capacity, const char* sName);
virtual IPageMappingHeap* CreatePageMappingHeap(size_t addressSpace, const char* sName);
};
#else
typedef IMemoryManager CCryMemoryManager;
#endif
#endif // CRYINCLUDE_CRYSYSTEM_MEMORYMANAGER_H
Code/CryEngine/CrySystem/PageMappingHeap.cpp
-258
View File
@@ -1,258 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySystem_precompiled.h"
#include "PageMappingHeap.h"
namespace
{
template <typename Func>
inline void FindZeroRanges(const uint32* str, size_t strLen, Func& yield)
{
size_t carry = 0;
size_t bitIdx = 0;
for (size_t wordIdx = 0; wordIdx < strLen; ++wordIdx)
{
size_t wordBitIdx = 0;
int64 word = str[wordIdx];
// Set up sign extension to insert bits that are the last bit, inverted.
if (!(word & 0x80000000))
{
reinterpret_cast<uint64&>(word) |= 0xffffffff00000000ULL;
}
do
{
size_t wordZeroRunLen = countTrailingZeroes(word);
wordBitIdx += wordZeroRunLen;
carry += wordZeroRunLen;
if (wordBitIdx == 32)
{
break;
}
yield(bitIdx, carry);
word >>= wordZeroRunLen;
bitIdx += carry;
carry = 0;
size_t wordOneRunLen = countTrailingZeroes(~word);
bitIdx += wordOneRunLen;
wordBitIdx += wordOneRunLen;
if (wordBitIdx == 32)
{
break;
}
word >>= wordOneRunLen;
}
while (true);
}
if (carry)
{
yield(bitIdx, carry);
}
}
struct DLMMapFindBest
{
DLMMapFindBest(size_t size)
: requiredLength(size)
, bestPosition(-1)
, bestFragmentLength(INT_MAX)
{
}
bool operator () (size_t position, size_t length)
{
if (length == requiredLength)
{
bestPosition = position;
bestFragmentLength = 0;
return false;
}
else if (length > requiredLength)
{
size_t fragment = length - requiredLength;
if (fragment < bestFragmentLength)
{
bestPosition = position;
bestFragmentLength = fragment;
}
}
return true;
}
size_t requiredLength;
ptrdiff_t bestPosition;
size_t bestFragmentLength;
};
struct FindLargest
{
FindLargest()
: largest(0)
{
}
bool operator () (size_t, size_t length)
{
largest = max(largest, length);
return true;
}
size_t largest;
};
}
CPageMappingHeap::CPageMappingHeap(char* pAddressSpace, size_t nNumPages, size_t nPageSize, const char* sName)
: m_addrRange(pAddressSpace, nPageSize, nNumPages, sName)
{
Init();
}
CPageMappingHeap::CPageMappingHeap(size_t addressSpace, const char* sName)
: m_addrRange(addressSpace, sName)
{
Init();
}
CPageMappingHeap::~CPageMappingHeap()
{
}
void CPageMappingHeap::Release()
{
delete this;
}
size_t CPageMappingHeap::GetGranularity() const
{
return m_addrRange.GetPageSize();
}
bool CPageMappingHeap::IsInAddressRange(void* ptr) const
{
return m_addrRange.IsInRange(ptr);
}
size_t CPageMappingHeap::FindLargestFreeBlockSize() const
{
CryAutoLock<CryCriticalSectionNonRecursive> lock(m_lock);
const size_t pageSize = m_addrRange.GetPageSize();
FindLargest findLargest;
FindZeroRanges(&m_pageBitmap[0], m_pageBitmap.size(), findLargest);
return findLargest.largest * pageSize;
}
void* CPageMappingHeap::Map(size_t length)
{
CryAutoLock<CryCriticalSectionNonRecursive> lock(m_lock);
const size_t pageBitmapElemBitSize = (sizeof(uint32) * 8);
const size_t pageSize = m_addrRange.GetPageSize();
const size_t numPages = m_addrRange.GetPageCount();
if (length % pageSize)
{
__debugbreak();
length = (length + (pageSize - 1)) & ~(pageSize - 1);
}
DLMMapFindBest findBest(length / pageSize);
FindZeroRanges(&m_pageBitmap[0], m_pageBitmap.size(), findBest);
if ((findBest.bestPosition == -1) || (findBest.bestPosition >= (int)numPages))
{
return NULL;
}
void* mapAddress = m_addrRange.GetBaseAddress() + pageSize * findBest.bestPosition;
for (size_t pageIdx = findBest.bestPosition, pageIdxEnd = pageIdx + length / pageSize; pageIdx != pageIdxEnd; ++pageIdx)
{
if (!m_addrRange.MapPage(pageIdx))
{
// Unwind the pages we've already mapped.
for (; pageIdx > static_cast<size_t>(findBest.bestPosition); --pageIdx)
{
m_addrRange.UnmapPage(pageIdx - 1);
}
return NULL;
}
}
for (size_t pageIdx = findBest.bestPosition, pageIdxEnd = pageIdx + length / pageSize; pageIdx != pageIdxEnd; ++pageIdx)
{
size_t pageSegment = pageIdx / pageBitmapElemBitSize;
uint32 pageMask = 1U << static_cast<uint32>(pageIdx % pageBitmapElemBitSize);
m_pageBitmap[pageSegment] |= pageMask;
}
return mapAddress;
}
void CPageMappingHeap::Unmap(void* mem, size_t length)
{
CryAutoLock<CryCriticalSectionNonRecursive> lock(m_lock);
const size_t pageSize = m_addrRange.GetPageSize();
if (length % pageSize)
{
__debugbreak();
length = (length + (pageSize - 1)) & ~(pageSize - 1);
}
char* mapAddress = reinterpret_cast<char*>(mem);
for (size_t pageIdx = (mapAddress - m_addrRange.GetBaseAddress()) / pageSize, pageIdxEnd = pageIdx + length / pageSize; pageIdx != pageIdxEnd; ++pageIdx)
{
m_addrRange.UnmapPage(pageIdx);
const size_t pageBitmapElemBitSize = (sizeof(uint32) * 8);
size_t pageSegment = pageIdx / pageBitmapElemBitSize;
uint32 pageMask = ~(1U << static_cast<uint32>(pageIdx % pageBitmapElemBitSize));
m_pageBitmap[pageSegment] &= pageMask;
}
}
void CPageMappingHeap::Init()
{
UINT_PTR start = (UINT_PTR)m_addrRange.GetBaseAddress();
UINT_PTR end = start + m_addrRange.GetPageCount() * m_addrRange.GetPageSize();
size_t addressSpace = end - start;
size_t pageSize = m_addrRange.GetPageSize();
size_t numPages = (addressSpace + pageSize - 1) / pageSize;
m_pageBitmap.resize((numPages + 31) / 32);
size_t pageCapacity = m_pageBitmap.size() * 32;
size_t numUnavailablePages = pageCapacity - numPages;
if (numUnavailablePages > 0)
{
m_pageBitmap.back() = ~((1 << (32 - numUnavailablePages)) - 1);
}
}
Code/CryEngine/CrySystem/PageMappingHeap.h
-55
View File
@@ -1,55 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_CRYSYSTEM_PAGEMAPPINGHEAP_H
#define CRYINCLUDE_CRYSYSTEM_PAGEMAPPINGHEAP_H
#pragma once
#include "MemoryAddressRange.h"
#include "IMemory.h"
class CPageMappingHeap
: public IPageMappingHeap
{
public:
CPageMappingHeap(char* pAddressSpace, size_t nNumPages, size_t nPageSize, const char* sName);
CPageMappingHeap(size_t addressSpace, const char* sName);
~CPageMappingHeap();
public: // IPageMappingHeap Members
virtual void Release();
virtual size_t GetGranularity() const;
virtual bool IsInAddressRange(void* ptr) const;
virtual size_t FindLargestFreeBlockSize() const;
virtual void* Map(size_t sz);
virtual void Unmap(void* ptr, size_t sz);
private:
CPageMappingHeap(const CPageMappingHeap&);
CPageMappingHeap& operator = (const CPageMappingHeap&);
private:
void Init();
private:
mutable CryCriticalSectionNonRecursive m_lock;
CMemoryAddressRange m_addrRange;
std::vector<uint32> m_pageBitmap;
};
#endif // CRYINCLUDE_CRYSYSTEM_PAGEMAPPINGHEAP_H
Code/CryEngine/CrySystem/ResourceManager.cpp
-868
View File
@@ -1,868 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Interface to the Resource Manager
#include "CrySystem_precompiled.h"
#include "ResourceManager.h"
#include "System.h"
#include "MaterialUtils.h"
#include <CryPath.h>
#include <AzFramework/API/ApplicationAPI.h>
#include <AzFramework/IO/FileOperations.h>
#include <AzFramework/Archive/Archive.h>
#include <AzFramework/Archive/INestedArchive.h>
#include <Pak/CryPakUtils.h>
#if defined(AZ_RESTRICTED_PLATFORM)
#undef AZ_RESTRICTED_SECTION
#define RESOURCEMANAGER_CPP_SECTION_1 1
#define RESOURCEMANAGER_CPP_SECTION_2 2
#endif
#if defined(AZ_RESTRICTED_PLATFORM)
#define AZ_RESTRICTED_SECTION RESOURCEMANAGER_CPP_SECTION_1
#include AZ_RESTRICTED_FILE(ResourceManager_cpp)
#endif
#define LEVEL_PAK_FILENAME "level.pak"
#define LEVEL_PAK_INMEMORY_MAXSIZE 10 * 1024 * 1024
#define ENGINE_PAK_FILENAME "engine.pak"
#define LEVEL_CACHE_PAK_FILENAME "xml.pak"
#define GAME_DATA_PAK_FILENAME "gamedata.pak"
#define FAST_LOADING_PAKS_SRC_FOLDER "_fastload/"
#define FRONTEND_COMMON_PAK_FILENAME_SP "modes/menucommon_sp.pak"
#define FRONTEND_COMMON_PAK_FILENAME_MP "modes/menucommon_mp.pak"
#define FRONTEND_COMMON_LIST_FILENAME "menucommon"
#define LEVEL_CACHE_SRC_FOLDER "_levelcache/"
#define LEVEL_CACHE_BIND_ROOT "LevelCache"
#define LEVEL_RESOURCE_LIST "resourcelist.txt"
#define AUTO_LEVEL_RESOURCE_LIST "auto_resourcelist.txt"
#define AUTO_LEVEL_SEQUENCE_RESOURCE_LIST "auto_resources_sequence.txt"
#define AUTO_LEVEL_TOTAL_RESOURCE_LIST "auto_resourcelist_total.txt"
#define AUTO_LEVEL_TOTAL_SEQUENCE_RESOURCE_LIST "auto_resources_total_sequence.txt"
//////////////////////////////////////////////////////////////////////////
// IResourceList implementation class.
//////////////////////////////////////////////////////////////////////////
class CLevelResourceList
: public AZ::IO::IResourceList
{
public:
CLevelResourceList()
{
m_pFileBuffer = 0;
m_nBufferSize = 0;
m_nCurrentLine = 0;
};
~CLevelResourceList()
{
Clear();
};
uint32 GetFilenameHash(const char* sResourceFile)
{
char filename[512];
azstrcpy(filename, AZ_ARRAY_SIZE(filename), sResourceFile);
MaterialUtils::UnifyMaterialName(filename);
uint32 code = CCrc32::ComputeLowercase(filename);
return code;
}
virtual void Add([[maybe_unused]] AZStd::string_view sResourceFile)
{
assert(0); // Not implemented.
}
virtual void Clear()
{
delete [] m_pFileBuffer;
m_pFileBuffer = 0;
m_nBufferSize = 0;
stl::free_container(m_lines);
stl::free_container(m_resources_crc32);
m_nCurrentLine = 0;
}
struct ComparePredicate
{
bool operator()(const char* s1, const char* s2)
{
return strcmp(s1, s2) < 0;
}
};
virtual bool IsExist(AZStd::string_view sResourceFile)
{
uint32 nHash = GetFilenameHash(sResourceFile.data());
if (stl::binary_find(m_resources_crc32.begin(), m_resources_crc32.end(), nHash) != m_resources_crc32.end())
{
return true;
}
return false;
}
virtual bool Load(AZStd::string_view sResourceListFilename)
{
Clear();
CCryFile file;
if (file.Open(sResourceListFilename.data(), "rb", AZ::IO::IArchive::FOPEN_ONDISK)) // File access can happen from disk as well.
{
m_nBufferSize = file.GetLength();
if (m_nBufferSize > 0)
{
m_pFileBuffer = new char[m_nBufferSize];
size_t numBytesRead = file.ReadRaw(m_pFileBuffer, file.GetLength());
if (numBytesRead <= 0 || numBytesRead != file.GetLength())
{
AZ_Error("ResourceManager", false, "Unable to read data for: %.*s", aznumeric_cast<int>(sResourceListFilename.size()), sResourceListFilename.data());
return false;
}
m_pFileBuffer[m_nBufferSize - 1] = 0;
char seps[] = "\r\n";
m_lines.reserve(5000);
// Parse file, every line in a file represents a resource filename.
char* nextToken = nullptr;
char* token = azstrtok(m_pFileBuffer, 0, seps, &nextToken);
while (token != NULL)
{
m_lines.push_back(token);
token = azstrtok(NULL, 0, seps, &nextToken);
}
m_resources_crc32.resize(m_lines.size());
for (int i = 0, numlines = m_lines.size(); i < numlines; i++)
{
MaterialUtils::UnifyMaterialName(const_cast<char*>(m_lines[i]));
m_resources_crc32[i] = CCrc32::ComputeLowercase(m_lines[i]);
}
std::sort(m_resources_crc32.begin(), m_resources_crc32.end());
}
return true;
}
return false;
}
virtual const char* GetFirst()
{
m_nCurrentLine = 0;
if (!m_lines.empty())
{
return m_lines[0];
}
return NULL;
}
virtual const char* GetNext()
{
m_nCurrentLine++;
if (m_nCurrentLine < (int)m_lines.size())
{
return m_lines[m_nCurrentLine];
}
return NULL;
}
void GetMemoryStatistics(ICrySizer* pSizer)
{
pSizer->Add(this, sizeof(*this));
pSizer->Add(m_pFileBuffer, m_nBufferSize);
pSizer->AddContainer(m_lines);
pSizer->AddContainer(m_resources_crc32);
}
public:
char* m_pFileBuffer;
int m_nBufferSize;
typedef std::vector<const char*> Lines;
Lines m_lines;
int m_nCurrentLine;
std::vector<uint32> m_resources_crc32;
};
//////////////////////////////////////////////////////////////////////////
CResourceManager::CResourceManager()
{
m_bRegisteredFileOpenSink = false;
m_bOwnResourceList = false;
m_bLevelTransitioning = false;
m_fastLoadPakPaths.reserve(8);
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::PrepareLevel(const char* sLevelFolder, const char* sLevelName)
{
LOADING_TIME_PROFILE_SECTION;
m_sLevelFolder = sLevelFolder;
m_sLevelName = sLevelName;
m_bLevelTransitioning = false;
m_currentLevelCacheFolder = CryPathString(LEVEL_CACHE_SRC_FOLDER) + sLevelName;
if (g_cvars.archiveVars.nLoadCache)
{
bool usePrefabSystemForLevels = false;
AzFramework::ApplicationRequests::Bus::BroadcastResult(
usePrefabSystemForLevels, &AzFramework::ApplicationRequests::IsPrefabSystemForLevelsEnabled);
// The prefab system doesn't use level.pak
if (!usePrefabSystemForLevels)
{
CryPathString levelpak = PathUtil::Make(sLevelFolder, LEVEL_PAK_FILENAME);
size_t nPakFileSize = gEnv->pCryPak->FGetSize(levelpak.c_str());
if (nPakFileSize < LEVEL_PAK_INMEMORY_MAXSIZE) // 10 megs.
{
// Force level.pak from this level in memory.
gEnv->pCryPak->LoadPakToMemory(LEVEL_PAK_FILENAME, AZ::IO::IArchive::eInMemoryPakLocale_GPU);
}
}
gEnv->pCryPak->LoadPakToMemory(ENGINE_PAK_FILENAME, AZ::IO::IArchive::eInMemoryPakLocale_GPU);
//
// Load _levelCache paks in the order they are stored on the disk - reduce seek time
//
if (gEnv->pConsole->GetCVar("e_StreamCgf") && gEnv->pConsole->GetCVar("e_StreamCgf")->GetIVal() != 0)
{
LoadLevelCachePak("cga.pak", "", true);
LoadLevelCachePak("cgf.pak", "", true);
if (g_cvars.archiveVars.nStreamCache)
{
LoadLevelCachePak("cgf_cache.pak", "", false);
}
}
LoadLevelCachePak("chr.pak", "", true);
if (g_cvars.archiveVars.nStreamCache)
{
LoadLevelCachePak("chr_cache.pak", "", false);
}
LoadLevelCachePak("dds0.pak", "", true);
if (g_cvars.archiveVars.nStreamCache)
{
LoadLevelCachePak("dds_cache.pak", "", false);
}
LoadLevelCachePak("skin.pak", "", true);
if (g_cvars.archiveVars.nStreamCache)
{
LoadLevelCachePak("skin_cache.pak", "", false);
}
LoadLevelCachePak(LEVEL_CACHE_PAK_FILENAME, "", true);
}
AZStd::intrusive_ptr<CLevelResourceList> pResList = new CLevelResourceList;
gEnv->pCryPak->SetResourceList(AZ::IO::IArchive::RFOM_Level, pResList.get());
m_bOwnResourceList = true;
// Load resourcelist.txt, TODO: make sure there are no duplicates
if (g_cvars.archiveVars.nSaveLevelResourceList == 0)
{
string filename = PathUtil::Make(sLevelFolder, AUTO_LEVEL_RESOURCE_LIST);
if (!pResList->Load(filename.c_str())) // If we saving resource list do not use auto_resourcelist.txt
{
// Try resource list created by the editor.
filename = PathUtil::Make(sLevelFolder, LEVEL_RESOURCE_LIST);
pResList->Load(filename.c_str());
}
}
//LoadFastLoadPaks();
if (g_cvars.archiveVars.nStreamCache)
{
m_AsyncPakManager.ParseLayerPaks(GetCurrentLevelCacheFolder());
}
}
//////////////////////////////////////////////////////////////////////////
bool CResourceManager::LoadFastLoadPaks(bool bToMemory)
{
if (g_cvars.archiveVars.nSaveFastloadResourceList != 0)
{
// Record a file list for _FastLoad/startup.pak
m_recordedFiles.clear();
gEnv->pCryPak->RegisterFileAccessSink(this);
m_bRegisteredFileOpenSink = true;
return false;
}
else
{
LOADING_TIME_PROFILE_SECTION;
// Load a special _fastload paks
int nPakPreloadFlags = AZ::IO::IArchive::FLAGS_FILENAMES_AS_CRC32 | AZ::IO::INestedArchive::FLAGS_OVERRIDE_PAK;
if (bToMemory && g_cvars.archiveVars.nLoadCache)
{
nPakPreloadFlags |= AZ::IO::IArchive::FLAGS_PAK_IN_MEMORY;
}
const char* const assetsDir = "@assets@";
#if defined(AZ_RESTRICTED_PLATFORM)
#define AZ_RESTRICTED_SECTION RESOURCEMANAGER_CPP_SECTION_2
#include AZ_RESTRICTED_FILE(ResourceManager_cpp)
#endif
gEnv->pCryPak->OpenPacks(assetsDir, AZ::IO::PathString(FAST_LOADING_PAKS_SRC_FOLDER) + "*.pak", nPakPreloadFlags, &m_fastLoadPakPaths);
gEnv->pCryPak->OpenPack(assetsDir, "Engine.pak", AZ::IO::IArchive::FLAGS_PAK_IN_MEMORY);
return !m_fastLoadPakPaths.empty();
}
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::UnloadFastLoadPaks()
{
for (uint32 i = 0; i < m_fastLoadPakPaths.size(); i++)
{
// Unload a special _fastload paks
gEnv->pCryPak->ClosePack(m_fastLoadPakPaths[i].c_str(), AZ::IO::IArchive::FLAGS_PATH_REAL);
}
m_fastLoadPakPaths.clear();
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::UnloadLevel()
{
gEnv->pCryPak->SetResourceList(AZ::IO::IArchive::RFOM_Level, NULL);
if (m_bRegisteredFileOpenSink)
{
if (g_cvars.archiveVars.nSaveTotalResourceList)
{
SaveRecordedResources(true);
m_recordedFiles.clear();
}
}
stl::free_container(m_sLevelFolder);
stl::free_container(m_sLevelName);
stl::free_container(m_currentLevelCacheFolder);
// should always be empty, since it is freed at the end of
// the level loading process, if it is not
// something went wrong and we have a levelheap leak
assert(m_openedPaks.capacity() == 0);
m_pSequenceResourceList = NULL;
}
//////////////////////////////////////////////////////////////////////////
AZ::IO::IResourceList* CResourceManager::GetLevelResourceList()
{
auto pResList = gEnv->pCryPak->GetResourceList(AZ::IO::IArchive::RFOM_Level);
return pResList;
}
//////////////////////////////////////////////////////////////////////////
bool CResourceManager::LoadLevelCachePak(const char* sPakName, const char* sBindRoot, bool bOnlyDuringLevelLoading)
{
LOADING_TIME_PROFILE_SECTION;
CryPathString pakPath = GetCurrentLevelCacheFolder() + "/" + sPakName;
pakPath.MakeLower();
pakPath.replace(AZ_WRONG_FILESYSTEM_SEPARATOR, AZ_CORRECT_FILESYSTEM_SEPARATOR);
// Check if pak is already loaded
for (int i = 0; i < (int)m_openedPaks.size(); i++)
{
if (strstr(m_openedPaks[i].filename.c_str(), pakPath.c_str()))
{
return true;
}
}
// check pak file size.
size_t nFileSize = gEnv->pCryPak->FGetSize(pakPath.c_str(), true);
if (nFileSize <= 0)
{
// Cached file does not exist
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "Level cache pak file %s does not exist", pakPath.c_str());
return false;
}
//set these flags as DLC LevelCache Paks are found via the mod paths,
//and the paks can never be inside other paks so we optimise the search
uint32 nOpenPakFlags = AZ::IO::IArchive::FLAGS_FILENAMES_AS_CRC32 | AZ::IO::IArchive::FLAGS_CHECK_MOD_PATHS | AZ::IO::IArchive::FLAGS_NEVER_IN_PAK;
if (nFileSize < LEVEL_PAK_INMEMORY_MAXSIZE) // 10 megs.
{
if (!(nOpenPakFlags & AZ::IO::IArchive::FLAGS_PAK_IN_MEMORY_CPU))
{
nOpenPakFlags |= AZ::IO::IArchive::FLAGS_PAK_IN_MEMORY;
}
}
SOpenedPak op;
if (gEnv->pCryPak->OpenPack(sBindRoot, { pakPath.c_str(), pakPath.size() }, nOpenPakFlags | AZ::IO::IArchive::FOPEN_HINT_QUIET, NULL, &op.filename))
{
op.bOnlyDuringLevelLoading = bOnlyDuringLevelLoading;
m_openedPaks.push_back(op);
return true;
}
return false;
}
//////////////////////////////////////////////////////////////////////////
bool CResourceManager::LoadModeSwitchPak(const char* sPakName, const bool multiplayer)
{
if (g_cvars.archiveVars.nSaveLevelResourceList)
{
//Don't load the pak if we're trying to save a resourcelist in order to build it.
m_recordedFiles.clear();
gEnv->pCryPak->RegisterFileAccessSink(this);
m_bRegisteredFileOpenSink = true;
return true;
}
else
{
if (g_cvars.archiveVars.nLoadModePaks)
{
// Unload SP common pak if switching to multiplayer
if (multiplayer)
{
UnloadMenuCommonPak(FRONTEND_COMMON_PAK_FILENAME_SP, FRONTEND_COMMON_LIST_FILENAME "_sp");
}
else
{
UnloadMenuCommonPak(FRONTEND_COMMON_PAK_FILENAME_MP, FRONTEND_COMMON_LIST_FILENAME "_mp");
}
//Load the mode switching pak. If this is available and up to date it speeds up this process considerably
bool bOpened = gEnv->pCryPak->OpenPack("@assets@", sPakName, 0);
bool bLoaded = gEnv->pCryPak->LoadPakToMemory(sPakName, AZ::IO::IArchive::eInMemoryPakLocale_GPU);
return (bOpened && bLoaded);
}
else
{
return true;
}
}
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::UnloadModeSwitchPak(const char* sPakName, const char* sResourceListName, const bool multiplayer)
{
if (g_cvars.archiveVars.nSaveLevelResourceList && m_bRegisteredFileOpenSink)
{
m_sLevelFolder = sResourceListName;
SaveRecordedResources();
gEnv->pCryPak->UnregisterFileAccessSink(this);
m_bRegisteredFileOpenSink = false;
}
else
{
if (g_cvars.archiveVars.nLoadModePaks)
{
//Unload the mode switching pak.
gEnv->pCryPak->LoadPakToMemory(sPakName, AZ::IO::IArchive::eInMemoryPakLocale_Unload);
gEnv->pCryPak->ClosePack(sPakName, 0);
//Load the frontend common mode switch pak, this can considerably reduce the time spent switching especially from disc, currently SP only
if (!multiplayer && LoadMenuCommonPak(FRONTEND_COMMON_PAK_FILENAME_SP) == false)
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "Could not load %s during init. This file can significantly reduce frontend loading times.\n", FRONTEND_COMMON_PAK_FILENAME_SP);
}
else if (multiplayer && LoadMenuCommonPak(FRONTEND_COMMON_PAK_FILENAME_MP) == false)
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "Could not load %s during init. This file can significantly reduce frontend loading times.\n", FRONTEND_COMMON_PAK_FILENAME_MP);
}
}
}
}
//////////////////////////////////////////////////////////////////////////
bool CResourceManager::LoadMenuCommonPak(const char* sPakName)
{
if (g_cvars.archiveVars.nSaveMenuCommonResourceList)
{
//Don't load the pak if we're trying to save a resourcelist in order to build it.
m_recordedFiles.clear();
gEnv->pCryPak->RegisterFileAccessSink(this);
m_bRegisteredFileOpenSink = true;
return true;
}
else
{
//Load the mode switching pak. If this is available and up to date it speeds up this process considerably
bool bOpened = gEnv->pCryPak->OpenPack("@assets@", sPakName, 0);
bool bLoaded = gEnv->pCryPak->LoadPakToMemory(sPakName, AZ::IO::IArchive::eInMemoryPakLocale_GPU);
return (bOpened && bLoaded);
}
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::UnloadMenuCommonPak(const char* sPakName, const char* sResourceListName)
{
if (g_cvars.archiveVars.nSaveMenuCommonResourceList)
{
m_sLevelFolder = sResourceListName;
SaveRecordedResources();
gEnv->pCryPak->UnregisterFileAccessSink(this);
m_bRegisteredFileOpenSink = false;
}
else
{
//Unload the mode switching pak.
gEnv->pCryPak->LoadPakToMemory(sPakName, AZ::IO::IArchive::eInMemoryPakLocale_Unload);
gEnv->pCryPak->ClosePack(sPakName, 0);
}
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::UnloadLevelCachePak(const char* sPakName)
{
LOADING_TIME_PROFILE_SECTION;
CryPathString pakPath = GetCurrentLevelCacheFolder() + "/" + sPakName;
pakPath.MakeLower();
pakPath.replace(AZ_WRONG_FILESYSTEM_SEPARATOR, AZ_CORRECT_FILESYSTEM_SEPARATOR);
for (int i = 0; i < (int)m_openedPaks.size(); i++)
{
if (strstr(m_openedPaks[i].filename.c_str(), pakPath.c_str()))
{
gEnv->pCryPak->ClosePack(m_openedPaks[i].filename.c_str(), AZ::IO::IArchive::FLAGS_PATH_REAL);
m_openedPaks.erase(m_openedPaks.begin() + i);
break;
}
}
if (m_openedPaks.empty())
{
stl::free_container(m_openedPaks);
}
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::UnloadAllLevelCachePaks(bool bLevelLoadEnd)
{
LOADING_TIME_PROFILE_SECTION;
if (!bLevelLoadEnd)
{
m_AsyncPakManager.Clear();
UnloadFastLoadPaks();
}
else
{
m_AsyncPakManager.UnloadLevelLoadPaks();
}
uint32 nClosePakFlags = AZ::IO::IArchive::FLAGS_PATH_REAL; //AZ::IO::IArchive::FLAGS_CHECK_MOD_PATHS | AZ::IO::IArchive::FLAGS_NEVER_IN_PAK | AZ::IO::IArchive::FLAGS_PATH_REAL;
for (int i = 0; i < (int)m_openedPaks.size(); i++)
{
if ((m_openedPaks[i].bOnlyDuringLevelLoading && bLevelLoadEnd) ||
!bLevelLoadEnd)
{
gEnv->pCryPak->ClosePack(m_openedPaks[i].filename.c_str(), nClosePakFlags);
}
}
if (g_cvars.archiveVars.nLoadCache)
{
gEnv->pCryPak->LoadPakToMemory(ENGINE_PAK_FILENAME, AZ::IO::IArchive::eInMemoryPakLocale_Unload);
bool usePrefabSystemForLevels = false;
AzFramework::ApplicationRequests::Bus::BroadcastResult(
usePrefabSystemForLevels, &AzFramework::ApplicationRequests::IsPrefabSystemForLevelsEnabled);
if (!usePrefabSystemForLevels)
{
// Force level.pak out of memory.
gEnv->pCryPak->LoadPakToMemory(LEVEL_PAK_FILENAME, AZ::IO::IArchive::eInMemoryPakLocale_Unload);
}
}
if (!bLevelLoadEnd)
{
stl::free_container(m_openedPaks);
}
}
//////////////////////////////////////////////////////////////////////////
bool CResourceManager::LoadPakToMemAsync(const char* pPath, bool bLevelLoadOnly)
{
return m_AsyncPakManager.LoadPakToMemAsync(pPath, bLevelLoadOnly);
}
bool CResourceManager::LoadLayerPak(const char* sLayerName)
{
return m_AsyncPakManager.LoadLayerPak(sLayerName);
}
void CResourceManager::UnloadLayerPak(const char* sLayerName)
{
m_AsyncPakManager.UnloadLayerPak(sLayerName);
}
void CResourceManager::GetLayerPakStats(SLayerPakStats& stats, bool bCollectAllStats) const
{
m_AsyncPakManager.GetLayerPakStats(stats, bCollectAllStats);
}
void CResourceManager::UnloadAllAsyncPaks()
{
m_AsyncPakManager.Clear();
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::Update()
{
m_AsyncPakManager.Update();
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::Init()
{
GetISystem()->GetISystemEventDispatcher()->RegisterListener(this);
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::Shutdown()
{
UnloadAllLevelCachePaks(false);
if (GetISystem() && GetISystem()->GetISystemEventDispatcher())
{
GetISystem()->GetISystemEventDispatcher()->RemoveListener(this);
}
}
//////////////////////////////////////////////////////////////////////////
bool CResourceManager::IsStreamingCachePak(const char* filename) const
{
const char* cachePaks[] = {
"dds_cache.pak",
"cgf_cache.pak",
"skin_cache.pak",
"chr_cache.pak"
};
for (int i = 0; i < sizeof(cachePaks) / sizeof(cachePaks[0]); ++i)
{
if (strstr(filename, cachePaks[i]))
{
return true;
}
}
return false;
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::OnSystemEvent(ESystemEvent event, [[maybe_unused]] UINT_PTR wparam, [[maybe_unused]] UINT_PTR lparam)
{
switch (event)
{
case ESYSTEM_EVENT_FRONTEND_INITIALISED:
{
GetISystem()->GetStreamEngine()->PauseStreaming(false, -1);
}
break;
case ESYSTEM_EVENT_GAME_POST_INIT_DONE:
{
if (g_cvars.archiveVars.nSaveFastloadResourceList != 0)
{
SaveRecordedResources();
if (g_cvars.archiveVars.nSaveLevelResourceList == 0 && g_cvars.archiveVars.nSaveTotalResourceList == 0)
{
m_recordedFiles.clear();
}
}
// Unload all paks from memory, after game init.
UnloadAllLevelCachePaks(false);
gEnv->pCryPak->LoadPaksToMemory(0, false);
if (g_cvars.archiveVars.nLoadCache)
{
//Load the frontend common mode switch pak, this can considerably reduce the time spent switching especially from disc
if (LoadMenuCommonPak(FRONTEND_COMMON_PAK_FILENAME_SP) == false)
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_WARNING, "Could not load %s during init. This file can significantly reduce frontend loading times.\n", FRONTEND_COMMON_PAK_FILENAME_SP);
}
}
break;
}
case ESYSTEM_EVENT_LEVEL_LOAD_PREPARE:
{
UnloadMenuCommonPak(FRONTEND_COMMON_PAK_FILENAME_SP, FRONTEND_COMMON_LIST_FILENAME "_sp");
m_bLevelTransitioning = !m_sLevelName.empty();
m_lastLevelLoadTime.SetValue(0);
m_beginLevelLoadTime = gEnv->pTimer->GetAsyncTime();
if (g_cvars.archiveVars.nSaveLevelResourceList || g_cvars.archiveVars.nSaveTotalResourceList)
{
if (!g_cvars.archiveVars.nSaveTotalResourceList)
{
m_recordedFiles.clear();
}
if (!m_bRegisteredFileOpenSink)
{
gEnv->pCryPak->RegisterFileAccessSink(this);
m_bRegisteredFileOpenSink = true;
}
}
// Cancel any async pak loading, it will fight with the impending sync IO
m_AsyncPakManager.CancelPendingJobs();
// Pause streaming engine for anything but sound, music, video and flash.
uint32 nMask = (1 << eStreamTaskTypeFlash) | (1 << eStreamTaskTypeVideo) | STREAM_TASK_TYPE_AUDIO_ALL; // Unblock specified streams
nMask = ~nMask; // Invert mask, bit set means blocking type.
GetISystem()->GetStreamEngine()->PauseStreaming(true, nMask);
}
break;
case ESYSTEM_EVENT_LEVEL_LOAD_END:
{
if (m_bOwnResourceList)
{
m_bOwnResourceList = false;
// Clear resource list, after level loading.
auto pResList = gEnv->pCryPak->GetResourceList(AZ::IO::IArchive::RFOM_Level);
if (pResList)
{
pResList->Clear();
}
}
}
break;
case ESYSTEM_EVENT_LEVEL_UNLOAD:
UnloadAllLevelCachePaks(false);
break;
case ESYSTEM_EVENT_LEVEL_PRECACHE_START:
{
// Unpause all streams in streaming engine.
GetISystem()->GetStreamEngine()->PauseStreaming(false, -1);
}
break;
case ESYSTEM_EVENT_LEVEL_PRECACHE_FIRST_FRAME:
{
UnloadAllLevelCachePaks(true);
}
break;
case ESYSTEM_EVENT_LEVEL_PRECACHE_END:
{
CTimeValue t = gEnv->pTimer->GetAsyncTime();
m_lastLevelLoadTime = t - m_beginLevelLoadTime;
if (g_cvars.archiveVars.nSaveLevelResourceList && m_bRegisteredFileOpenSink)
{
SaveRecordedResources();
if (!g_cvars.archiveVars.nSaveTotalResourceList)
{
gEnv->pCryPak->UnregisterFileAccessSink(this);
m_bRegisteredFileOpenSink = false;
}
}
UnloadAllLevelCachePaks(true);
}
break;
}
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::GetMemoryStatistics(ICrySizer* pSizer)
{
pSizer->AddContainer(m_openedPaks);
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::ReportFileOpen([[maybe_unused]] AZ::IO::HandleType inFileHandle, AZStd::string_view szFullPath)
{
if (!g_cvars.archiveVars.nSaveLevelResourceList && !g_cvars.archiveVars.nSaveFastloadResourceList && !g_cvars.archiveVars.nSaveMenuCommonResourceList && !g_cvars.archiveVars.nSaveTotalResourceList)
{
return;
}
string file = PathUtil::MakeGamePath(string(szFullPath.data(), szFullPath.size()));
file.replace('\\', '/');
file.MakeLower();
{
CryAutoCriticalSection lock(recordedFilesLock);
m_recordedFiles.push_back(file);
}
}
//////////////////////////////////////////////////////////////////////////
void CResourceManager::SaveRecordedResources(bool bTotalList)
{
CryAutoCriticalSection lock(recordedFilesLock);
std::set<string> fileset;
// eliminate duplicate values
std::vector<string>::iterator endLocation = std::unique(m_recordedFiles.begin(), m_recordedFiles.end());
m_recordedFiles.erase(endLocation, m_recordedFiles.end());
fileset.insert(m_recordedFiles.begin(), m_recordedFiles.end());
string sSequenceFilename = PathUtil::AddSlash(m_sLevelFolder) + (bTotalList ? AUTO_LEVEL_TOTAL_SEQUENCE_RESOURCE_LIST : AUTO_LEVEL_SEQUENCE_RESOURCE_LIST);
{
AZ::IO::HandleType fileHandle = fxopen(sSequenceFilename, "wb", true);
if (fileHandle != AZ::IO::InvalidHandle)
{
for (std::vector<string>::iterator it = m_recordedFiles.begin(); it != m_recordedFiles.end(); ++it)
{
const char* str = it->c_str();
AZ::IO::Print(fileHandle, "%s\n", str);
}
gEnv->pFileIO->Close(fileHandle);
}
}
string sResourceSetFilename = PathUtil::AddSlash(m_sLevelFolder) + (bTotalList ? AUTO_LEVEL_TOTAL_RESOURCE_LIST : AUTO_LEVEL_RESOURCE_LIST);
{
AZ::IO::HandleType fileHandle = fxopen(sResourceSetFilename, "wb", true);
if (fileHandle != AZ::IO::InvalidHandle)
{
for (std::set<string>::iterator it = fileset.begin(); it != fileset.end(); ++it)
{
const char* str = it->c_str();
AZ::IO::Print(fileHandle, "%s\n", str);
}
gEnv->pFileIO->Close(fileHandle);
}
}
}
//////////////////////////////////////////////////////////////////////////
CTimeValue CResourceManager::GetLastLevelLoadTime() const
{
return m_lastLevelLoadTime;
}
Code/CryEngine/CrySystem/ResourceManager.h
-116
View File
@@ -1,116 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Interface to the Resource Manager
#ifndef CRYINCLUDE_CRYSYSTEM_RESOURCEMANAGER_H
#define CRYINCLUDE_CRYSYSTEM_RESOURCEMANAGER_H
#pragma once
#include <IResourceManager.h>
#include "AsyncPakManager.h"
//////////////////////////////////////////////////////////////////////////
// IResource manager interface
//////////////////////////////////////////////////////////////////////////
class CResourceManager
: public IResourceManager
, public ISystemEventListener
, public AZ::IO::IArchiveFileAccessSink
{
public:
CResourceManager();
void Init();
void Shutdown();
bool IsStreamingCachePak(const char* filename) const;
//////////////////////////////////////////////////////////////////////////
// IResourceManager interface implementation.
//////////////////////////////////////////////////////////////////////////
void PrepareLevel(const char* sLevelFolder, const char* sLevelName);
void UnloadLevel();
AZ::IO::IResourceList* GetLevelResourceList();
bool LoadLevelCachePak(const char* sPakName, const char* sBindRoot, bool bOnlyDuringLevelLoading);
void UnloadLevelCachePak(const char* sPakName);
bool LoadModeSwitchPak(const char* sPakName, const bool multiplayer);
void UnloadModeSwitchPak(const char* sPakName, const char* sResourceListName, const bool multiplayer);
bool LoadMenuCommonPak(const char* sPakName);
void UnloadMenuCommonPak(const char* sPakName, const char* sResourceListName);
bool LoadPakToMemAsync(const char* pPath, bool bLevelLoadOnly);
void UnloadAllAsyncPaks();
bool LoadLayerPak(const char* sLayerName);
void UnloadLayerPak(const char* sLayerName);
void UnloadAllLevelCachePaks(bool bLevelLoadEnd);
void GetMemoryStatistics(ICrySizer* pSizer);
bool LoadFastLoadPaks(bool bToMemory);
void UnloadFastLoadPaks();
CTimeValue GetLastLevelLoadTime() const;
void GetLayerPakStats(SLayerPakStats& stats, bool bCollectAllStats) const;
//////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////
// ISystemEventListener interface implementation.
//////////////////////////////////////////////////////////////////////////
virtual void OnSystemEvent(ESystemEvent event, UINT_PTR wparam, UINT_PTR lparam);
//////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////
// IArchiveFileAccessSink interface implementation.
//////////////////////////////////////////////////////////////////////////
virtual void ReportFileOpen(AZ::IO::HandleType inFileHandle, AZStd::string_view szFullPath);
//////////////////////////////////////////////////////////////////////////
// Per frame update of the resource manager.
void Update();
CryPathString GetCurrentLevelCacheFolder() const { return m_currentLevelCacheFolder; };
void SaveRecordedResources(bool bTotalList = false);
private:
//////////////////////////////////////////////////////////////////////////
CryPathString m_currentLevelCacheFolder;
struct SOpenedPak
{
AZStd::fixed_string<AZ::IO::IArchive::MaxPath> filename;
bool bOnlyDuringLevelLoading;
};
std::vector<SOpenedPak> m_openedPaks;
CAsyncPakManager m_AsyncPakManager;
string m_sLevelFolder;
string m_sLevelName;
bool m_bLevelTransitioning;
bool m_bRegisteredFileOpenSink;
bool m_bOwnResourceList;
CTimeValue m_beginLevelLoadTime;
CTimeValue m_lastLevelLoadTime;
AZStd::intrusive_ptr<AZ::IO::IResourceList> m_pSequenceResourceList;
CryCriticalSection recordedFilesLock;
std::vector<string> m_recordedFiles;
AZStd::vector< AZStd::fixed_string<AZ::IO::IArchive::MaxPath> > m_fastLoadPakPaths;
};
#endif // CRYINCLUDE_CRYSYSTEM_RESOURCEMANAGER_H
Code/CryEngine/CrySystem/StreamEngine/AZRequestReadStream.cpp
-410
View File
@@ -1,410 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Engine
#include "CrySystem_precompiled.h"
#include <ISystem.h>
#include <ILog.h>
#include "AZRequestReadStream.h"
#include <AzCore/std/parallel/semaphore.h>
#include <AzCore/IO/IStreamer.h>
#include <AzCore/Component/TickBus.h>
#include "StreamEngine.h"
AZRequestReadStream* AZRequestReadStream::Allocate(const EStreamTaskType tSource, const char* filename, IStreamCallback* callback,
const StreamReadParams* params)
{
//Once an async method is available to read file sizes this code should be removed:
// and the file size should be known before calling this method and pass it as a
// parameter to this method.
//REMOVE In the Future START.
AZ::IO::SizeType fileSize = 0;
if (params && params->nSize)
{
fileSize = params->nSize;
}
else
{
AZ::IO::FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
AZ::IO::Result res = fileIO->Size(filename, fileSize);
if (!res)
{
AZ_Error("AZRequestReadStream", false, "Failed to read file size of %s", filename);
return nullptr;
}
//REMOVE In the Future END.
}
auto streamer = AZ::Interface<AZ::IO::IStreamer>::Get();
AZRequestReadStream* retReq;
retReq = aznew AZRequestReadStream();
retReq->m_Type = tSource;
retReq->m_fileName = filename;
retReq->m_callback = callback;
retReq->m_fileSize = fileSize;
//REMARK: if params->pBuffer is NOT NULL, then retReq->m_buffer
//should become params->pBuffer, this is called stream-in-place.
//The only reason we are not doing this here is because
//some platforms support stream-in-place to WRITE ONLY buffers.
//Because there are no guarantees that low level streaming and decompression apis
//would treat the output buffer as WRITE ONLY, we still allocate the buffer and memcpy
//to params->pBuffer upon the completion callback being called.
//Once LY-98089 is complete/fixed, we should be able to safely
//set retReq->m_buffer = params->pBuffer and skip the memory allocation.
retReq->m_buffer = azmalloc(fileSize, streamer->GetRecommendations().m_memoryAlignment);
if (params)
{
retReq->m_params = *params;
}
return retReq;
}
//////////////////////////////////////////////////////////////////////////
AZRequestReadStream::AZRequestReadStream() : m_fileName(""), m_fileRequest(nullptr),
m_buffer(nullptr), m_Type(eStreamTaskTypeTexture),
m_callback(nullptr), m_fileSize(0), m_numBytesRead(0), m_isAsyncCallbackExecuted(false),
m_isSyncCallbackExecuted(false), m_isFileRequestComplete(false), m_isError(false), m_isFinished(false),
m_IOError(0)
{
AZStd::atomic_init<int>(&m_refCount, 0);
m_params = StreamReadParams();
}
//////////////////////////////////////////////////////////////////////////
AZRequestReadStream::~AZRequestReadStream()
{
azfree(m_buffer);
}
// tries to stop reading the stream; this is advisory and may have no effect
// all the callbacks will be called after this. If you just destructing object,
// dereference this object and it will automatically abort and release all associated resources.
void AZRequestReadStream::Abort()
{
{
CryAutoCriticalSection lock(m_callbackLock);
// Increase ref counting to avoid preliminary destruction
AZRequestReadStream_AutoPtr refCountLock(this);
if (m_isFileRequestComplete || m_isError)
{
// It is possible the file I/O request to be completed by AZ::IO::Streamer,
// but if the completion callback is deferred for the main thread then
// the stream is not finished. So, only if it is finished then
// it is safe to do nothing.
if (m_isFinished)
{
return;
}
}
m_isError = true;
m_IOError = ERROR_USER_ABORT;
m_isFileRequestComplete = true;
m_numBytesRead = 0;
if (m_fileRequest)
{
auto streamer = AZ::Interface<AZ::IO::IStreamer>::Get();
streamer->QueueRequest(streamer->Cancel(m_fileRequest));
}
// all the callbacks have to handle error cases and needs to be called anyway, even if the stream I/O is aborted
ExecuteAsyncCallback_CBLocked();
ExecuteSyncCallback_CBLocked();
m_callback = nullptr;
}
}
bool AZRequestReadStream::TryAbort()
{
// Increase ref counting to avoid preliminary destruction
AZRequestReadStream_AutoPtr refCountLock(this);
if (!m_callbackLock.TryLock())
{
return false;
}
if (m_isFileRequestComplete || m_isError)
{
// It is possible the file I/O request to be completed by AZ::IO::Streamer,
// but if the completion callback is deferred for the main thread then
// the stream is not finished. So, only if it is finished then
// it is safe to do nothing.
if (m_isFinished)
{
m_callbackLock.Unlock();
return false;
}
}
m_isError = true;
m_IOError = ERROR_USER_ABORT;
m_isFileRequestComplete = true;
m_numBytesRead = 0;
if (m_fileRequest)
{
auto streamer = AZ::Interface<AZ::IO::IStreamer>::Get();
streamer->QueueRequest(streamer->Cancel(m_fileRequest));
}
// all the callbacks have to handle error cases and needs to be called anyway, even if the stream I/O is aborted
ExecuteAsyncCallback_CBLocked();
ExecuteSyncCallback_CBLocked();
m_callback = nullptr;
m_callbackLock.Unlock();
return true;
}
// tries to raise the priority of the read; this is advisory and may have no effect
void AZRequestReadStream::SetPriority(EStreamTaskPriority ePriority)
{
CryAutoCriticalSection lock(m_callbackLock);
if (m_params.ePriority != ePriority)
{
m_params.ePriority = ePriority;
if (m_fileRequest)
{
AZ::Interface<AZ::IO::IStreamer>::Get()->RescheduleRequest(m_fileRequest, AZ::IO::IStreamerTypes::s_noDeadline,
CStreamEngine::CryStreamPriorityToAZStreamPriority(ePriority));
}
}
}
// unconditionally waits until the callback is called
// i.e. if the stream hasn't yet finish, it's guaranteed that the user-supplied callback
// is called before return from this function (unless no callback was specified)
void AZRequestReadStream::Wait(int maxWaitMillis)
{
// lock this object to avoid preliminary destruction
AZRequestReadStream_AutoPtr refCountLock(this);
if (!m_isFinished && !m_isError && !m_fileRequest)
{
AZ_Error("AZRequestReadStream", false, "Stream for file %s is unwaitable", m_fileName.c_str());
return;
}
if (maxWaitMillis > 0)
{
m_wait.try_acquire_for(AZStd::chrono::milliseconds(maxWaitMillis));
}
else
{
m_wait.acquire();
}
}
//////////////////////////////////////////////////////////////////////////
const char* AZRequestReadStream::GetErrorName() const
{
switch (m_IOError)
{
case ERROR_UNKNOWN_ERROR:
return "Unknown error";
case ERROR_UNEXPECTED_DESTRUCTION:
return "Unexpected destruction";
case ERROR_INVALID_CALL:
return "Invalid call";
case ERROR_CANT_OPEN_FILE:
return "Cannot open the file";
case ERROR_REFSTREAM_ERROR:
return "Refstream error";
case ERROR_OFFSET_OUT_OF_RANGE:
return "Offset out of range";
case ERROR_REGION_OUT_OF_RANGE:
return "Region out of range";
case ERROR_SIZE_OUT_OF_RANGE:
return "Size out of range";
case ERROR_CANT_START_READING:
return "Cannot start reading";
case ERROR_OUT_OF_MEMORY:
return "Out of memory";
case ERROR_ABORTED_ON_SHUTDOWN:
return "Aborted on shutdown";
case ERROR_OUT_OF_MEMORY_QUOTA:
return "Out of memory quota";
case ERROR_ZIP_CACHE_FAILURE:
return "ZIP cache failure";
case ERROR_USER_ABORT:
return "User aborted";
}
return "Unrecognized error";
}
int AZRequestReadStream::AddRef()
{
return m_refCount.fetch_add(1) + 1;
}
int AZRequestReadStream::Release()
{
int refCount = m_refCount.fetch_sub(1);
#ifndef _RELEASE
if (refCount < 1)
{
__debugbreak();
}
#endif
if (refCount == 1)
{
//UNUSUAL, yet necessary.
//Why "delete this"?
//So, AZRequestReadStream is a replacement of CReadStream. The original design of
//Cry Texture Mips Streaming makes use of CReadStream through IReadStreamPtr, which
//is a smart pointer design that calls AddRef() and Release() but never calls "delete",
//like AZStd::shared_ptr<> does. This means the original Cry design had a memory leak
//because it never called delete on IReadStream objects. If you look at the original
//code of CReadStream (StreamReadStream.cpp) , the static Allocate method has two paths
//to allocate memory, one used a stack based memory allocation hack, and the other path
//was doing a "new CReadStream". Using VS2017 debugger I found both paths to be used, but
//"delete" and hence the destructor of CReadStream is never called causing minor memory leaks.
//The best solution I found was to call "delete this" here and later when we chnage IReadStreamPtr
//for AZstd::smart_ptr then AddRef() and Release() won't be needed anymore and this "delete this"
//hack won't be necessary either.
delete this;
}
return refCount - 1;
}
//////////////////////////////////////////////////////////////////////////
void AZRequestReadStream::ExecuteAsyncCallback_CBLocked()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
if (!m_isAsyncCallbackExecuted && m_callback)
{
m_isAsyncCallbackExecuted = true;
m_callback->StreamAsyncOnComplete(this, m_IOError);
}
}
void AZRequestReadStream::ExecuteSyncCallback_CBLocked()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
if (!m_isSyncCallbackExecuted && m_callback && (0 == (m_params.nFlags & IStreamEngine::FLAGS_NO_SYNC_CALLBACK)))
{
m_isSyncCallbackExecuted = true;
AZRequestReadStream_AutoPtr refCountLock(this); // Stream can be freed inside the callback!
m_callback->StreamOnComplete(this, m_IOError);
m_isFinished = true;
FreeTemporaryMemory();
}
}
//////////////////////////////////////////////////////////////////////////
void AZRequestReadStream::FreeTemporaryMemory()
{
// Make sure m_buffer is not freed if the file request is still in flight, as Streamer can still write to m_buffer in that case
if (!m_fileRequest || AZ::Interface<AZ::IO::IStreamer>::Get()->HasRequestCompleted(m_fileRequest))
{
azfree(m_buffer);
m_buffer = nullptr;
m_numBytesRead = 0;
}
}
//////////////////////////////////////////////////////////////////////////
void AZRequestReadStream::OnRequestComplete(AZ::IO::SizeType numBytesRead, [[maybe_unused]] void* buffer, AZ::IO::IStreamerTypes::RequestStatus requestState)
{
CryAutoCriticalSection lock(m_callbackLock);
if (!m_isFileRequestComplete)
{
switch (requestState)
{
case AZ::IO::IStreamerTypes::RequestStatus::Completed:
m_IOError = 0;
m_numBytesRead = static_cast<uint32>(numBytesRead);
m_isError = false;
if (m_params.pBuffer)
{
//In some systems, streaming-in-place is supported. The caveat
//is that in some cases, the destination buffer is write-only. This is why
//a final memcpy must be done here until support is added to AZ::IO::Streamer API
//to decompress/load data into write-only buffers. SEE: LY-98089
AZ_Assert(m_params.pBuffer != m_buffer, "Streaming-In-Place requires destination and source buffers to be different");
memcpy(m_params.pBuffer, m_buffer, numBytesRead);
}
break;
case AZ::IO::IStreamerTypes::RequestStatus::Canceled:
m_IOError = ERROR_USER_ABORT;
m_numBytesRead = 0;
m_isError = true;
break;
default:
m_IOError = ERROR_UNKNOWN_ERROR;
m_numBytesRead = 0;
m_isError = true;
break;
}
ExecuteAsyncCallback_CBLocked();
m_isFileRequestComplete = true;
if (m_params.nFlags & IStreamEngine::FLAGS_NO_SYNC_CALLBACK)
{
// We do not need FileRequest here anymore, and not its temporary memory.
m_fileRequest = nullptr;
m_isFinished = true;
}
else
{
//The completion must be triggered from MainThread. (Typically only happens when loading Terrain Macro Textures
AddRef();
AZ::SystemTickBus::QueueFunction([this] {
RequestCompleteOnMainThread();
});
}
}
m_wait.release();
}
void AZRequestReadStream::RequestCompleteOnMainThread()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
// call asynchronous callback function if needed synchronously
{
CryAutoCriticalSection lock(m_callbackLock);
ExecuteSyncCallback_CBLocked();
}
//Always called because before enqueuing this call was called AddRef()
Release();
}
Code/CryEngine/CrySystem/StreamEngine/AZRequestReadStream.h
-151
View File
@@ -1,151 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Description : An IReadStream implementation designed to work with AZ::IO::Streamer
// instead of CStreamEngine.
#pragma once
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/Memory/SystemAllocator.h>
#include <AzCore/std/parallel/semaphore.h>
#include <AzCore/std/smart_ptr/shared_ptr.h>
#include "IStreamEngine.h"
namespace AZ
{
namespace IO
{
class Request;
}
}
//This class is a wrapper of AZ::IO::Request so Cry Classes can use AZ::IO::Streamer.
//Basicallythis replaces CReadStream.
class AZRequestReadStream
: public IReadStream
{
public:
AZ_CLASS_ALLOCATOR(AZRequestReadStream, AZ::SystemAllocator, 0);
static AZRequestReadStream* Allocate(const EStreamTaskType tSource, const char* filename, IStreamCallback* callback,
const StreamReadParams* params);
int AddRef() override;
int Release() override;
DWORD_PTR GetUserData() override {return m_params.dwUserData; }
// set user defined data into stream's params
void SetUserData(DWORD_PTR userData) override { m_params.dwUserData = userData; };
// returns true if the file read was not successful.
bool IsError() override { return m_isError; };
// returns true if the file read was completed (successfully or unsuccessfully)
// check IsError to check if the whole requested file (piece) was read
bool IsFinished() override { return m_isFinished; };
// returns the number of bytes read so far (the whole buffer size if IsFinished())
unsigned int GetBytesRead([[maybe_unused]] bool bWait) override { return static_cast<unsigned int>(m_numBytesRead); };
// returns the buffer into which the data has been or will be read
// at least GetBytesRead() bytes in this buffer are guaranteed to be already read
const void* GetBuffer() override { return m_buffer; };
// tries to stop reading the stream; this is advisory and may have no effect
// but the callback will not be called after this. If you just destructing object,
// dereference this object and it will automatically abort and release all associated resources.
void Abort() override;
bool TryAbort() override;
// unconditionally waits until the callback is called
// i.e. if the stream hasn't yet finish, it's guaranteed that the user-supplied callback
// is called before return from this function (unless no callback was specified)
void Wait(int maxWaitMillis = -1) override;
const StreamReadParams& GetParams() const override {return m_params; }
const EStreamTaskType GetCallerType() const override { return m_Type; }
//We must define this one. But it is never used in the context of AZ::IO::Streamer.
//Legacy Cry StreamEngine stuff.
EStreamSourceMediaType GetMediaType() const override { return EStreamSourceMediaType::eStreamSourceTypeUnknown; }
// return pointer to callback routine(can be NULL)
IStreamCallback* GetCallback() const override { return m_callback; };
// return IO error #
unsigned GetError() const override { return m_IOError; };
// Returns IO error name
const char* GetErrorName() const override;
// return stream name
const char* GetName() const override { return m_fileName.c_str(); };
void FreeTemporaryMemory() override;
// tries to raise the priority of the read; this is advisory and may have no effect
void SetPriority(EStreamTaskPriority EPriority);
uint64 GetPriority() const { return m_params.ePriority; };
void* GetFileReadBuffer() { return m_buffer; } //GetBuffer from IReadStream is "const void *"
AZStd::size_t GetFileSize() { return m_fileSize; }
void SetFileRequest(AZ::IO::FileRequestPtr request) { m_fileRequest = AZStd::move(request); }
AZ::IO::FileRequestPtr GetFileRequest() { return m_fileRequest; }
void OnRequestComplete(AZ::IO::SizeType numBytesRead, void* buffer, AZ::IO::IStreamerTypes::RequestStatus requestState);
private:
AZRequestReadStream();
virtual ~AZRequestReadStream();
// call the async callback
void ExecuteAsyncCallback_CBLocked();
void ExecuteSyncCallback_CBLocked();
void RequestCompleteOnMainThread();
AZStd::atomic_int m_refCount;
CryCriticalSection m_callbackLock;
StreamReadParams m_params;
AZStd::semaphore m_wait;
CryStringLocal m_fileName;
AZ::IO::FileRequestPtr m_fileRequest;
// Bytes actually read from media.
void* m_buffer;
// the type of the task
EStreamTaskType m_Type;
// the initial data from the user
// the callback; may be NULL
IStreamCallback* m_callback;
AZ::IO::SizeType m_fileSize; //Expected number of bytes to be read.
AZ::IO::SizeType m_numBytesRead; //On a successful read m_nBytesRead == m_fileSize;
bool m_isAsyncCallbackExecuted;
bool m_isSyncCallbackExecuted;
bool m_isFileRequestComplete;
bool m_isError;
bool m_isFinished;
unsigned int m_IOError;
};
TYPEDEF_AUTOPTR(AZRequestReadStream);
Code/CryEngine/CrySystem/StreamEngine/StreamAsyncFileRequest.cpp
-1027
View File
File diff suppressed because it is too large Load Diff
Code/CryEngine/CrySystem/StreamEngine/StreamAsyncFileRequest.h
-571
View File
@@ -1,571 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Thread for IO
#ifndef CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMASYNCFILEREQUEST_H
#define CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMASYNCFILEREQUEST_H
#pragma once
#include <IStreamEngineDefs.h>
#include <AzCore/Jobs/LegacyJobExecutor.h>
#include "TimeValue.h"
#define STREAMENGINE_LL_ALIGN _MS_ALIGN(MEMORY_ALLOCATION_ALIGNMENT)
class CStreamEngine;
class CAsyncIOFileRequest;
struct z_stream_s;
class CStreamingIOThread;
namespace AZ::IO
{
struct CCachedFileData;
}
class CCryFile;
struct SStreamJobEngineState;
class CMTSafeHeap;
class CAsyncIOFileRequest_TransferPtr;
struct SStreamEngineTempMemStats;
#if !defined(USE_EDGE_ZLIB)
// Prevent compilation conflicts - zconf.h (included by zlib.h) defines WINDOWS and WIN32 - those
// definitions conflict with CryEngine's definitions.
# if defined(CRY_TMP_DEFINED_WINDOWS) || defined(CRY_TMP_DEFINED_WIN32)
# error CRY_TMP_DEFINED_WINDOWS and/or CRY_TMP_DEFINED_WIN32 already defined
# endif
# if defined(WINDOWS)
# define CRY_TMP_DEFINED_WINDOWS 1
# endif
# if defined(WIN32)
# define CRY_TMP_DEFINED_WIN32 1
# endif
# include <zlib.h>
# if !defined(CRY_TMP_DEFINED_WINDOWS)
# undef WINDOWS
#endif
# undef CRY_TMP_DEFINED_WINDOWS
# if !defined(CRY_TMP_DEFINED_WIN32)
# undef WIN32
# endif
# undef CRY_TMP_DEFINED_WIN32
// Undefine macros defined in zutil.h to prevent compilation errors in 'steamclientpublic.h', 'OVR_Math.h' etc.
# undef Assert
# undef Trace
# undef Tracev
# undef Tracevv
# undef Tracec
# undef Tracecv
#endif // !defined(USE_EDGE_ZLIB)
namespace AZ::IO::ZipDir {
struct UncompressLookahead;
}
struct IAsyncIOFileCallback
{
virtual ~IAsyncIOFileCallback(){}
// Asynchronous finished event.
// Must be thread safe, can be called from a different thread.
virtual void OnAsyncFinished(CAsyncIOFileRequest* pFileRequest) = 0;
};
struct SStreamPageHdr
{
explicit SStreamPageHdr(int size)
: nRefs()
, nSize(size)
{}
volatile int nRefs;
int nSize;
};
struct SStreamJobQueue
{
enum
{
MaxJobs = 256,
};
struct Job
{
void* pSrc;
SStreamPageHdr* pSrcHdr;
uint32 nOffs;
uint32 nBytes : 31;
uint32 bLast : 1;
};
SStreamJobQueue()
: m_sema(MaxJobs, MaxJobs)
{
m_nQueueLen = 0;
m_nPush = 0;
m_nPop = 0;
memset(m_jobs, 0, sizeof(m_jobs));
}
void Flush(SStreamEngineTempMemStats& tms);
int Push(void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nOffs, uint32 nBytes, bool bLast);
int Pop();
CryFastSemaphore m_sema;
Job m_jobs[MaxJobs];
volatile int m_nQueueLen;
volatile int m_nPush;
volatile int m_nPop;
};
// This class represent a request to read some file from disk asynchronously via one of the IO threads.
class CAsyncIOFileRequest
{
public:
enum EStatus
{
eStatusNotReady,
eStatusInFileQueue,
eStatusFailed,
eStatusUnzipComplete,
eStatusDone,
};
enum
{
BUFFER_ALIGNMENT = 128,
WINDOW_SIZE = 1 << 15,
#if defined(ANDROID)
STREAMING_PAGE_SIZE = (128 * 1024),
#else
STREAMING_PAGE_SIZE = (1 * 1024 * 1024),
#endif
#if defined(ANDROID)
STREAMING_BLOCK_SIZE = (64 * 1024),
#else
STREAMING_BLOCK_SIZE = (32 * 1024),
#endif
};
public:
static CAsyncIOFileRequest* Allocate(EStreamTaskType eType);
static void Flush();
public:
void AddRef();
int Release();
public:
void Init(EStreamTaskType eType);
void Finalize();
void Reset();
ILINE bool IsCancelled() const { return m_nError == ERROR_USER_ABORT; }
ILINE bool HasFailed() const { return m_nError != 0; }
void Failed(uint32 nError)
{
CryInterlockedCompareExchange(reinterpret_cast<volatile LONG*>(&m_nError), nError, 0);
}
uint32 OpenFile(CCryFile& file);
uint32 ReadFile(CStreamingIOThread* pIOThread);
uint32 ReadFileResume(CStreamingIOThread* pIOThread);
uint32 ReadFileInPages(CStreamingIOThread* pIOThread, CCryFile& file);
uint32 ReadFileCheckPreempt(CStreamingIOThread* pIOThread);
uint32 ConfigureRead(AZ::IO::CCachedFileData* pFileData);
bool CanReadInPages();
uint32 AllocateOutput(AZ::IO::CCachedFileData* pZipEntry);
unsigned char* AllocatePage(size_t sz, bool bOnlyPakMem, SStreamPageHdr*& pHdrOut);
static void JobFinalize_Read(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState);
uint32 PushDecompressPage(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nBytes, bool bLast);
uint32 PushDecompressBlock(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nOffs, uint32 nBytes, bool bLast);
static void JobStart_Decompress(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState, int nSlot);
void DecompressBlockEntry(SStreamJobEngineState engineState, int nJob);
void Cancel();
bool TryCancel();
void SyncWithDecompress();
void ComputeSortKey(uint64 nCurrentKeyInProgress);
void SetPriority(EStreamTaskPriority estp);
void BumpSweep();
void FreeBuffer();
bool IgnoreOutofTmpMem() const;
CStreamEngine* GetStreamEngine();
EStreamSourceMediaType GetMediaType();
private:
void* operator new (size_t sz);
void operator delete(void* p);
private:
static void JobFinalize_Decompress(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState);
static void JobFinalize_Transfer(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState);
private:
void JobFinalize_Buffer(const SStreamJobEngineState& engineState);
void JobFinalize_Validate(const SStreamJobEngineState& engineState);
private:
CAsyncIOFileRequest();
~CAsyncIOFileRequest();
public:
static volatile int s_nLiveRequests;
static SLockFreeSingleLinkedListHeader s_freeRequests;
public:
// Must be first
STREAMENGINE_LL_ALIGN SLockFreeSingleLinkedListEntry m_nextFree;
volatile int m_nRefCount;
// Locks to be held whilst the file is being read, and an external memory buffer is in use
// (to ensure that if cancelled, the stream engine doesn't write to the external buffer)
// Separate locks for read and decomp as they can overlap (block decompress)
// Cancel() must acquire both
CryCriticalSection m_externalBufferLockRead;
CryCriticalSection m_externalBufferLockDecompress;
CryStringLocal m_strFileName;
string m_pakFile;
// If request come from stream, it will be not 0.
IReadStreamPtr m_pReadStream;
AZStd::unique_ptr<AZ::LegacyJobExecutor> m_decompJobExecutor;
// Only POD data should exist beyond this point - will be memsetted to 0 on Reset !
uint64 m_nSortKey;
EStreamTaskPriority m_ePriority;
EStreamSourceMediaType m_eMediaType;
EStreamTaskType m_eType;
volatile EStatus m_status;
volatile uint32 m_nError;
uint32 m_nRequestedOffset;
uint32 m_nRequestedSize;
// the file size, or 0 if the file couldn't be opened
uint32 m_nFileSize;
uint32 m_nFileSizeCompressed;
void* m_pMemoryBuffer;
uint32 m_nMemoryBufferSize;
volatile int m_nMemoryBufferUsers;
void* m_pExternalMemoryBuffer;
void* m_pOutputMemoryBuffer;
void* m_pReadMemoryBuffer;
uint32 m_nReadMemoryBufferSize;
uint32 m_bCompressedBuffer : 1;
uint32 m_bStatsUpdated : 1;
uint32 m_bStreamInPlace : 1;
uint32 m_bWriteOnlyExternal : 1;
uint32 m_bSortKeyComputed : 1;
uint32 m_bOutputAllocated : 1;
uint32 m_bReadBegun : 1;
// Actual size of the data on the media.
uint32 m_nSizeOnMedia;
int64 m_nDiskOffset;
int32 m_nReadHeadOffsetKB; // Offset of the Read Head when reading from media.
int32 m_nTimeGroup;
int32 m_nSweep;
IAsyncIOFileCallback* m_pCallback;
//
// Block based streaming
//
uint32 m_nPageReadStart;
uint32 m_nPageReadCurrent;
uint32 m_nPageReadEnd;
volatile uint32 m_nBytesDecompressed;
uint32 m_crc32FromHeader;
volatile LONG m_nFinalised;
z_stream_s* m_pZlibStream;
AZ::IO::ZipDir::UncompressLookahead* m_pLookahead;
SStreamJobQueue* m_pDecompQueue;
#ifdef STREAMENGINE_ENABLE_STATS
// Time that read operation took.
CTimeValue m_readTime;
CTimeValue m_unzipTime;
CTimeValue m_verifyTime;
CTimeValue m_startTime;
CTimeValue m_completionTime;
uint32 m_nReadCounter;
#endif
};
TYPEDEF_AUTOPTR(CAsyncIOFileRequest);
struct SStreamRequestQueue
{
CryCriticalSection m_lock;
std::vector<CAsyncIOFileRequest*> m_requests;
CryEvent m_awakeEvent;
SStreamRequestQueue();
~SStreamRequestQueue();
void Reset();
bool IsEmpty() const;
// Transfers ownership (rather than shares ownership) to the queue
void TransferRequest(CAsyncIOFileRequest_TransferPtr& pReq);
bool TryPopRequest(CAsyncIOFileRequest_AutoPtr& pOut);
private:
SStreamRequestQueue(const SStreamRequestQueue&);
SStreamRequestQueue& operator = (const SStreamRequestQueue&);
};
#if defined(STREAMENGINE_ENABLE_STATS)
struct SStreamEngineDecompressStats
{
uint64 m_nTotalBytesUnziped;
uint64 m_nTempBytesUnziped;
uint64 m_nTotalBytesVerified;
uint64 m_nTempBytesVerified;
CTimeValue m_totalUnzipTime;
CTimeValue m_tempUnzipTime;
CTimeValue m_totalVerifyTime;
CTimeValue m_tempVerifyTime;
};
#endif
class CAsyncIOFileRequest_TransferPtr
{
public:
explicit CAsyncIOFileRequest_TransferPtr(CAsyncIOFileRequest* p)
: m_p(p)
{
}
~CAsyncIOFileRequest_TransferPtr()
{
if (m_p)
{
m_p->Release();
}
}
CAsyncIOFileRequest* operator -> () { return m_p; }
CAsyncIOFileRequest& operator * () { return *m_p; }
const CAsyncIOFileRequest* operator -> () const { return m_p; }
const CAsyncIOFileRequest& operator * () const { return *m_p; }
operator bool () const {
return m_p != NULL;
}
CAsyncIOFileRequest* Relinquish()
{
CAsyncIOFileRequest* p = m_p;
m_p = NULL;
return p;
}
CAsyncIOFileRequest_TransferPtr& operator = (CAsyncIOFileRequest* p)
{
#ifndef _RELEASE
if (m_p)
{
__debugbreak();
}
#endif
m_p = p;
return *this;
}
private:
CAsyncIOFileRequest_TransferPtr(const CAsyncIOFileRequest_TransferPtr&);
CAsyncIOFileRequest_TransferPtr& operator = (const CAsyncIOFileRequest_TransferPtr&);
private:
CAsyncIOFileRequest* m_p;
};
class CStreamEngineWakeEvent
{
public:
CStreamEngineWakeEvent()
: m_state(0)
{
}
void Set()
{
volatile LONG oldState, newState;
bool bSignalInner;
do
{
bSignalInner = false;
oldState = m_state;
newState = oldState | 0x80000000;
if (oldState & 0x7fffffff)
{
bSignalInner = true;
}
}
while (CryInterlockedCompareExchange(&m_state, newState, oldState) != oldState);
if (bSignalInner)
{
m_innerEvent.Set();
}
}
bool Wait(uint32 timeout = 0)
{
bool bTimedOut = false;
bool bAcquiredSignal = false;
while (!bTimedOut && !bAcquiredSignal)
{
volatile long oldState, newState;
do
{
bAcquiredSignal = false;
oldState = m_state;
if (oldState & 0x80000000)
{
// Signalled
newState = oldState & 0x7fffffff;
bAcquiredSignal = true;
}
else
{
newState = oldState + 1;
}
}
while (CryInterlockedCompareExchange(&m_state, newState, oldState) != oldState);
if (!bAcquiredSignal)
{
if (!timeout)
{
m_innerEvent.Wait();
}
else
{
bTimedOut = !m_innerEvent.Wait(timeout);
}
if (!bTimedOut)
{
m_innerEvent.Reset();
}
do
{
bAcquiredSignal = false;
oldState = m_state;
if (!bTimedOut && (oldState & 0x80000000))
{
newState = (oldState & 0x7fffffff) - 1;
bAcquiredSignal = true;
}
else
{
newState = oldState - 1;
}
}
while (CryInterlockedCompareExchange(&m_state, newState, oldState) != oldState);
}
}
return bAcquiredSignal;
}
private:
CStreamEngineWakeEvent(const CStreamEngineWakeEvent&);
CStreamEngineWakeEvent& operator = (const CStreamEngineWakeEvent&);
private:
volatile LONG m_state;
CryEvent m_innerEvent;
};
struct SStreamEngineTempMemStats
{
enum
{
MaxWakeEvents = 8,
};
SStreamEngineTempMemStats()
{
memset(this, 0, sizeof(*this));
}
void* TempAlloc(CMTSafeHeap* pHeap, size_t nSize, const char* szDbgSource, bool bFallBackToMalloc = true, bool bUrgent = false, uint32 align = 0);
void TempFree(CMTSafeHeap* pHeap, const void* p, size_t nSize);
void ReportTempMemAlloc(uint32 nSizeAlloc, uint32 nSizeFree, bool bTriggerWake);
volatile LONG m_nTempAllocatedMemory;
volatile LONG m_nTempAllocatedMemoryFrameMax;
int m_nTempMemoryBudget;
CStreamEngineWakeEvent* m_wakeEvents[MaxWakeEvents];
int m_nWakeEvents;
};
struct SStreamJobEngineState
{
std::vector<SStreamRequestQueue*>* pReportQueues;
#if defined(STREAMENGINE_ENABLE_STATS)
SStreamEngineStatistics* pStats;
SStreamEngineDecompressStats* pDecompressStats;
#endif
SStreamEngineTempMemStats* pTempMem;
CMTSafeHeap* pHeap;
};
#endif // CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMASYNCFILEREQUEST_H
Code/CryEngine/CrySystem/StreamEngine/StreamAsyncFileRequest_Jobs.cpp
-531
View File
@@ -1,531 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySystem_precompiled.h"
#include <AzCore/Debug/Profiler.h>
#include <CryPath.h>
#include "StreamAsyncFileRequest.h"
#include "MTSafeAllocator.h"
namespace AZ::IO::ZipDir::ZipDirStructuresInternal
{
extern void ZlibInflateElementPartial_Impl(
int* pReturnCode, z_stream* pZStream, ZipDir::UncompressLookahead* pLookahead,
uint8_t* pOutput, size_t nOutputLen, bool bOutputWriteOnly,
const uint8_t* pInput, size_t nInputLen, size_t* pTotalOut);
}
#ifdef STREAMENGINE_ENABLE_LISTENER
#include "IStreamEngine.h"
class NotifyListener
{
public:
NotifyListener(IStreamEngineListener* pL, CAsyncIOFileRequest* pReq)
: m_pL(pL)
, m_pReq(pReq)
, m_bInProgress(false) {}
virtual ~NotifyListener() {}
protected:
IStreamEngineListener* m_pL;
CAsyncIOFileRequest* m_pReq;
bool m_bInProgress;
};
class NotifyListenerInflate
: NotifyListener
{
public:
NotifyListenerInflate(IStreamEngineListener* pL, CAsyncIOFileRequest* pReq)
: NotifyListener(pL, pReq)
{
if (m_pL)
{
m_pL->OnStreamBeginInflate(m_pReq);
m_bInProgress = true;
}
}
~NotifyListenerInflate()
{
End();
}
void End()
{
if (m_bInProgress)
{
m_pL->OnStreamEndInflate(m_pReq);
m_bInProgress = false;
}
}
};
#endif
#if defined(STREAMENGINE_ENABLE_STATS)
#define STREAMENGINE_ENABLE_TIMING
#endif
//#define STREAM_DECOMPRESS_TRACE(...) OutputDebugString(AZStd::string::format(__VA_ARGS__).c_str());
#define STREAM_DECOMPRESS_TRACE(...)
void SStreamJobQueue::Flush(SStreamEngineTempMemStats& tms)
{
extern CMTSafeHeap* g_pPakHeap;
for (int c = m_nQueueLen, i = m_nPop % MaxJobs; c; --c, i = (i + 1) % MaxJobs)
{
Job& j = m_jobs[i];
if (j.pSrcHdr && CryInterlockedDecrement(&j.pSrcHdr->nRefs) == 0)
{
tms.TempFree(g_pPakHeap, j.pSrc, j.pSrcHdr->nSize);
}
j.pSrc = NULL;
}
}
int SStreamJobQueue::Push(void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nOffs, uint32 nBytes, bool bLast)
{
m_sema.Acquire();
int nSlot = (m_nPush++) % MaxJobs;
Job& j = m_jobs[nSlot];
j.pSrc = pSrc;
j.pSrcHdr = pSrcHdr;
j.nOffs = nOffs;
j.nBytes = nBytes;
j.bLast = (uint32)bLast;
bool bStartNext = CryInterlockedIncrement(&m_nQueueLen) == 1;
return bStartNext ? nSlot : -1;
}
int SStreamJobQueue::Pop()
{
int nSlot = (++m_nPop) % MaxJobs;
bool bStartNext = CryInterlockedDecrement(&m_nQueueLen) > 0;
m_sema.Release();
return bStartNext ? nSlot : -1;
}
void CAsyncIOFileRequest::AddRef()
{
//int nRef =
CryInterlockedIncrement(&m_nRefCount);
STREAM_DECOMPRESS_TRACE("[StreamDecompress],AddRef,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nRef);
}
int CAsyncIOFileRequest::Release()
{
int nRef = CryInterlockedDecrement(&m_nRefCount);
STREAM_DECOMPRESS_TRACE("[StreamDecompress],Release,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nRef);
#ifndef _RELEASE
if (nRef < 0)
{
__debugbreak();
}
#endif
if (nRef == 0)
{
Finalize();
CryInterlockedPushEntrySList(s_freeRequests, m_nextFree);
}
return nRef;
}
void CAsyncIOFileRequest::DecompressBlockEntry(SStreamJobEngineState engineState, int nJob)
{
AZ_PROFILE_FUNCTION(AZ::Debug::ProfileCategory::System);
STREAM_DECOMPRESS_TRACE("[StreamDecompress],DecompressBlockEntry,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nJob);
CAsyncIOFileRequest_TransferPtr pSelf(this);
SStreamJobQueue::Job& job = m_pDecompQueue->m_jobs[nJob];
void* pSrc = job.pSrc;
SStreamPageHdr* const pSrcHdr = job.pSrcHdr;
const uint32 nOffs = job.nOffs;
const uint32 nBytes = job.nBytes;
const bool bLast = job.bLast;
const bool bFailed = HasFailed();
if (!bFailed)
{
#if defined(STREAMENGINE_ENABLE_TIMING)
LARGE_INTEGER liStart;
QueryPerformanceCounter(&liStart);
#endif
//printf("Inflate: %s Avail in: %d, Avail Out: %d, Next In: 0x%p, Next Out: 0x%p\n", m_strFileName.c_str(), m_pZlibStream->avail_in, m_pZlibStream->avail_out, m_pZlibStream->next_in, m_pZlibStream->next_out);
#ifdef STREAMENGINE_ENABLE_LISTENER
NotifyListenerInflate inflateListener(gEnv->pSystem->GetStreamEngine()->GetListener(), this);
#endif
size_t nBytesDecomped = m_nBytesDecompressed;
STREAM_DECOMPRESS_TRACE ("[StreamDecompress],ZlibInflateElementPartial_Impl,0x%x,%s,0x%p,%i,0x%p,%i,%i\n",
CryGetCurrentThreadId(),
m_strFileName.c_str(),
(uint8_t*)m_pReadMemoryBuffer + nBytesDecomped,
m_nFileSize - nBytesDecomped,
(uint8_t*)pSrc + nOffs,
nBytes,
nBytesDecomped);
int readStatus = Z_OK;
{
CryOptionalAutoLock<CryCriticalSection> decompLock(m_externalBufferLockDecompress, m_pExternalMemoryBuffer != NULL);
AZ::IO::ZipDir::ZipDirStructuresInternal::ZlibInflateElementPartial_Impl(
&readStatus,
m_pZlibStream,
m_pLookahead,
(uint8_t*)m_pReadMemoryBuffer + nBytesDecomped,
m_nFileSize - nBytesDecomped,
m_bWriteOnlyExternal,
(uint8_t*)pSrc + nOffs,
nBytes,
&nBytesDecomped
);
}
m_nBytesDecompressed = nBytesDecomped;
//inform listen, so aysnc callback does not overlap
#ifdef STREAMENGINE_ENABLE_LISTENER
inflateListener.End();
#endif
if (readStatus == Z_OK || readStatus == Z_STREAM_END)
{
#if defined(STREAMENGINE_ENABLE_TIMING)
LARGE_INTEGER liEnd, liFreq;
QueryPerformanceCounter(&liEnd);
QueryPerformanceFrequency(&liFreq);
m_unzipTime += CTimeValue((int64)((liEnd.QuadPart - liStart.QuadPart) * CTimeValue::TIMEVALUE_PRECISION / liFreq.QuadPart));
#endif
}
else
{
#ifndef _RELEASE
AZ_Assert(false, "Decomp Error: %s : %s\n", m_strFileName.c_str(), m_pZlibStream ? m_pZlibStream->msg : "m_pZlibStream == NULL, no message available");
#endif
Failed(ERROR_DECOMPRESSION_FAIL);
}
}
if (pSrcHdr)
{
if (CryInterlockedDecrement(&pSrcHdr->nRefs) == 0)
{
engineState.pTempMem->TempFree(engineState.pHeap, pSrc, pSrcHdr->nSize);
}
}
job.pSrc = NULL;
int nPopSlot = m_pDecompQueue->Pop();
// job is no longer valid
if (HasFailed() || bLast)
{
JobFinalize_Decompress(pSelf, engineState);
}
else if (nPopSlot >= 0)
{
// Chain start the next job, we're responsible for it.
STREAM_DECOMPRESS_TRACE("[StreamDecompress],Chaining,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nPopSlot);
JobStart_Decompress(pSelf, engineState, nPopSlot);
}
#if defined(STREAMENGINE_ENABLE_STATS)
CryInterlockedDecrement(&engineState.pStats->nCurrentDecompressCount);
#endif
}
//////////////////////////////////////////////////////////////////////////
uint32 CAsyncIOFileRequest::PushDecompressPage(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nBytes, bool bLast)
{
uint32 nError = 0;
for (uint32 nBlockPos = 0; !nError && (nBlockPos < nBytes); nBlockPos += STREAMING_BLOCK_SIZE)
{
bool bLastBlock = (nBlockPos + STREAMING_BLOCK_SIZE) >= nBytes;
uint32 nBlockSize = min(nBytes - nBlockPos, (uint32)STREAMING_BLOCK_SIZE);
nError = PushDecompressBlock(engineState, pSrc, pSrcHdr, nBlockPos, nBlockSize, bLast && bLastBlock);
}
return nError;
}
uint32 CAsyncIOFileRequest::PushDecompressBlock(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nOffs, uint32 nBytes, bool bLast)
{
uint32 nError = m_nError;
if (!nError)
{
if (pSrcHdr)
{
CryInterlockedIncrement(&pSrcHdr->nRefs);
}
int nPushJob = m_pDecompQueue->Push(pSrc, pSrcHdr, nOffs, nBytes, bLast);
if (nPushJob >= 0)
{
STREAM_DECOMPRESS_TRACE("[StreamDecompress],PushDecompressBlock,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nPushJob);
AddRef();
CAsyncIOFileRequest_TransferPtr pSelf(this);
JobStart_Decompress(pSelf, engineState, nPushJob);
}
}
return nError;
}
void CAsyncIOFileRequest::JobStart_Decompress(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState, int nJob)
{
STREAM_DECOMPRESS_TRACE("[StreamDecompress],QueueDecompressBlockAppend,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), pSelf->m_strFileName.c_str(), &pSelf, nJob);
#if defined(STREAMENGINE_ENABLE_STATS)
CryInterlockedIncrement(&engineState.pStats->nCurrentDecompressCount);
#endif
CAsyncIOFileRequest* request = pSelf.Relinquish();
if (!request->m_decompJobExecutor)
{
request->m_decompJobExecutor = AZStd::make_unique<AZ::LegacyJobExecutor>();
}
request->m_decompJobExecutor->StartJob([request, engineState, nJob]()
{
request->DecompressBlockEntry(engineState, nJob);
}); // Legacy JobManager priority: eStreamPriority
}
//////////////////////////////////////////////////////////////////////////
void CAsyncIOFileRequest::JobFinalize_Read(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState)
{
if (!pSelf->m_bCompressedBuffer || pSelf->HasFailed())
{
JobFinalize_Transfer(pSelf, engineState);
}
}
void CAsyncIOFileRequest::JobFinalize_Decompress(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState)
{
STREAM_DECOMPRESS_TRACE("[StreamDecompress],FinalizeDecompress,0x%x,%s,0x%p,0x%p,0x%p,0x%p\n", CryGetCurrentThreadId(), pSelf->m_strFileName.c_str(), &pSelf, &engineState, engineState.pStats, engineState.pDecompressStats);
CAsyncIOFileRequest* pReq = &*pSelf;
if (!pReq->HasFailed())
{
// Handle reads of subsections of a compressed file, by copying the section to the output
uint8_t* pDst = (uint8_t*)pReq->m_pOutputMemoryBuffer;
uint8_t* pSrc = (uint8_t*)pReq->m_pReadMemoryBuffer + pReq->m_nRequestedOffset;
if (pDst != pSrc)
{
memmove(pReq->m_pOutputMemoryBuffer, pSrc, pReq->m_nRequestedSize);
}
pReq->JobFinalize_Validate(engineState);
}
pReq->JobFinalize_Buffer(engineState);
#if defined(STREAMENGINE_ENABLE_STATS) && defined(STREAMENGINE_ENABLE_TIMING)
if (pReq->m_unzipTime.GetValue() != 0)
{
engineState.pDecompressStats->m_nTotalBytesUnziped += pReq->m_nFileSize;
engineState.pDecompressStats->m_totalUnzipTime += pReq->m_unzipTime;
engineState.pDecompressStats->m_nTempBytesUnziped += pReq->m_nFileSize;
engineState.pDecompressStats->m_tempUnzipTime += pReq->m_unzipTime;
}
#endif
JobFinalize_Transfer(pSelf, engineState);
}
void CAsyncIOFileRequest::JobFinalize_Buffer(const SStreamJobEngineState& engineState)
{
if (CryInterlockedDecrement(&m_nMemoryBufferUsers) == 0)
{
z_stream_s* pZlib = m_pZlibStream;
if (pZlib)
{
//if the stream was cancelled in flight, inform zlib to free internal allocs
if (pZlib->state)
{
inflateEnd(pZlib);
}
m_pZlibStream = NULL;
}
if (m_pMemoryBuffer)
{
engineState.pTempMem->TempFree(engineState.pHeap, m_pMemoryBuffer, m_nMemoryBufferSize);
m_pMemoryBuffer = NULL;
m_nMemoryBufferSize = 0;
}
}
}
void CAsyncIOFileRequest::JobFinalize_Validate([[maybe_unused]] const SStreamJobEngineState& engineState)
{
#if defined(SKIP_CHECKSUM_FROM_OPTICAL_MEDIA)
if (m_eMediaType != eStreamSourceTypeDisc)
#endif //SKIP_CHECKSUM_FROM_OPTICAL_MEDIA
{
CryOptionalAutoLock<CryCriticalSection> readLock(m_externalBufferLockRead, m_pExternalMemoryBuffer != NULL);
if (!HasFailed())
{
if (m_crc32FromHeader != 0 && m_nPageReadStart == 0 && m_nRequestedSize == m_nFileSize) //Compute the CRC32 if appropriate.
{
#if defined(STREAMENGINE_ENABLE_TIMING)
LARGE_INTEGER liStart, liEnd, liFreq;
QueryPerformanceCounter(&liStart);
#endif //STREAMENGINE_ENABLE_TIMING
uint32 nCRC32 = crc32(0, (uint8_t*)m_pReadMemoryBuffer + m_nPageReadStart, m_nRequestedSize);
#if defined(STREAMENGINE_ENABLE_TIMING)
QueryPerformanceCounter(&liEnd);
QueryPerformanceFrequency(&liFreq);
m_verifyTime += CTimeValue((int64)((liEnd.QuadPart - liStart.QuadPart) * CTimeValue::TIMEVALUE_PRECISION / liFreq.QuadPart));
engineState.pDecompressStats->m_nTotalBytesVerified += m_nFileSize;
engineState.pDecompressStats->m_totalVerifyTime += m_verifyTime;
engineState.pDecompressStats->m_nTempBytesVerified += m_nFileSize;
engineState.pDecompressStats->m_tempVerifyTime += m_verifyTime;
#endif //STREAMENGINE_ENABLE_TIMING
if (m_crc32FromHeader != nCRC32)
{
//The contents of this file don't match what the header expects
#if !defined(_RELEASE)
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_ERROR_DBGBRK, "Streaming Engine Failed to verify a file (%s). Computed CRC32 %d does not match stored CRC32 %d", m_strFileName.c_str(), nCRC32, m_crc32FromHeader);
#endif //!_RELEASE
Failed(ERROR_VERIFICATION_FAIL);
}
}
}
}
}
void CAsyncIOFileRequest::JobFinalize_Transfer(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState)
{
STREAM_DECOMPRESS_TRACE("[StreamDecompress],FinalizeTransform,0x%x,%s,0x%p,0x%p,0x%p,0x%p\n", CryGetCurrentThreadId(), pSelf->m_strFileName.c_str(), &pSelf, &engineState, engineState.pStats, engineState.pDecompressStats);
if (CryInterlockedCompareExchange(&pSelf->m_nFinalised, 1, 0) == 0)
{
#if defined(STREAMENGINE_ENABLE_STATS)
CryInterlockedIncrement(&engineState.pStats->nCurrentAsyncCount);
#endif
#if defined(STREAMENGINE_ENABLE_TIMING)
pSelf->m_completionTime = gEnv->pTimer->GetAsyncTime();
#endif
int nCallbackThreads = engineState.pReportQueues->size();
EStreamTaskType eType = pSelf->m_eType;
if (nCallbackThreads > 1 && eType == eStreamTaskTypeGeometry)
{
// If we have more then 1 call back threads, use this one for geometry only.
(*engineState.pReportQueues)[1]->TransferRequest(pSelf);
}
else if (nCallbackThreads > 2 && eType == eStreamTaskTypeTexture)
{
// If we have more then 1 call back threads, use this one for textures only.
(*engineState.pReportQueues)[2]->TransferRequest(pSelf);
}
else if (nCallbackThreads > 3 && eType == eStreamTaskTypeMergedMesh)
{
// If we have more then 3 call back threads, use this one for merged meshes only.
(*engineState.pReportQueues)[3]->TransferRequest(pSelf);
}
else if (nCallbackThreads > 0)
{
(*engineState.pReportQueues)[0]->TransferRequest(pSelf);
}
else
{
__debugbreak();
}
}
}
//////////////////////////////////////////////////////////////////////////
void SStreamRequestQueue::TransferRequest(CAsyncIOFileRequest_TransferPtr& pRequest)
{
{
CryAutoLock<CryCriticalSection> l(m_lock);
m_requests.push_back(pRequest.Relinquish());
}
m_awakeEvent.Set();
}
//////////////////////////////////////////////////////////////////////////
void SStreamEngineTempMemStats::TempFree(CMTSafeHeap* pHeap, const void* p, size_t nSize)
{
#if MTSAFE_USE_GENERAL_HEAP
bool bInGenHeap = pHeap->IsInGeneralHeap(p);
#else
bool bInGenHeap = false;
#endif
pHeap->FreeTemporary(const_cast<void*>(p));
ReportTempMemAlloc(0, bInGenHeap ? 0 : nSize, true);
}
void SStreamEngineTempMemStats::ReportTempMemAlloc(uint32 nSizeAlloc, uint32 nSizeFree, bool bTriggerWake)
{
int nAdd = (int)nSizeAlloc - (int)nSizeFree;
int const nOldSize = CryInterlockedExchangeAdd(&m_nTempAllocatedMemory, nAdd);
int const nNewSize = nOldSize + nAdd;
LONG nNewMax = 0;
LONG nOldMax = 0;
do
{
nOldMax = m_nTempAllocatedMemoryFrameMax;
nNewMax = (LONG)max((int)nNewSize, (int)nOldMax);
}
while (CryInterlockedCompareExchange(&m_nTempAllocatedMemoryFrameMax, nNewMax, nOldMax) != nOldMax);
if (bTriggerWake)
{
for (int i = 0, c = m_nWakeEvents; i != c; ++i)
{
m_wakeEvents[i]->Set();
}
}
}
Code/CryEngine/CrySystem/StreamEngine/StreamEngine.cpp
-1698
View File
File diff suppressed because it is too large Load Diff
Code/CryEngine/CrySystem/StreamEngine/StreamEngine.h
-239
View File
@@ -1,239 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Engine
#ifndef CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMENGINE_H
#define CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMENGINE_H
#pragma once
#include "IStreamEngine.h"
#include "ISystem.h"
#include "TimeValue.h"
#include <CryThread.h>
#include "StreamIOThread.h"
#include "StreamReadStream.h"
#include <AzFramework/Input/Events/InputChannelEventListener.h>
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/std/chrono/clocks.h>
#include <AzCore/std/containers/queue.h>
enum EIOThread
{
eIOThread_HDD = 0,
eIOThread_Optical = 1,
eIOThread_InMemory = 2,
eIOThread_Last = 3,
};
//////////////////////////////////////////////////////////////////////////
class CStreamEngine
: public IStreamEngine
, public ISystemEventListener
, public AzFramework::InputChannelEventListener
{
public:
CStreamEngine();
~CStreamEngine();
void Shutdown();
// This is called to cancel all pending requests, without sending callbacks.
void CancelAll();
//Helper added to aid in migration from Cry's CStreamEngine to AZ::IO::Streamer
static AZ::IO::IStreamerTypes::Priority CryStreamPriorityToAZStreamPriority(EStreamTaskPriority cryPriority);
static AZStd::chrono::milliseconds AZDeadlineFromReadParams(const StreamReadParams& params);
//////////////////////////////////////////////////////////////////////////
// IStreamEngine interface
//////////////////////////////////////////////////////////////////////////
IReadStreamPtr StartRead (const EStreamTaskType tSource, const char* szFile, IStreamCallback* pCallback, const StreamReadParams* pParams = NULL);
size_t StartBatchRead(IReadStreamPtr* pStreamsOut, const StreamReadBatchParams* pReqs, size_t numReqs, AZStd::function<void()>* preRequestCallback = nullptr);
void BeginReadGroup();
void EndReadGroup();
bool IsStreamDataOnHDD() const { return m_bStreamDataOnHDD; }
void SetStreamDataOnHDD(bool bFlag) { m_bStreamDataOnHDD = bFlag; }
void Update();
void UpdateAndWait(bool bAbortAll = false);
void Update(uint32 nUpdateTypesBitmask);
void GetMemoryStatistics(ICrySizer* pSizer);
#if defined(STREAMENGINE_ENABLE_STATS)
SStreamEngineStatistics& GetStreamingStatistics();
void ClearStatistics();
void GetBandwidthStats(EStreamTaskType type, float* bandwidth);
#endif
void GetStreamingOpenStatistics(SStreamEngineOpenStats& openStatsOut);
const char* GetStreamTaskTypeName(EStreamTaskType type);
SStreamJobEngineState GetJobEngineState();
SStreamEngineTempMemStats& GetTempMemStats() { return m_tempMem; }
// Will pause or unpause streaming of specified by mask data types
void PauseStreaming(bool bPause, uint32 nPauseTypesBitmask);
// Pause/resumes any IO active from the streaming engine
void PauseIO(bool bPause);
uint32 GetPauseMask() const { return m_nPausedDataTypesMask; }
#if defined(STREAMENGINE_ENABLE_LISTENER)
void SetListener(IStreamEngineListener* pListener);
IStreamEngineListener* GetListener();
#endif
//////////////////////////////////////////////////////////////////////////
// updates the job priority of an IO job into the IOQueue while maintaining order in the queue
void UpdateJobPriority(IReadStreamPtr pJobStream);
void ReportAsyncFileRequestComplete(CAsyncIOFileRequest_AutoPtr pFileRequest);
void AbortJob(CReadStream* pStream);
// Dispatches synchrnous callbacks, free temporary memory hold for callbacks.
void MainThread_FinalizeIOJobs();
void MainThread_FinalizeIOJobs(uint32 type);
void* TempAlloc(size_t nSize, const char* szDbgSource, bool bFallBackToMalloc = true, bool bUrgent = false, uint32 align = 0);
void TempFree(void* p, size_t nSize);
uint32 GetCurrentTempMemorySize() const { return m_tempMem.m_nTempAllocatedMemory; }
void FlagTempMemOutOfBudget()
{
#ifdef STREAMENGINE_ENABLE_STATS
m_bTempMemOutOfBudget = true;
#endif
}
//////////////////////////////////////////////////////////////////////////
// AzFramework::InputChannelEventListener
//////////////////////////////////////////////////////////////////////////
bool OnInputChannelEventFiltered(const AzFramework::InputChannel& inputChannel) override;
//////////////////////////////////////////////////////////////////////////
bool StartFileRequest(CAsyncIOFileRequest* pFileRequest);
void SignalToStartWork(EIOThread e, bool bForce);
private:
void StartThreads();
void StopThreads();
void ResumePausedStreams_PauseLocked();
#if defined(STREAMENGINE_ENABLE_STATS)
// add job to current statistics
void UpdateStatistics(CReadStream* pReadStream);
void DrawStatistics();
#endif
void QueueRequestCompleteJob(class AZRequestReadStream* stream, AZ::IO::SizeType numBytesRead, void* buffer,
AZ::IO::IStreamerTypes::RequestStatus requestState);
//////////////////////////////////////////////////////////////////////////
// ISystemEventListener
//////////////////////////////////////////////////////////////////////////
virtual void OnSystemEvent(ESystemEvent event, UINT_PTR wparam, UINT_PTR lparam);
//////////////////////////////////////////////////////////////////////////
private:
//////////////////////////////////////////////////////////////////////////
CryMT::set<CReadStream_AutoPtr> m_streams;
CryMT::vector<CReadStream_AutoPtr> m_finishedStreams;
std::vector<CReadStream_AutoPtr> m_tempFinishedStreams;
CryCriticalSection m_pendingRequestCompletionsLock;
AZStd::queue<AZ::Job*> m_pendingRequestCompletions;
// 2 IO threads.
_smart_ptr<CStreamingIOThread> m_pThreadIO[eIOThread_Last];
std::vector<_smart_ptr<CStreamingWorkerThread> > m_asyncCallbackThreads;
std::vector<SStreamRequestQueue*> m_asyncCallbackQueues;
CryCriticalSection m_pausedLock;
std::vector<CReadStream_AutoPtr> m_pausedStreams;
volatile uint32 m_nPausedDataTypesMask;
bool m_bStreamDataOnHDD;
bool m_bUseOpticalDriveThread;
//////////////////////////////////////////////////////////////////////////
// Streaming statistics.
//////////////////////////////////////////////////////////////////////////
#ifdef STREAMENGINE_ENABLE_LISTENER
IStreamEngineListener* m_pListener;
#endif
#ifdef STREAMENGINE_ENABLE_STATS
SStreamEngineStatistics m_Statistics;
SStreamEngineDecompressStats m_decompressStats;
CTimeValue m_TimeOfLastReset;
CTimeValue m_TimeOfLastUpdate;
CryCriticalSection m_csStats;
std::vector<CAsyncIOFileRequest_AutoPtr> m_statsRequestList;
struct SExtensionInfo
{
SExtensionInfo()
: m_fTotalReadTime(0.0f)
, m_nTotalRequests(0)
, m_nTotalReadSize(0)
, m_nTotalRequestSize(0)
{
}
float m_fTotalReadTime;
size_t m_nTotalRequests;
uint64 m_nTotalReadSize;
uint64 m_nTotalRequestSize;
};
typedef std::map<string, SExtensionInfo> TExtensionInfoMap;
TExtensionInfoMap m_PerExtensionInfo;
//////////////////////////////////////////////////////////////////////////
// Used to calculate unzip/verify bandwidth for statistics.
uint32 m_nUnzipBandwidth;
uint32 m_nUnzipBandwidthAverage;
uint32 m_nVerifyBandwidth;
uint32 m_nVerifyBandwidthAverage;
CTimeValue m_nLastBandwidthUpdateTime;
bool m_bStreamingStatsPaused;
bool m_bInputCallback;
bool m_bTempMemOutOfBudget;
//////////////////////////////////////////////////////////////////////////
#endif
SStreamEngineOpenStats m_OpenStatistics;
bool m_bShutDown;
volatile int m_nBatchMode;
// Memory currently allocated by streaming engine for temporary storage.
SStreamEngineTempMemStats m_tempMem;
};
#endif // CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMENGINE_H
Code/CryEngine/CrySystem/StreamEngine/StreamIOThread.cpp
-819
View File
@@ -1,819 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Thread for IO
#include "CrySystem_precompiled.h"
#include "StreamIOThread.h"
#include "StreamEngine.h"
#include "../System.h"
extern SSystemCVars g_cvars;
//#pragma("control %push O=0") // to disable optimization
//////////////////////////////////////////////////////////////////////////
CStreamingIOThread::CStreamingIOThread(CStreamEngine* pStreamEngine, EStreamSourceMediaType mediaType, const char* name)
{
m_pStreamEngine = pStreamEngine;
m_bCancelThreadRequest = false;
m_bNeedSorting = false;
m_bNeedReset = false;
m_bNewRequests = false;
m_name = name;
m_eMediaType = mediaType;
m_nFallbackMTs = 0;
m_iUrgentRequests = 0;
m_bPaused = false;
m_bAbortReads = false;
m_nReadCounter = 0;
m_nStreamingCPU = -1;
Start((unsigned)(1 << g_cvars.sys_streaming_cpu), name);
}
CStreamingIOThread::~CStreamingIOThread()
{
Cancel();
Stop();
WaitForThread();
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::AddRequest(CAsyncIOFileRequest* pRequest, bool bStartImmidietly)
{
pRequest->AddRef(); // Acquire ownership on file request.
pRequest->m_status = CAsyncIOFileRequest::eStatusInFileQueue;
if (pRequest->m_eMediaType != eStreamSourceTypeMemory)
{
pRequest->m_eMediaType = m_eMediaType;
}
// does this ignore the tmp out of memory
if (pRequest->IgnoreOutofTmpMem())
{
CryInterlockedIncrement(&m_iUrgentRequests);
}
m_newFileRequests.push_back(pRequest);
if (bStartImmidietly)
{
READ_WRITE_BARRIER
m_bNewRequests = true;
m_awakeEvent.Set();
}
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::SignalStartWork(bool bForce)
{
if (!m_newFileRequests.empty() || bForce)
{
READ_WRITE_BARRIER
m_bNewRequests = true;
m_awakeEvent.Set();
}
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::Pause(bool bPause)
{
m_bPaused = bPause;
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::Run()
{
SetName(m_name);
CTimeValue t0 = gEnv->pTimer->GetAsyncTime();
m_nLastReadDiskOffset = 0;
//
// Main thread loop
while (!m_bCancelThreadRequest)
{
if (m_nStreamingCPU != g_cvars.sys_streaming_cpu)
{
m_nStreamingCPU = g_cvars.sys_streaming_cpu;
#if defined(AZ_RESTRICTED_PLATFORM)
#undef AZ_RESTRICTED_SECTION
#define STREAMIOTHREAD_CPP_SECTION_1 1
#define STREAMIOTHREAD_CPP_SECTION_2 2
#endif
#if defined(AZ_RESTRICTED_PLATFORM)
#define AZ_RESTRICTED_SECTION STREAMIOTHREAD_CPP_SECTION_1
#include AZ_RESTRICTED_FILE(StreamEngine/StreamIOThread_cpp)
#endif
}
if (m_bNewRequests || !m_newFileRequests.empty())
{
READ_WRITE_BARRIER
ProcessNewRequests();
}
else
{
#if defined(_RELEASE)
m_awakeEvent.Wait();
#elif defined(STREAMENGINE_ENABLE_STATS)
// compute max time to wait - revive thread every second at least once to update stats
bool bWaiting = true;
while (bWaiting)
{
CTimeValue t1 = gEnv->pTimer->GetAsyncTime();
CTimeValue deltaT = t1 - t0;
uint64 msec = deltaT.GetMilliSecondsAsInt64();
if (msec < 1000)
{
bWaiting = !m_awakeEvent.Wait(1000 - (uint32)msec);
}
if (bWaiting)
{
// update the delta time again
t1 = gEnv->pTimer->GetAsyncTime();
deltaT = t1 - t0;
m_InMemoryStats.Update(deltaT);
m_NotInMemoryStats.Update(deltaT);
t0 = t1;
}
}
#endif
}
if (m_bNeedReset)
{
ProcessReset();
}
bool bIsOOM = false;
while (!m_bCancelThreadRequest && !m_fileRequestQueue.empty())
{
CAsyncIOFileRequest_TransferPtr pFileRequest(m_fileRequestQueue.back());
m_fileRequestQueue.pop_back();
assert (&*pFileRequest);
if (pFileRequest->HasFailed())
{
// check if request was high prio, then decr open count
if (pFileRequest->IgnoreOutofTmpMem())
{
CryInterlockedDecrement(&m_iUrgentRequests);
}
CAsyncIOFileRequest::JobFinalize_Read(pFileRequest, m_pStreamEngine->GetJobEngineState());
continue;
}
//////////////////////////////////////////////////////////////////////////
// When temporary memory goes out of budget we must loop here and wait until previous file requests are finished and free up memory.
// Only allow processing of requests which are flagged for processing when out of tmp memory
//////////////////////////////////////////////////////////////////////////
if (bIsOOM && !m_bCancelThreadRequest)
{
m_pStreamEngine->FlagTempMemOutOfBudget();
if (m_iUrgentRequests > 0)
{
if (m_bNewRequests || !m_newFileRequests.empty())
{
READ_WRITE_BARRIER
ProcessNewRequests();
}
// readd the current request
m_fileRequestQueue.push_back(pFileRequest.Relinquish());
// search for the first request which ignores the current out of mem state
// Search for next highest priority request
std::vector<CAsyncIOFileRequest*>::reverse_iterator rit;
for (rit = m_fileRequestQueue.rbegin(); rit != m_fileRequestQueue.rend(); ++rit)
{
if ((*rit)->IgnoreOutofTmpMem())
{
pFileRequest = *rit;
std::vector<CAsyncIOFileRequest*>::iterator it(rit.base());
--it;
m_fileRequestQueue.erase(it);
break;
}
}
}
else
{
// read the current request
m_fileRequestQueue.push_back(pFileRequest.Relinquish());
}
}
// Simply let the io thread sleep when paused before doing any actual IO
while (m_bPaused)
{
CrySleep(10);
}
// If at this point, the filerequest is zero, the above prioritization of
// urgent requests couldn't find a new task to displace the current
// one. As the current one had been pushed back previously, we can safely
// assume that restarting the loop will grab it again (eventually).
if (!pFileRequest)
{
break;
}
// check if request was high prio, then decr open count
if (pFileRequest->IgnoreOutofTmpMem())
{
CryInterlockedDecrement(&m_iUrgentRequests);
}
bIsOOM = false;
uint32 nSizeOnMedia = pFileRequest->m_nSizeOnMedia;
uint32 nError = 0;
// Handle file request.
if (m_bAbortReads)
{
nError = ERROR_ABORTED_ON_SHUTDOWN;
}
else if (pFileRequest->m_bReadBegun)
{
nError = pFileRequest->ReadFileResume(this);
}
else
{
nError = pFileRequest->ReadFile(this);
}
#ifdef STREAMENGINE_ENABLE_STATS
pFileRequest->m_nReadCounter = m_nReadCounter++;
#endif
if (nError == 0)
{
if (pFileRequest->m_eMediaType != eStreamSourceTypeMemory)
{
pFileRequest->m_nReadHeadOffsetKB = (int32)(((int64)pFileRequest->m_nDiskOffset - m_nLastReadDiskOffset) >> 10); // in KB
m_nLastReadDiskOffset = pFileRequest->m_nDiskOffset + nSizeOnMedia;
#ifdef STREAMENGINE_ENABLE_STATS
m_NotInMemoryStats.m_nTempReadOffset += abs(pFileRequest->m_nReadHeadOffsetKB);
m_NotInMemoryStats.m_nTotalReadOffset += abs(pFileRequest->m_nReadHeadOffsetKB);
m_NotInMemoryStats.m_nTempRequestCount++;
// Calc IO bandwidth only for non memory files.
m_NotInMemoryStats.m_nTempBytesRead += nSizeOnMedia;
m_NotInMemoryStats.m_TempReadTime += pFileRequest->m_readTime;
#endif
}
else
{
#ifdef STREAMENGINE_ENABLE_STATS
m_InMemoryStats.m_nTempRequestCount++;
// Calc IO bandwidth only for in memory files.
m_InMemoryStats.m_nTempBytesRead += nSizeOnMedia;
m_InMemoryStats.m_TempReadTime += pFileRequest->m_readTime;
#endif
}
CAsyncIOFileRequest::JobFinalize_Read(pFileRequest, m_pStreamEngine->GetJobEngineState());
}
else
{
switch (nError)
{
case ERROR_OUT_OF_MEMORY:
bIsOOM = true;
pFileRequest->SetPriority(estpPreempted);
if (pFileRequest->IgnoreOutofTmpMem())
{
CryInterlockedIncrement(&m_iUrgentRequests);
}
m_fileRequestQueue.push_back(pFileRequest.Relinquish());
m_bNewRequests = true;
break;
case ERROR_PREEMPTED:
pFileRequest->SetPriority(estpPreempted);
if (pFileRequest->IgnoreOutofTmpMem())
{
CryInterlockedIncrement(&m_iUrgentRequests);
}
m_fileRequestQueue.push_back(pFileRequest.Relinquish());
m_bNewRequests = true;
break;
case ERROR_MISSCHEDULED:
// Request tried to read a file that has changed media type. Reset the sort key
// and reschedule.
pFileRequest->m_bSortKeyComputed = 0;
AddRequest(&*pFileRequest, false);
break;
default:
pFileRequest->SyncWithDecompress();
pFileRequest->Failed(nError);
CAsyncIOFileRequest::JobFinalize_Read(pFileRequest, m_pStreamEngine->GetJobEngineState());
break;
}
}
//////////////////////////////////////////////////////////////////////////
if (m_bNewRequests)
{
READ_WRITE_BARRIER
ProcessNewRequests();
}
if (m_bNeedReset)
{
ProcessReset();
}
if (m_bNeedSorting)
{
SortRequests();
}
//////////////////////////////////////////////////////////////////////////
#ifdef STREAMENGINE_ENABLE_STATS
if (g_cvars.sys_streaming_max_bandwidth != 0)
{
CTimeValue t1 = gEnv->pTimer->GetAsyncTime();
CTimeValue deltaT = t1 - t0;
// Sleep in case we are streaming too fast.
const float fTheoreticalReadTime = float(nSizeOnMedia) / g_cvars.sys_streaming_max_bandwidth * 0.00000095367431640625f; // / (1024*1024)
if (fTheoreticalReadTime - deltaT.GetSeconds() > FLT_EPSILON)
{
uint32 nSleepTime = uint32(1000.f * (fTheoreticalReadTime - deltaT.GetSeconds()));
CrySleep(nSleepTime);
}
}
CTimeValue t1 = gEnv->pTimer->GetAsyncTime();
CTimeValue deltaT = t1 - t0;
// update the stats every second
if (deltaT.GetMilliSecondsAsInt64() > 1000)
{
m_InMemoryStats.Update(deltaT);
m_NotInMemoryStats.Update(deltaT);
t0 = t1;
}
#endif
}
}
}
#ifdef STREAMENGINE_ENABLE_STATS
void CStreamingIOThread::SStats::Update(const CTimeValue& deltaT)
{
m_nReadBytesInLastSecond = (uint32)m_nTempBytesRead;
m_nRequestCountInLastSecond = m_nTempRequestCount;
m_nTotalReadBytes += (uint32)m_nTempBytesRead;
m_nTotalRequestCount += m_nTempRequestCount;
m_TotalReadTime += m_TempReadTime;
if (m_TempReadTime.GetValue() != 0)
{
m_nActualReadBandwith = (uint32)(m_nTempBytesRead / m_TempReadTime.GetSeconds());
}
else
{
m_nActualReadBandwith = 0;
}
m_nCurrentReadBandwith = (uint32)(m_nTempBytesRead / deltaT.GetSeconds());
m_fReadingDuringLastSecond = m_TempReadTime.GetSeconds() / deltaT.GetSeconds() * 100;
if (m_nTempRequestCount > 0)
{
m_nReadOffsetInLastSecond = m_nTempReadOffset / m_nTempRequestCount;
}
else
{
m_nReadOffsetInLastSecond = 0;
}
m_TempReadTime.SetValue(0);
m_nTempBytesRead = 0;
m_nTempReadOffset = 0;
m_nTempRequestCount = 0;
}
#endif
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::Cancel()
{
m_bCancelThreadRequest = true;
m_awakeEvent.Set();
}
//////////////////////////////////////////////////////////////////////////
struct SCompareAsyncFileRequest
{
bool operator()(CAsyncIOFileRequest* pFile1, CAsyncIOFileRequest* pFile2) const
{
return pFile1->m_nSortKey > pFile2->m_nSortKey;
}
};
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::SortRequests()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
std::sort(m_fileRequestQueue.begin(), m_fileRequestQueue.end(), SCompareAsyncFileRequest());
/*
int nStartOfQueue = 0;
int64 nDiskOffsetLimit = m_nLastReadDiskOffset - 32*1024; // 32KB less only
int nCount = (int)m_fileRequestQueue.size();
for (int i = nCount-1; i >= 0; i--)
{
if (m_fileRequestQueue[i]->m_nDiskOffset > nDiskOffsetLimit)
{
nStartOfQueue = i+1;
break;
}
}
if (nStartOfQueue < nCount && nStartOfQueue > 0)
{
int nElements = nCount - nStartOfQueue;
// Move all elements up to nStartOfQueue, from begining of the request array to the end.
m_temporaryArray.resize(0);
// Copy to temp array elements up to nStartOfQueue
m_temporaryArray.insert( m_temporaryArray.end(),m_fileRequestQueue.begin()+nStartOfQueue,m_fileRequestQueue.end() );
// Remove elements up to nStartOfQueue from request list
m_fileRequestQueue.erase( m_fileRequestQueue.begin()+nStartOfQueue,m_fileRequestQueue.end() );
// Add elemenets at the end from temp array.
m_fileRequestQueue.insert( m_fileRequestQueue.begin(),m_temporaryArray.begin(),m_temporaryArray.end() );
}
*/
m_bNeedSorting = false;
}
void CStreamingIOThread::NeedSorting()
{
m_bNeedSorting = true;
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::ProcessNewRequests()
{
m_bNewRequests = false;
std::vector<CAsyncIOFileRequest*> temporaryArray;
temporaryArray.reserve(m_newFileRequests.size());
m_newFileRequests.swap(temporaryArray);
std::vector<CAsyncIOFileRequest*>& newFiles = temporaryArray;
if (!newFiles.empty())
{
uint64 nCurrentKeyInProgress = m_fileRequestQueue.size() ? m_fileRequestQueue.back()->m_nSortKey : 0;
// Compute sorting key for new file entries.
int iWakeFallback(0);
const TFallbackIOVecConstIt itEnd = m_FallbackIOThreads.end();
const size_t fallbackNum = m_FallbackIOThreads.size();
PREFAST_SUPPRESS_WARNING(6255)
uint8 * pFallbackSignals = fallbackNum ? (uint8*)alloca(fallbackNum) : NULL;
for (uint32 fb = 0; fb < fallbackNum; ++fb)
{
pFallbackSignals[fb] = 0;
}
for (size_t i = 0, num = newFiles.size(); i < num; i++)
{
CAsyncIOFileRequest* pFilepRequest = newFiles[i];
pFilepRequest->ComputeSortKey(nCurrentKeyInProgress);
static_cast<CReadStream*>(&*pFilepRequest->m_pReadStream)->ComputedMediaType(pFilepRequest->m_eMediaType);
#ifdef STREAMENGINE_ENABLE_LISTENER
IStreamEngineListener* pListener = m_pStreamEngine->GetListener();
if (pListener)
{
pListener->OnStreamComputedSortKey(pFilepRequest, pFilepRequest->m_nSortKey);
}
#endif
bool bFallback = false;
int idx = -1;
for (TFallbackIOVecConstIt it = m_FallbackIOThreads.begin(); it != itEnd && !bFallback; ++it)
{
++idx;
if (it->second == pFilepRequest->GetMediaType())
{
if (pFilepRequest->IgnoreOutofTmpMem())
{
CryInterlockedDecrement(&m_iUrgentRequests);
}
(it->first)->AddRequest(pFilepRequest, true);
pFilepRequest->Release(); // Release local ownership of request (moved to fallback IO thread)
iWakeFallback++;
bFallback = true;
pFallbackSignals[idx] = 1;
}
}
if (!bFallback)
{
m_fileRequestQueue.push_back(pFilepRequest);
}
}
for (uint32 fb = 0; fb < fallbackNum; ++fb)
{
if (pFallbackSignals[fb] != 0)
{
(m_FallbackIOThreads[fb].first)->SignalStartWork(false);
}
}
SortRequests();
/*
if (m_fileRequestQueue.back() != pRequest && pRequest != 0)
{
// Highest priority changed.
if (m_fileRequestQueue.back()->m_nDiskOffset < (m_nLastReadDiskOffset-32*1024))
{
//CryLog( "Bad Offset in Queue" );
}
}
*/
}
}
void CStreamingIOThread::ProcessReset()
{
if (!m_fileRequestQueue.empty())
{
for (std::vector<CAsyncIOFileRequest*>::iterator it = m_fileRequestQueue.begin(), itEnd = m_fileRequestQueue.end(); it != itEnd; ++it)
{
(*it)->Release();
}
}
stl::free_container(m_fileRequestQueue);
if (!m_temporaryArray.empty())
{
for (std::vector<CAsyncIOFileRequest*>::iterator it = m_temporaryArray.begin(), itEnd = m_temporaryArray.end(); it != itEnd; ++it)
{
(*it)->Release();
}
}
stl::free_container(m_temporaryArray);
m_bNeedReset = false;
m_resetDoneEvent.Set();
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::CancelAll()
{
{
CryMT::vector<CAsyncIOFileRequest*>::AutoLock lock(m_newFileRequests.get_lock());
if (!m_newFileRequests.empty())
{
CAsyncIOFileRequest* const* it = &m_newFileRequests.front();
CAsyncIOFileRequest* const* itEnd = it + m_newFileRequests.size();
for (; it != itEnd; ++it)
{
(*it)->Release();
}
}
}
m_newFileRequests.free_memory();
m_iUrgentRequests = 0;
}
void CStreamingIOThread::AbortAll(bool bAbort)
{
m_bAbortReads = bAbort;
}
void CStreamingIOThread::BeginReset()
{
CancelAll();
m_resetDoneEvent.Reset();
m_bNeedReset = true;
m_awakeEvent.Set();
}
void CStreamingIOThread::EndReset()
{
m_resetDoneEvent.Wait();
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::RegisterFallbackIOThread(EStreamSourceMediaType mediaType, CStreamingIOThread* pIOThread)
{
//check if media has not yet been registered
if (!pIOThread)
{
return;//no need for NULL register anymore
}
const TFallbackIOVecConstIt itEnd = m_FallbackIOThreads.end();
for (TFallbackIOVecConstIt it = m_FallbackIOThreads.begin(); it != itEnd; ++it)
{
if (it->second == mediaType)
{
return;
}
}
m_FallbackIOThreads.push_back(std::make_pair(pIOThread, mediaType));
m_nFallbackMTs |= 1 << mediaType;
}
bool CStreamingIOThread::HasUrgentRequests()
{
bool ret = false;
if (m_iUrgentRequests > 0)
{
//lock to prevent list modification whilst traversing
m_newFileRequests.get_lock().Lock();
int nRequests = m_newFileRequests.size();
if (nRequests)
{
for (int i = 0; i < nRequests; i++)
{
if (m_newFileRequests[i]->m_ePriority == estpUrgent)
{
//printf("Urgent task pending: %s\n", m_newFileRequests[i]->m_strFileName.c_str());
ret = true;
break;
}
}
}
m_newFileRequests.get_lock().Unlock();
}
return ret;
}
bool CStreamingIOThread::IsMisscheduled(EStreamSourceMediaType mt) const
{
if (mt == m_eMediaType)
{
return false;
}
if (m_nFallbackMTs & (1 << mt))
{
return true;
}
return false;
}
//////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////
CStreamingWorkerThread::CStreamingWorkerThread(CStreamEngine* pStreamEngine, const char* name, EWorkerType type, SStreamRequestQueue* pQueue)
{
m_type = type;
m_name = name;
m_pStreamEngine = pStreamEngine;
m_pQueue = pQueue;
m_bCancelThreadRequest = false;
m_bNeedsReset = false;
Start((unsigned)1 << g_cvars.sys_streaming_cpu_worker, name);
}
CStreamingWorkerThread::~CStreamingWorkerThread()
{
Cancel();
Stop();
WaitForThread();
}
//////////////////////////////////////////////////////////////////////////
void CStreamingWorkerThread::Run()
{
SetName(m_name);
#if defined(AZ_RESTRICTED_PLATFORM)
#define AZ_RESTRICTED_SECTION STREAMIOTHREAD_CPP_SECTION_2
#include AZ_RESTRICTED_FILE(StreamEngine/StreamIOThread_cpp)
#endif
// Main thread loop
while (!m_bCancelThreadRequest)
{
m_pQueue->m_awakeEvent.Wait();
m_pQueue->m_awakeEvent.Reset();
CAsyncIOFileRequest_AutoPtr pFileRequest;
while (!m_bCancelThreadRequest && !m_bNeedsReset && m_pQueue->TryPopRequest(pFileRequest))
{
switch (m_type)
{
case eWorkerAsyncCallback:
{
#ifndef _RELEASE
float fTime = gEnv->pTimer->GetAsyncCurTime();
#endif
m_pStreamEngine->ReportAsyncFileRequestComplete(pFileRequest);
#ifndef _RELEASE
float fTime1 = gEnv->pTimer->GetAsyncCurTime();
#endif
#ifdef STREAMENGINE_ENABLE_STATS
CryInterlockedDecrement(&m_pStreamEngine->GetStreamingStatistics().nCurrentAsyncCount);
#endif
#ifndef _RELEASE
if ((fTime1 - fTime) > 1.f && !pFileRequest->m_strFileName.empty())
{
string str;
str.Format("[ACALL] %s time=%.5f\n", pFileRequest->m_strFileName.c_str(), (fTime1 - fTime));
if (gEnv && gEnv->pSystem && gEnv->pLog)
{
gEnv->pLog->Log(str.c_str());
}
}
#endif
}
break;
}
}
if (m_bNeedsReset)
{
m_pQueue->Reset();
m_bNeedsReset = false;
m_resetDoneEvent.Set();
}
}
}
//////////////////////////////////////////////////////////////////////////
void CStreamingWorkerThread::Cancel()
{
m_bCancelThreadRequest = true;
m_pQueue->m_awakeEvent.Set();
}
//////////////////////////////////////////////////////////////////////////
void CStreamingWorkerThread::CancelAll()
{
m_pQueue->Reset();
}
void CStreamingWorkerThread::BeginReset()
{
CancelAll();
m_resetDoneEvent.Reset();
m_bNeedsReset = true;
m_pQueue->m_awakeEvent.Set();
}
void CStreamingWorkerThread::EndReset()
{
m_resetDoneEvent.Wait();
}
Code/CryEngine/CrySystem/StreamEngine/StreamIOThread.h
-193
View File
@@ -1,193 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Thread for IO
#ifndef CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMIOTHREAD_H
#define CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMIOTHREAD_H
#pragma once
#include <IStreamEngine.h>
#include "StreamAsyncFileRequest.h"
class CStreamEngine;
//////////////////////////////////////////////////////////////////////////
// Thread that performs IO operations.
//////////////////////////////////////////////////////////////////////////
class CStreamingIOThread
: public CrySimpleThread<CStreamingIOThread>
, public CMultiThreadRefCount
{
public:
CStreamingIOThread(CStreamEngine* pStreamEngine, EStreamSourceMediaType mediaType, const char* name);
~CStreamingIOThread();
void CancelAll();
void AbortAll(bool bAbort);
void BeginReset();
void EndReset();
void AddRequest(CAsyncIOFileRequest* pRequest, bool bStartImmidietly);
int GetRequestCount() const { return m_fileRequestQueue.size(); };
void SortRequests();
void NeedSorting();
void SignalStartWork(bool bForce);
bool HasUrgentRequests();
EStreamSourceMediaType GetMediaType() const { return m_eMediaType; }
bool IsMisscheduled(EStreamSourceMediaType mt) const;
void Pause(bool bPause);
void RegisterFallbackIOThread(EStreamSourceMediaType mediaType, CStreamingIOThread* pIOThread);
CStreamEngineWakeEvent& GetWakeEvent() { return m_awakeEvent; }
//////////////////////////////////////////////////////////////////////////
// CrySimpleThread
//////////////////////////////////////////////////////////////////////////
virtual void Run();
virtual void Cancel();
//////////////////////////////////////////////////////////////////////////
protected:
void ProcessNewRequests();
void ProcessReset();
public:
#ifdef STREAMENGINE_ENABLE_STATS
struct SStats
{
SStats()
: m_nTotalReadBytes(0)
, m_nCurrentReadBandwith(0)
, m_nReadBytesInLastSecond(0)
, m_fReadingDuringLastSecond(.0f)
, m_nTempBytesRead(0)
, m_nActualReadBandwith(0)
, m_nTempReadOffset(0)
, m_nTotalReadOffset(0)
, m_nReadOffsetInLastSecond(0)
, m_nTempRequestCount(0)
, m_nTotalRequestCount(0)
, m_nRequestCountInLastSecond(0)
{}
void Update(const CTimeValue& deltaT);
void Reset()
{
m_nTotalReadBytes = 0;
m_nTotalReadOffset = 0;
m_nTotalRequestCount = 0;
m_TotalReadTime.SetValue(0);
}
float m_fReadingDuringLastSecond;
CTimeValue m_TotalReadTime;
uint64 m_nTotalReadBytes;
uint64 m_nTotalReadOffset;
uint32 m_nTotalRequestCount;
uint32 m_nCurrentReadBandwith; // Read bandwidth over one second
uint32 m_nActualReadBandwith; // Actual read bandwidth extrapolated over one second
uint32 m_nReadBytesInLastSecond;
uint32 m_nRequestCountInLastSecond;
uint64 m_nReadOffsetInLastSecond;
uint32 m_nTempRequestCount;
uint64 m_nTempBytesRead;
uint64 m_nTempReadOffset;
CTimeValue m_TempReadTime;
};
SStats m_InMemoryStats;
SStats m_NotInMemoryStats;
#endif
int64 m_nLastReadDiskOffset;
int m_nStreamingCPU;
private:
CStreamEngine* m_pStreamEngine;
std::vector<CAsyncIOFileRequest*> m_fileRequestQueue;
std::vector<CAsyncIOFileRequest*> m_temporaryArray;
CryMT::vector<CAsyncIOFileRequest*> m_newFileRequests;
EStreamSourceMediaType m_eMediaType;
uint32 m_nFallbackMTs;
typedef std::pair<CStreamingIOThread*, EStreamSourceMediaType> TFallbackIOPair;
typedef std::vector<TFallbackIOPair> TFallbackIOVec;
typedef TFallbackIOVec::iterator TFallbackIOVecConstIt;
TFallbackIOVec m_FallbackIOThreads;
volatile bool m_bCancelThreadRequest;
volatile bool m_bNeedSorting;
volatile bool m_bNewRequests;
volatile bool m_bPaused;
volatile bool m_bNeedReset;
volatile bool m_bAbortReads;
volatile int m_iUrgentRequests;
CStreamEngineWakeEvent m_awakeEvent;
CryEvent m_resetDoneEvent;
string m_name;
uint32 m_nReadCounter;
};
//////////////////////////////////////////////////////////////////////////
// Thread that performs IO operations.
//////////////////////////////////////////////////////////////////////////
class CStreamingWorkerThread
: public CrySimpleThread<CStreamingIOThread>
, public CMultiThreadRefCount
{
public:
enum EWorkerType
{
eWorkerAsyncCallback,
};
CStreamingWorkerThread(CStreamEngine* pStreamEngine, const char* name, EWorkerType type, SStreamRequestQueue* pQueue);
~CStreamingWorkerThread();
void BeginReset();
void EndReset();
void CancelAll();
//////////////////////////////////////////////////////////////////////////
// CrySimpleThread
//////////////////////////////////////////////////////////////////////////
virtual void Run();
virtual void Cancel();
//////////////////////////////////////////////////////////////////////////
private:
EWorkerType m_type;
CStreamEngine* m_pStreamEngine;
SStreamRequestQueue* m_pQueue;
volatile bool m_bCancelThreadRequest;
volatile bool m_bNeedsReset;
CryEvent m_resetDoneEvent;
string m_name;
};
#endif // CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMIOTHREAD_H
Code/CryEngine/CrySystem/StreamEngine/StreamReadStream.cpp
-516
View File
@@ -1,516 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Engine
#include "CrySystem_precompiled.h"
#include <ISystem.h>
#include <ILog.h>
#include "StreamReadStream.h"
#include "StreamEngine.h"
#include "MTSafeAllocator.h"
extern CMTSafeHeap* g_pPakHeap;
SLockFreeSingleLinkedListHeader CReadStream::s_freeRequests;
CReadStream* CReadStream::Allocate(CStreamEngine* pEngine, const EStreamTaskType tSource, const char* szFilename, IStreamCallback* pCallback, const StreamReadParams* pParams)
{
char* pFree = reinterpret_cast<char*>(CryInterlockedPopEntrySList(s_freeRequests));
CReadStream* pReq;
IF_LIKELY (pFree)
{
AZ_PUSH_DISABLE_WARNING(,"-Winvalid-offsetof")
ptrdiff_t offs = offsetof(CReadStream, m_nextFree);
AZ_POP_DISABLE_WARNING
pReq = reinterpret_cast<CReadStream*>(pFree - offs);
}
else
{
pReq = new CReadStream;
}
pReq->m_pEngine = pEngine;
pReq->m_Type = tSource;
pReq->m_strFileName = szFilename;
pReq->m_pCallback = pCallback;
if (pParams)
{
pReq->m_Params = *pParams;
}
pReq->m_pBuffer = pReq->m_Params.pBuffer;
#ifdef STREAMENGINE_ENABLE_STATS
pReq->m_requestTime = gEnv->pTimer->GetAsyncTime();
#endif
return pReq;
}
void CReadStream::Flush()
{
AZ_PUSH_DISABLE_WARNING(, "-Winvalid-offsetof")
ptrdiff_t offs = offsetof(CReadStream, m_nextFree);
AZ_POP_DISABLE_WARNING
for (char* pFree = reinterpret_cast<char*>(CryInterlockedPopEntrySList(s_freeRequests));
pFree;
pFree = reinterpret_cast<char*>(CryInterlockedPopEntrySList(s_freeRequests)))
{
CReadStream* pReq = reinterpret_cast<CReadStream*>(pFree - offs);
delete pReq;
}
}
//////////////////////////////////////////////////////////////////////////
CReadStream::CReadStream()
{
Reset();
}
//////////////////////////////////////////////////////////////////////////
CReadStream::~CReadStream()
{
}
// returns true if the file read was completed (successfully or unsuccessfully)
// check IsError to check if the whole requested file (piece) was read
bool CReadStream::IsFinished()
{
return m_bFinished;
}
// returns the number of bytes read so far (the whole buffer size if IsFinished())
unsigned int CReadStream::GetBytesRead ([[maybe_unused]] bool bWait)
{
if (!m_bError)
{
return m_Params.nSize;
}
return 0;
}
// returns the buffer into which the data has been or will be read
// at least GetBytesRead() bytes in this buffer are guaranteed to be already read
const void* CReadStream::GetBuffer ()
{
return m_pBuffer;
}
void CReadStream::AbortShutdown()
{
{
CryAutoCriticalSection lock(m_callbackLock);
m_bError = true;
m_nIOError = ERROR_ABORTED_ON_SHUTDOWN;
m_bFileRequestComplete = true;
if (m_pFileRequest)
{
__debugbreak();
}
}
// lock this object to avoid preliminary destruction
CReadStream_AutoPtr pLock(this);
{
CryAutoCriticalSection lock(m_callbackLock);
// all the callbacks have to handle error cases and needs to be called anyway, even if the stream I/O is aborted
ExecuteAsyncCallback_CBLocked();
ExecuteSyncCallback_CBLocked();
m_pCallback = NULL;
}
}
// tries to stop reading the stream; this is advisory and may have no effect
// all the callbacks will be called after this. If you just destructing object,
// dereference this object and it will automatically abort and release all associated resources.
void CReadStream::Abort()
{
{
CryAutoCriticalSection lock(m_callbackLock);
m_bError = true;
m_nIOError = ERROR_USER_ABORT;
m_bFileRequestComplete = true;
if (m_pFileRequest)
{
m_pFileRequest->Cancel();
m_pFileRequest = 0;
}
}
// lock this object to avoid preliminary destruction
CReadStream_AutoPtr pLock(this);
{
CryAutoCriticalSection lock(m_callbackLock);
// all the callbacks have to handle error cases and needs to be called anyway, even if the stream I/O is aborted
ExecuteAsyncCallback_CBLocked();
ExecuteSyncCallback_CBLocked();
m_pCallback = NULL;
}
m_pEngine->AbortJob(this);
}
bool CReadStream::TryAbort()
{
if (!m_callbackLock.TryLock())
{
return false;
}
if (m_pFileRequest && !m_pFileRequest->TryCancel())
{
m_callbackLock.Unlock();
return false;
}
m_bError = true;
m_nIOError = ERROR_USER_ABORT;
m_bFileRequestComplete = true;
m_pFileRequest = 0;
// lock this object to avoid preliminary destruction
CReadStream_AutoPtr pLock(this);
// all the callbacks have to handle error cases and needs to be called anyway, even if the stream I/O is aborted
ExecuteAsyncCallback_CBLocked();
ExecuteSyncCallback_CBLocked();
m_pCallback = NULL;
m_callbackLock.Unlock();
m_pEngine->AbortJob(this);
return true;
}
// tries to raise the priority of the read; this is advisory and may have no effect
void CReadStream::SetPriority (EStreamTaskPriority ePriority)
{
if (m_Params.ePriority != ePriority)
{
m_Params.ePriority = ePriority;
if (m_pFileRequest && m_pFileRequest->m_status == CAsyncIOFileRequest::eStatusInFileQueue)
{
m_pEngine->UpdateJobPriority(this);
}
}
}
// unconditionally waits until the callback is called
// i.e. if the stream hasn't yet finish, it's guaranteed that the user-supplied callback
// is called before return from this function (unless no callback was specified)
void CReadStream::Wait(int nMaxWaitMillis)
{
// lock this object to avoid preliminary destruction
CReadStream_AutoPtr pLock(this);
bool bNeedFinalize = (m_Params.nFlags & IStreamEngine::FLAGS_NO_SYNC_CALLBACK) == 0;
if (!m_bFinished && !m_bError && !m_pFileRequest)
{
assert(m_pFileRequest != NULL); // If we want to Wait for stream its file request must not be NULL.
// This will almost certainly cause Dead-Lock
CryFatalError("Waiting for stream when StreamingEngine is paused");
}
CTimeValue t0;
if (nMaxWaitMillis > 0)
{
t0 = gEnv->pTimer->GetAsyncTime();
}
while (!m_bFinished && !m_bError)
{
if (bNeedFinalize)
{
m_pEngine->MainThread_FinalizeIOJobs();
}
if (!m_bFileRequestComplete)
{
CrySleep(5);
}
if (nMaxWaitMillis > 0)
{
CTimeValue t1 = gEnv->pTimer->GetAsyncTime();
if (CTimeValue(t1 - t0).GetMilliSeconds() > nMaxWaitMillis)
{
// Break if we are waiting for too long.
break;
}
}
}
}
//////////////////////////////////////////////////////////////////////////
uint64 CReadStream::GetPriority() const
{
return 0;
}
// this gets called upon the IO has been executed to call the callbacks
void CReadStream::MainThread_Finalize()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
// call asynchronous callback function if needed synchronously
{
CryAutoCriticalSection lock(m_callbackLock);
ExecuteSyncCallback_CBLocked();
}
m_pFileRequest = 0;
}
IStreamCallback* CReadStream::GetCallback() const
{
return m_pCallback;
}
unsigned CReadStream::GetError() const
{
return m_nIOError;
}
const char* CReadStream::GetErrorName() const
{
switch (m_nIOError)
{
case ERROR_UNKNOWN_ERROR:
return "Unknown error";
case ERROR_UNEXPECTED_DESTRUCTION:
return "Unexpected destruction";
case ERROR_INVALID_CALL:
return "Invalid call";
case ERROR_CANT_OPEN_FILE:
return "Cannot open the file";
case ERROR_REFSTREAM_ERROR:
return "Refstream error";
case ERROR_OFFSET_OUT_OF_RANGE:
return "Offset out of range";
case ERROR_REGION_OUT_OF_RANGE:
return "Region out of range";
case ERROR_SIZE_OUT_OF_RANGE:
return "Size out of range";
case ERROR_CANT_START_READING:
return "Cannot start reading";
case ERROR_OUT_OF_MEMORY:
return "Out of memory";
case ERROR_ABORTED_ON_SHUTDOWN:
return "Aborted on shutdown";
case ERROR_OUT_OF_MEMORY_QUOTA:
return "Out of memory quota";
case ERROR_ZIP_CACHE_FAILURE:
return "ZIP cache failure";
case ERROR_USER_ABORT:
return "User aborted";
}
return "Unrecognized error";
}
int CReadStream::AddRef()
{
return CryInterlockedIncrement(&m_nRefCount);
}
int CReadStream::Release()
{
int nRef = CryInterlockedDecrement(&m_nRefCount);
#ifndef _RELEASE
if (nRef < 0)
{
__debugbreak();
}
#endif
if (nRef == 0)
{
Reset();
CryInterlockedPushEntrySList(s_freeRequests, m_nextFree);
}
return nRef;
}
void CReadStream::Reset()
{
m_strFileName.clear();
m_pFileRequest = NULL;
m_Params = StreamReadParams();
memset((void*)&m_nRefCount, 0, (char*)(this + 1) - (char*)(&m_nRefCount));
}
void CReadStream::SetUserData(DWORD_PTR dwUserData)
{
m_Params.dwUserData = dwUserData;
}
//////////////////////////////////////////////////////////////////////////
void CReadStream::ExecuteAsyncCallback_CBLocked()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
if (!m_bIsAsyncCallbackExecuted && m_pCallback)
{
m_bIsAsyncCallbackExecuted = true;
m_pCallback->StreamAsyncOnComplete(this, m_nIOError);
}
}
void CReadStream::ExecuteSyncCallback_CBLocked()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
if (!m_bIsSyncCallbackExecuted && m_pCallback && (0 == (m_Params.nFlags & IStreamEngine::FLAGS_NO_SYNC_CALLBACK)))
{
m_bIsSyncCallbackExecuted = true;
CReadStream_AutoPtr protectMe(this); // Stream can be freed inside the callback!
m_pCallback->StreamOnComplete(this, m_nIOError);
// We do not need FileRequest here anymore, and not its temporary memory.
m_pFileRequest = 0;
m_pBuffer = NULL;
m_bFinished = true;
}
else
{
m_pFileRequest = 0;
m_pBuffer = NULL;
m_bFinished = true;
}
#ifdef STREAMENGINE_ENABLE_LISTENER
IStreamEngineListener* pListener = m_pEngine->GetListener();
if (pListener)
{
pListener->OnStreamDone(this);
}
#endif
}
void* CReadStream::operator new (size_t sz)
{
return CryModuleMemalign(sz, alignof(CReadStream));
}
void CReadStream::operator delete(void* p)
{
CryModuleMemalignFree(p);
}
//////////////////////////////////////////////////////////////////////////
void CReadStream::FreeTemporaryMemory()
{
// Free temporary block.
if (m_pFileRequest)
{
m_pFileRequest->SyncWithDecompress();
m_pFileRequest->FreeBuffer();
}
m_pBuffer = 0;
}
//////////////////////////////////////////////////////////////////////////
bool CReadStream::IsReqReading()
{
if (m_strFileName.empty())
{
return false;
}
return true;
}
//////////////////////////////////////////////////////////////////////////
CAsyncIOFileRequest* CReadStream::CreateFileRequest()
{
m_pFileRequest = CAsyncIOFileRequest::Allocate(m_Type);
m_pFileRequest->m_nRequestedSize = m_Params.nSize;
m_pFileRequest->m_nRequestedOffset = m_Params.nOffset;
m_pFileRequest->m_pExternalMemoryBuffer = m_pBuffer;
m_pFileRequest->m_bWriteOnlyExternal = (m_Params.nFlags & IStreamEngine::FLAGS_WRITE_ONLY_EXTERNAL_BUFFER) != 0;
m_pFileRequest->m_pReadStream = this;
m_pFileRequest->m_strFileName = m_strFileName;
m_pFileRequest->m_ePriority = m_Params.ePriority;
m_pFileRequest->m_eMediaType = m_Params.eMediaType;
m_bFileRequestComplete = false;
return m_pFileRequest;
}
void* CReadStream::OnNeedStorage(size_t size, bool& bAbortOnFailToAlloc)
{
CryAutoCriticalSection lock(m_callbackLock);
if (m_pCallback)
{
return m_pCallback->StreamOnNeedStorage(this, size, bAbortOnFailToAlloc);
}
return NULL;
}
//////////////////////////////////////////////////////////////////////////
void CReadStream::OnAsyncFileRequestComplete()
{
CryAutoCriticalSection lock(m_callbackLock);
if (!m_bFileRequestComplete)
{
if (m_pFileRequest)
{
m_Params.nSize = m_pFileRequest->m_nRequestedSize;
m_pBuffer = m_pFileRequest->m_pOutputMemoryBuffer;
m_nBytesRead = m_pFileRequest->m_nSizeOnMedia;
m_nIOError = m_pFileRequest->m_nError;
m_bError = m_nIOError != 0;
if (m_bError)
{
m_nBytesRead = 0;
}
#ifdef STREAMENGINE_ENABLE_STATS
m_ReadTime = m_pFileRequest->m_readTime;
#endif
}
ExecuteAsyncCallback_CBLocked();
if (m_Params.nFlags & IStreamEngine::FLAGS_NO_SYNC_CALLBACK)
{
// We do not need FileRequest here anymore, and not its temporary memory.
m_pFileRequest = 0;
m_bFinished = true;
}
m_bFileRequestComplete = true;
}
}
Code/CryEngine/CrySystem/StreamEngine/StreamReadStream.h
-178
View File
@@ -1,178 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Engine
#ifndef CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMREADSTREAM_H
#define CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMREADSTREAM_H
#pragma once
#include "IStreamEngine.h"
#include "StreamAsyncFileRequest.h"
class CStreamEngine;
class CReadStream
: public IReadStream
{
friend class CStreamEngine;
public:
static CReadStream* Allocate(CStreamEngine* pEngine, const EStreamTaskType tSource, const char* szFilename, IStreamCallback* pCallback, const StreamReadParams* pParams);
static void Flush();
public:
CReadStream();
virtual ~CReadStream ();
virtual int AddRef();
virtual int Release();
virtual DWORD_PTR GetUserData() {return m_Params.dwUserData; }
// set user defined data into stream's params
virtual void SetUserData(DWORD_PTR dwUserData);
// returns true if the file read was not successful.
virtual bool IsError() { return m_bError; };
// returns true if the file read was completed (successfully or unsuccessfully)
// check IsError to check if the whole requested file (piece) was read
virtual bool IsFinished();
// returns the number of bytes read so far (the whole buffer size if IsFinished())
virtual unsigned int GetBytesRead (bool bWait);
// returns the buffer into which the data has been or will be read
// at least GetBytesRead() bytes in this buffer are guaranteed to be already read
virtual const void* GetBuffer ();
void AbortShutdown();
// tries to stop reading the stream; this is advisory and may have no effect
// but the callback will not be called after this. If you just destructing object,
// dereference this object and it will automatically abort and release all associated resources.
virtual void Abort();
virtual bool TryAbort();
// tries to raise the priority of the read; this is advisory and may have no effect
virtual void SetPriority (EStreamTaskPriority EPriority);
// unconditionally waits until the callback is called
// i.e. if the stream hasn't yet finish, it's guaranteed that the user-supplied callback
// is called before return from this function (unless no callback was specified)
virtual void Wait(int nMaxWaitMillis = -1);
virtual uint64 GetPriority() const;
virtual const StreamReadParams& GetParams() const {return m_Params; }
virtual const EStreamTaskType GetCallerType() const { return m_Type; }
virtual EStreamSourceMediaType GetMediaType() const { return m_MediaType; }
// return pointer to callback routine(can be NULL)
virtual IStreamCallback* GetCallback() const;
// return IO error #
virtual unsigned GetError() const;
// Returns IO error name
virtual const char* GetErrorName() const;
// return stream name
virtual const char* GetName() const { return m_strFileName.c_str(); };
virtual void FreeTemporaryMemory();
// this gets called upon the IO has been executed to call the callbacks
void MainThread_Finalize();
bool IsReqReading();
#ifdef STREAMENGINE_ENABLE_STATS
void SetRequestTime(CTimeValue& time) { m_requestTime = time; }
const CTimeValue& GetRequestTime() { return m_requestTime; }
#endif
// decompression of zip-compressed files with default behavior
CAsyncIOFileRequest* CreateFileRequest();
void ComputedMediaType(EStreamSourceMediaType eMT) { m_MediaType = eMT; }
void* OnNeedStorage(size_t size, bool& bAbortOnFailToAlloc);
void OnAsyncFileRequestComplete();
CAsyncIOFileRequest* GetFileRequest() { return m_pFileRequest; }
private:
void Reset();
// call the async callback
void ExecuteAsyncCallback_CBLocked();
// call the sync callback
void ExecuteSyncCallback_CBLocked();
private:
void* operator new (size_t sz);
void operator delete(void* p);
private:
static SLockFreeSingleLinkedListHeader s_freeRequests;
private:
STREAMENGINE_LL_ALIGN SLockFreeSingleLinkedListEntry m_nextFree;
CryStringLocal m_strFileName;
CryCriticalSection m_callbackLock;
CAsyncIOFileRequest_AutoPtr m_pFileRequest;
StreamReadParams m_Params;
// Only POD types must exist below here. They will be memset!
volatile int m_nRefCount;
CStreamEngine* m_pEngine;
// the type of the task
EStreamTaskType m_Type;
EStreamSourceMediaType m_MediaType;
// the initial data from the user
// the callback; may be NULL
IStreamCallback* m_pCallback;
// Bytes actually read from media.
uint32 m_nBytesRead;
volatile bool m_bIsAsyncCallbackExecuted;
volatile bool m_bIsSyncCallbackExecuted;
volatile bool m_bFileRequestComplete;
// the actual buffer to read to
void* m_pBuffer;
volatile bool m_bError;
volatile bool m_bFinished;
unsigned int m_nIOError;
#ifdef STREAMENGINE_ENABLE_STATS
// time when request was made
CTimeValue m_requestTime;
// Time for actual reading
CTimeValue m_ReadTime;
#endif
};
TYPEDEF_AUTOPTR(CReadStream);
#endif // CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMREADSTREAM_H
Code/CryEngine/CrySystem/System.cpp
+1 -65
View File
@@ -135,14 +135,12 @@ LRESULT WINAPI WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
#include "XML/xml.h"
#include "XML/ReadWriteXMLSink.h"
#include "StreamEngine/StreamEngine.h"
#include "PhysRenderer.h"
#include "LocalizedStringManager.h"
#include "XML/XmlUtils.h"
#include "SystemEventDispatcher.h"
#include "ServerThrottle.h"
#include "ResourceManager.h"
#include "HMDBus.h"
#include "IZLibCompressor.h"
@@ -157,7 +155,6 @@ LRESULT WINAPI WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
#include "CryWaterMark.h"
WATERMARKDATA(_m);
#include "ImageHandler.h"
#include <LyShine/Bus/UiCursorBus.h>
#include <AzFramework/Asset/AssetSystemBus.h>
#include <AzFramework/Input/Buses/Requests/InputSystemRequestBus.h>
@@ -179,7 +176,6 @@ WATERMARKDATA(_m);
#include <ILevelSystem.h>
#include <CrtDebugStats.h>
#include <AzFramework/IO/LocalFileIO.h>
// profilers api.
@@ -191,14 +187,6 @@ SSystemCVars g_cvars;
#include "ITextModeConsole.h"
extern int CryMemoryGetAllocatedSize();
// these heaps are used by underlying System structures
// to allocate, accordingly, small (like elements of std::set<..*>) and big (like memory for reading files) objects
// hopefully someday we'll have standard MT-safe heap
//CMTSafeHeap g_pakHeap;
CMTSafeHeap* g_pPakHeap = 0;// = &g_pakHeap;
//////////////////////////////////////////////////////////////////////////
#include "Validator.h"
@@ -266,7 +254,6 @@ namespace
// System Implementation.
//////////////////////////////////////////////////////////////////////////
CSystem::CSystem(SharedEnvironmentInstance* pSharedEnvironment)
: m_imageHandler(std::make_unique<ImageHandler>())
{
CrySystemRequestBus::Handler::BusConnect();
@@ -307,8 +294,6 @@ CSystem::CSystem(SharedEnvironmentInstance* pSharedEnvironment)
m_env.pSharedEnvironment = pSharedEnvironment;
//////////////////////////////////////////////////////////////////////////
m_pStreamEngine = NULL;
m_pIFont = NULL;
m_pIFontUi = NULL;
m_rWidth = NULL;
@@ -322,7 +307,6 @@ CSystem::CSystem(SharedEnvironmentInstance* pSharedEnvironment)
m_rStencilBits = NULL;
m_rFullscreen = NULL;
m_sysNoUpdate = NULL;
m_pMemoryManager = NULL;
m_pProcess = NULL;
m_pValidator = NULL;
@@ -387,12 +371,8 @@ CSystem::CSystem(SharedEnvironmentInstance* pSharedEnvironment)
m_pXMLUtils = new CXmlUtils(this);
m_pMemoryManager = CryGetIMemoryManager();
m_pResourceManager = new CResourceManager;
m_pTextModeConsole = NULL;
g_pPakHeap = new CMTSafeHeap;
if (!AZ::AllocatorInstance<AZ::OSAllocator>::IsReady())
{
m_initedOSAllocator = true;
@@ -433,11 +413,8 @@ CSystem::~CSystem()
CRY_ASSERT(m_windowMessageHandlers.empty() && "There exists a dangling window message handler somewhere");
SAFE_DELETE(m_pXMLUtils);
SAFE_DELETE(m_pResourceManager);
SAFE_DELETE(m_pSystemEventDispatcher);
SAFE_DELETE(g_pPakHeap);
AZCoreLogSink::Disconnect();
if (m_initedSysAllocator)
{
@@ -477,12 +454,6 @@ void CSystem::FreeLib(AZStd::unique_ptr<AZ::DynamicModuleHandle>& hLibModule)
}
}
//////////////////////////////////////////////////////////////////////////
IStreamEngine* CSystem::GetStreamEngine()
{
return m_pStreamEngine;
}
//////////////////////////////////////////////////////////////////////////
IRemoteConsole* CSystem::GetIRemoteConsole()
{
@@ -582,9 +553,6 @@ void CSystem::ShutDown()
// Shutdown any running VR devices.
EBUS_EVENT(AZ::VR::HMDInitRequestBus, Shutdown);
// Shutdown resource manager.
m_pResourceManager->Shutdown();
if (gEnv && gEnv->pLyShine)
{
gEnv->pLyShine->Release();
@@ -645,11 +613,6 @@ void CSystem::ShutDown()
SAFE_DELETE(m_pLocalizationManager);
//DebugStats(false, false);//true);
//CryLogAlways("");
//CryLogAlways("release mode memory manager stats:");
//DumpMMStats(true);
SAFE_DELETE(m_pCpu);
delete m_pCmdLine;
@@ -659,8 +622,7 @@ void CSystem::ShutDown()
// Shut down audio as late as possible but before the streaming system and console get released!
Audio::Gem::AudioSystemGemRequestBus::Broadcast(&Audio::Gem::AudioSystemGemRequestBus::Events::Release);
// Shut down the streaming system and console as late as possible and after audio!
SAFE_DELETE(m_pStreamEngine);
// Shut down console as late as possible and after audio!
SAFE_RELEASE(m_env.pConsole);
// Log must be last thing released.
@@ -904,12 +866,6 @@ bool CSystem::UpdatePreTickBus(int updateFlags, int nPauseMode)
}
#endif //PROFILE_WITH_VTUNE
if (m_pStreamEngine)
{
FRAME_PROFILER("StreamEngine::Update()", this, PROFILE_SYSTEM);
m_pStreamEngine->Update();
}
#ifndef EXCLUDE_UPDATE_ON_CONSOLE
if (m_bIgnoreUpdates)
{
@@ -1024,14 +980,6 @@ bool CSystem::UpdatePreTickBus(int updateFlags, int nPauseMode)
}
}
//////////////////////////////////////////////////////////////////////////
// Update Resource Manager.
//////////////////////////////////////////////////////////////////////////
{
FRAME_PROFILER("SysUpdate:ResourceManager", this, PROFILE_SYSTEM);
m_pResourceManager->Update();
}
// Use UI timer for CryMovie, because it should not be affected by pausing game time
const float fMovieFrameTime = m_Time.GetFrameTime(ITimer::ETIMER_UI);
@@ -1419,24 +1367,12 @@ void CSystem::Relaunch(bool bRelaunch)
SaveConfiguration();
}
//////////////////////////////////////////////////////////////////////////
uint32 CSystem::GetUsedMemory()
{
return CryMemoryGetAllocatedSize();
}
//////////////////////////////////////////////////////////////////////////
ILocalizationManager* CSystem::GetLocalizationManager()
{
return m_pLocalizationManager;
}
//////////////////////////////////////////////////////////////////////////
IResourceManager* CSystem::GetIResourceManager()
{
return m_pResourceManager;
}
//////////////////////////////////////////////////////////////////////////
void CSystem::debug_GetCallStackRaw(void** callstack, uint32& callstackLength)
{
Code/CryEngine/CrySystem/System.h
-60
View File
@@ -23,7 +23,6 @@
#include "CmdLine.h"
#include "CryName.h"
#include "MTSafeAllocator.h"
#include "CPUDetect.h"
#include <AzFramework/Archive/ArchiveVars.h>
#include "RenderBus.h"
@@ -276,33 +275,6 @@ extern SSystemCVars g_cvars;
class CSystem;
struct SmallModuleInfo
{
string name;
CryModuleMemoryInfo memInfo;
};
struct SCryEngineStatsModuleInfo
{
string name;
CryModuleMemoryInfo memInfo;
uint32 moduleStaticSize;
uint32 usedInModule;
uint32 SizeOfCode;
uint32 SizeOfInitializedData;
uint32 SizeOfUninitializedData;
};
struct SCryEngineStatsGlobalMemInfo
{
int totalUsedInModules;
int totalCodeAndStatic;
int countedMemoryModules;
uint64 totalAllocatedInModules;
int totalNumAllocsInModules;
std::vector<SCryEngineStatsModuleInfo> modules;
};
struct CProfilingSystem
: public IProfilingSystem
{
@@ -338,17 +310,6 @@ class CSystem
, public CrySystemRequestBus::Handler
{
public:
inline void* operator new(std::size_t)
{
size_t allocated = 0;
return CryMalloc(sizeof(CSystem), allocated, 64);
}
inline void operator delete(void* p)
{
CryFree(p, 64);
}
CSystem(SharedEnvironmentInstance* pSharedEnvironment);
~CSystem();
@@ -390,8 +351,6 @@ public:
ISystem* GetCrySystem() override;
////////////////////////////////////////////////////////////////////////
uint32 GetUsedMemory();
virtual bool SteamInit();
void Relaunch(bool bRelaunch);
@@ -412,17 +371,14 @@ public:
IConsole* GetIConsole() { return m_env.pConsole; };
IRemoteConsole* GetIRemoteConsole();
IMovieSystem* GetIMovieSystem() { return m_env.pMovieSystem; };
IMemoryManager* GetIMemoryManager(){ return m_pMemoryManager; }
ICryFont* GetICryFont(){ return m_env.pCryFont; }
ILog* GetILog(){ return m_env.pLog; }
ICmdLine* GetICmdLine(){ return m_pCmdLine; }
IStreamEngine* GetStreamEngine();
IValidator* GetIValidator() { return m_pValidator; };
INameTable* GetINameTable() { return m_env.pNameTable; };
IViewSystem* GetIViewSystem();
ILevelSystem* GetILevelSystem();
ISystemEventDispatcher* GetISystemEventDispatcher() { return m_pSystemEventDispatcher; }
IResourceManager* GetIResourceManager();
ITextModeConsole* GetITextModeConsole();
IProfilingSystem* GetIProfilingSystem() { return &m_ProfilingSystem; }
IZLibCompressor* GetIZLibCompressor() { return m_pIZLibCompressor; }
@@ -509,11 +465,6 @@ public:
virtual int ShowMessage(const char* text, const char* caption, unsigned int uType);
bool CheckLogVerbosity(int verbosity);
virtual void DebugStats(bool checkpoint, bool leaks);
void DumpWinHeaps();
virtual int DumpMMStats(bool log);
//! Return pointer to user defined callback.
ISystemUserCallback* GetUserCallback() const { return m_pUserCallback; };
@@ -541,8 +492,6 @@ public:
void SetVersionInfo(const char* const szVersion);
#endif
virtual const IImageHandler* GetImageHandler() const override { return m_imageHandler.get(); }
void ShutdownModuleLibraries();
#if defined(WIN32)
@@ -570,7 +519,6 @@ private:
bool InitConsole();
bool InitFileSystem();
bool InitFileSystem_LoadEngineFolders(const SSystemInitParams& initParams);
bool InitStreamEngine();
bool InitAudioSystem(const SSystemInitParams& initParams);
bool InitShine(const SSystemInitParams& initParams);
@@ -597,7 +545,6 @@ private:
#endif // #ifndef _RELEASE
bool ReLaunchMediaCenter();
void LogSystemInfo();
void UpdateAudioSystems();
void AddCVarGroupDirectory(const string& sPath);
@@ -690,14 +637,9 @@ private: // ------------------------------------------------------
std::map<CCryNameCRC, AZStd::unique_ptr<AZ::DynamicModuleHandle> > m_moduleDLLHandles;
//! THe streaming engine
class CStreamEngine* m_pStreamEngine;
//! current active process
IProcess* m_pProcess;
IMemoryManager* m_pMemoryManager;
CCamera m_PhysRendererCamera;
ICVar* m_p_draw_helpers_str;
int m_iJumpToPhysProfileEnt;
@@ -914,7 +856,6 @@ public:
protected: // -------------------------------------------------------------
CCmdLine* m_pCmdLine;
class CResourceManager* m_pResourceManager;
ITextModeConsole* m_pTextModeConsole;
string m_currentLanguageAudio;
@@ -943,7 +884,6 @@ protected: // -------------------------------------------------------------
bool m_bIsSteamInitialized;
std::unique_ptr<IImageHandler> m_imageHandler;
std::vector<IWindowMessageHandler*> m_windowMessageHandlers;
bool m_initedOSAllocator = false;
bool m_initedSysAllocator = false;
Code/CryEngine/CrySystem/SystemInit.cpp
+1 -82
View File
@@ -95,7 +95,6 @@
#include "XConsole.h"
#include "Log.h"
#include "XML/xml.h"
#include "StreamEngine/StreamEngine.h"
#include "PhysRenderer.h"
#include "LocalizedStringManager.h"
#include "SystemEventDispatcher.h"
@@ -103,8 +102,6 @@
#include "ServerThrottle.h"
#include "SystemCFG.h"
#include "AutoDetectSpec.h"
#include "ResourceManager.h"
#include "MTSafeAllocator.h"
#include "ZLibCompressor.h"
#include "ZLibDecompressor.h"
#include "ZStdDecompressor.h"
@@ -245,8 +242,6 @@ CUNIXConsole* pUnixConsole;
#define AZ_TRACE_SYSTEM_WINDOW AZ::Debug::Trace::GetDefaultSystemWindow()
extern CMTSafeHeap* g_pPakHeap;
#ifdef WIN32
extern HMODULE gDLLHandle;
#endif
@@ -274,7 +269,6 @@ struct SCVarsClientConfigSink
//////////////////////////////////////////////////////////////////////////
static inline void InlineInitializationProcessing([[maybe_unused]] const char* sDescription)
{
assert(CryMemory::IsHeapValid());
if (gEnv->pLog)
{
gEnv->pLog->UpdateLoadingScreen(0);
@@ -1101,12 +1095,7 @@ bool CSystem::InitFileSystem_LoadEngineFolders(const SSystemInitParams&)
auto projectName = AZ::Utils::GetProjectName();
AZ_Printf(AZ_TRACE_SYSTEM_WINDOW, "Project Name: %s\n", projectName.empty() ? "None specified" : projectName.c_str());
// simply open all paks if fast load pak can't be found
if (!m_pResourceManager->LoadFastLoadPaks(true))
{
OpenBasicPaks();
}
OpenBasicPaks();
// Load game-specific folder.
LoadConfiguration("game.cfg");
@@ -1120,21 +1109,6 @@ bool CSystem::InitFileSystem_LoadEngineFolders(const SSystemInitParams&)
return (true);
}
//////////////////////////////////////////////////////////////////////////
bool CSystem::InitStreamEngine()
{
LOADING_TIME_PROFILE_SECTION(GetISystem());
if (m_pUserCallback)
{
m_pUserCallback->OnInitProgress("Initializing Stream Engine...");
}
m_pStreamEngine = new CStreamEngine();
return true;
}
//////////////////////////////////////////////////////////////////////////
bool CSystem::InitAudioSystem(const SSystemInitParams& initParams)
{
@@ -1299,8 +1273,6 @@ void CSystem::OpenBasicPaks()
//////////////////////////////////////////////////////////////////////////
const char* const assetsDir = "@assets@";
const char* shaderCachePakDir = "@assets@/shadercache.pak";
const char* shaderCacheStartupPakDir = "@assets@/shadercachestartup.pak";
// After game paks to have same search order as with files on disk
m_env.pCryPak->OpenPack(assetsDir, "Engine.pak");
@@ -1310,11 +1282,6 @@ void CSystem::OpenBasicPaks()
#include AZ_RESTRICTED_FILE(SystemInit_cpp)
#endif
m_env.pCryPak->OpenPack(assetsDir, shaderCachePakDir);
m_env.pCryPak->OpenPack(assetsDir, shaderCacheStartupPakDir);
m_env.pCryPak->OpenPack(assetsDir, "Shaders.pak");
m_env.pCryPak->OpenPack(assetsDir, "ShadersBin.pak");
#ifdef AZ_PLATFORM_ANDROID
// Load Android Obb files if available
const char* obbStorage = AZ::Android::Utils::GetObbStoragePath();
@@ -1326,22 +1293,6 @@ void CSystem::OpenBasicPaks()
InlineInitializationProcessing("CSystem::OpenBasicPaks OpenPacks( Engine... )");
//////////////////////////////////////////////////////////////////////////
// Open paks in MOD subfolders.
//////////////////////////////////////////////////////////////////////////
#if !defined(_RELEASE)
if (const ICmdLineArg* pModArg = GetICmdLine()->FindArg(eCLAT_Pre, "MOD"))
{
if (IsMODValid(pModArg->GetValue()))
{
AZStd::string modFolder = "Mods\\";
modFolder += pModArg->GetValue();
modFolder += "\\*.pak";
GetIPak()->OpenPacks(assetsDir, modFolder, AZ::IO::IArchive::FLAGS_PATH_REAL | AZ::IO::INestedArchive::FLAGS_OVERRIDE_PAK);
}
}
#endif // !defined(_RELEASE)
// Load paks required for game init to mem
gEnv->pCryPak->LoadPakToMemory("Engine.pak", AZ::IO::IArchive::eInMemoryPakLocale_GPU);
}
@@ -1687,8 +1638,6 @@ bool CSystem::Init(const SSystemInitParams& startupParams)
m_systemConfigName += ".cfg";
}
AZ_Assert(CryMemory::IsHeapValid(), "Memory heap must be valid before continuing SystemInit.");
#if defined(WIN32) || defined(WIN64)
// check OS version - we only want to run on XP or higher - talk to Martin Mittring if you want to change this
{
@@ -1709,8 +1658,6 @@ AZ_POP_DISABLE_WARNING
}
#endif
m_pResourceManager->Init();
// Get file version information.
QueryVersionInfo();
DetectGameFolderAccessRights();
@@ -1983,11 +1930,6 @@ AZ_POP_DISABLE_WARNING
return false;
}
if (!startupParams.bSkipConsole)
{
LogSystemInfo();
}
InlineInitializationProcessing("CSystem::Init Load Engine Folders");
//////////////////////////////////////////////////////////////////////////
@@ -2052,29 +1994,6 @@ AZ_POP_DISABLE_WARNING
gEnv->bNoAssertDialog = true;
}
//////////////////////////////////////////////////////////////////////////
// Stream Engine
//////////////////////////////////////////////////////////////////////////
AZ_Printf(AZ_TRACE_SYSTEM_WINDOW, "Stream Engine Initialization");
InitStreamEngine();
InlineInitializationProcessing("CSystem::Init StreamEngine");
{
if (m_pCmdLine->FindArg(eCLAT_Pre, "NullRenderer"))
{
m_env.pConsole->LoadConfigVar("r_Driver", "NULL");
}
else if (m_pCmdLine->FindArg(eCLAT_Pre, "DX11"))
{
m_env.pConsole->LoadConfigVar("r_Driver", "DX11");
}
else if (m_pCmdLine->FindArg(eCLAT_Pre, "GL"))
{
m_env.pConsole->LoadConfigVar("r_Driver", "GL");
}
}
LogBuildInfo();
InlineInitializationProcessing("CSystem::Init LoadConfigurations");
Code/CryEngine/CrySystem/SystemRender.cpp
-3
View File
@@ -47,13 +47,10 @@
#define SYSTEMRENDERER_CPP_SECTION_2 2
#endif
extern CMTSafeHeap* g_pPakHeap;
#if defined(AZ_PLATFORM_ANDROID)
#include <AzCore/Android/Utils.h>
#endif
extern int CryMemoryGetAllocatedSize();
/////////////////////////////////////////////////////////////////////////////////
bool CSystem::GetPrimaryPhysicalDisplayDimensions([[maybe_unused]] int& o_widthPixels, [[maybe_unused]] int& o_heightPixels)
{
Code/CryEngine/CrySystem/SystemWin32.cpp
-534
View File
@@ -51,7 +51,6 @@
#endif
#include "XConsole.h"
#include "StreamEngine/StreamEngine.h"
#include "LocalizedStringManager.h"
#include "XML/XmlUtils.h"
#include "AutoDetectSpec.h"
@@ -121,19 +120,6 @@ struct PEHeader_DLL
#pragma pack(pop)
#endif
const SmallModuleInfo* FindModuleInfo(std::vector<SmallModuleInfo>& vec, const char* name)
{
for (size_t i = 0; i < vec.size(); ++i)
{
if (!vec[i].name.compareNoCase(name))
{
return &vec[i];
}
}
return 0;
}
//////////////////////////////////////////////////////////////////////////
const char* CSystem::GetUserName()
{
@@ -231,30 +217,6 @@ int CSystem::GetApplicationLogInstance([[maybe_unused]] const char* logFilePath)
#endif
}
// these 2 functions are duplicated in System.cpp in editor
//////////////////////////////////////////////////////////////////////////
#if !defined(LINUX)
extern int CryStats(char* buf);
#endif
int CSystem::DumpMMStats(bool log)
{
#if defined(LINUX)
return 0;
#else
if (log)
{
char buf[1024];
int n = CryStats(buf);
GetILog()->Log(buf);
return n;
}
else
{
return CryStats(NULL);
};
#endif
};
//////////////////////////////////////////////////////////////////////////
struct CryDbgModule
{
@@ -264,273 +226,7 @@ struct CryDbgModule
DWORD dwSize;
};
//////////////////////////////////////////////////////////////////////////
void CSystem::DebugStats([[maybe_unused]] bool checkpoint, [[maybe_unused]] bool leaks)
{
#ifdef WIN32
std::vector<CryDbgModule> dbgmodules;
//////////////////////////////////////////////////////////////////////////
// Use windows Performance Monitoring API to enumerate all modules of current process.
//////////////////////////////////////////////////////////////////////////
HANDLE hSnapshot;
hSnapshot = CreateToolhelp32Snapshot (TH32CS_SNAPMODULE, 0);
if (hSnapshot != INVALID_HANDLE_VALUE)
{
MODULEENTRY32 me;
memset (&me, 0, sizeof(me));
me.dwSize = sizeof(me);
if (Module32First (hSnapshot, &me))
{
// the sizes of each module group
do
{
CryDbgModule module;
module.handle = me.hModule;
module.name = me.szModule;
module.dwSize = me.modBaseSize;
dbgmodules.push_back(module);
} while (Module32Next(hSnapshot, &me));
}
CloseHandle (hSnapshot);
}
//////////////////////////////////////////////////////////////////////////
int nolib = 0;
#ifdef _DEBUG
ILog* log = GetILog();
int totalal = 0;
int totalbl = 0;
int extrastats[10];
#endif
int totalUsedInModules = 0;
int countedMemoryModules = 0;
for (int i = 0; i < (int)(dbgmodules.size()); i++)
{
if (!dbgmodules[i].handle)
{
CryLogAlways("WARNING: CSystem::DebugStats: NULL handle for %s", dbgmodules[i].name.c_str());
nolib++;
continue;
}
;
typedef int (* PFN_MODULEMEMORY)();
PFN_MODULEMEMORY fpCryModuleGetAllocatedMemory = (PFN_MODULEMEMORY)::GetProcAddress((HMODULE)dbgmodules[i].handle, "CryModuleGetAllocatedMemory");
if (fpCryModuleGetAllocatedMemory)
{
int allocatedMemory = fpCryModuleGetAllocatedMemory();
totalUsedInModules += allocatedMemory;
countedMemoryModules++;
CryLogAlways("%8d K used in Module %s: ", allocatedMemory / 1024, dbgmodules[i].name.c_str());
}
#ifdef _DEBUG
typedef void (* PFNUSAGESUMMARY)(ILog* log, const char*, int*);
typedef void (* PFNCHECKPOINT)();
PFNUSAGESUMMARY fpu = (PFNUSAGESUMMARY)::GetProcAddress((HMODULE)dbgmodules[i].handle, "UsageSummary");
PFNCHECKPOINT fpc = (PFNCHECKPOINT)::GetProcAddress((HMODULE)dbgmodules[i].handle, "CheckPoint");
if (fpu && fpc)
{
if (checkpoint)
{
fpc();
}
else
{
extrastats[2] = (int)leaks;
fpu(log, dbgmodules[i].name.c_str(), extrastats);
totalal += extrastats[0];
totalbl += extrastats[1];
};
}
else
{
CryLogAlways("WARNING: CSystem::DebugStats: could not retrieve function from DLL %s", dbgmodules[i].name.c_str());
nolib++;
};
#endif
typedef HANDLE(* PFNGETDLLHEAP)();
PFNGETDLLHEAP fpg = (PFNGETDLLHEAP)::GetProcAddress((HMODULE)dbgmodules[i].handle, "GetDLLHeap");
if (fpg)
{
dbgmodules[i].heap = fpg();
}
;
}
;
CryLogAlways("-------------------------------------------------------");
CryLogAlways("%8d K Total Memory Allocated in %d Modules", totalUsedInModules / 1024, countedMemoryModules);
#ifdef _DEBUG
CryLogAlways("$8GRAND TOTAL: %d k, %d blocks (%d dlls not included)", totalal / 1024, totalbl, nolib);
CryLogAlways("estimated debugalloc overhead: between %d k and %d k", totalbl * 36 / 1024, totalbl * 72 / 1024);
#endif
//////////////////////////////////////////////////////////////////////////
// Get HeapQueryInformation pointer if on windows XP.
//////////////////////////////////////////////////////////////////////////
typedef BOOL (WINAPI * FUNC_HeapQueryInformation)(HANDLE, HEAP_INFORMATION_CLASS, PVOID, SIZE_T, PSIZE_T);
FUNC_HeapQueryInformation pFnHeapQueryInformation = NULL;
HMODULE hKernelInstance = CryLoadLibrary("Kernel32.dll");
if (hKernelInstance)
{
pFnHeapQueryInformation = (FUNC_HeapQueryInformation)(::GetProcAddress(hKernelInstance, "HeapQueryInformation"));
}
//////////////////////////////////////////////////////////////////////////
const int MAXHANDLES = 100;
HANDLE handles[MAXHANDLES];
int realnumh = GetProcessHeaps(MAXHANDLES, handles);
char hinfo[1024];
PROCESS_HEAP_ENTRY phe;
CryLogAlways("$6--------------------- dump of windows heaps ---------------------");
int nTotalC = 0, nTotalCP = 0, nTotalUC = 0, nTotalUCP = 0, totalo = 0;
for (int i = 0; i < realnumh; i++)
{
HANDLE hHeap = handles[i];
HeapCompact(hHeap, 0);
hinfo[0] = 0;
if (pFnHeapQueryInformation)
{
pFnHeapQueryInformation(hHeap, HeapCompatibilityInformation, hinfo, 1024, NULL);
}
else
{
for (int m = 0; m < (int)(dbgmodules.size()); m++)
{
if (dbgmodules[m].heap == handles[i])
{
azstrcpy(hinfo, AZ_ARRAY_SIZE(hinfo), dbgmodules[m].name.c_str());
}
}
}
phe.lpData = NULL;
int nCommitted = 0, nUncommitted = 0, nOverhead = 0;
int nCommittedPieces = 0, nUncommittedPieces = 0;
#if !defined(NDEBUG)
int nPrevRegionIndex = -1;
#endif
while (HeapWalk(hHeap, &phe))
{
if (phe.wFlags & PROCESS_HEAP_REGION)
{
assert (++nPrevRegionIndex == phe.iRegionIndex);
nCommitted += phe.Region.dwCommittedSize;
nUncommitted += phe.Region.dwUnCommittedSize;
assert (phe.cbData == 0 || (phe.wFlags & PROCESS_HEAP_ENTRY_BUSY));
}
else
if (phe.wFlags & PROCESS_HEAP_UNCOMMITTED_RANGE)
{
nUncommittedPieces += phe.cbData;
}
else
{
//if (phe.wFlags & PROCESS_HEAP_ENTRY_BUSY)
nCommittedPieces += phe.cbData;
}
{
/*
MEMORY_BASIC_INFORMATION mbi;
if (VirtualQuery(phe.lpData, &mbi,sizeof(mbi)) == sizeof(mbi))
{
if (mbi.State == MEM_COMMIT)
nCommittedPieces += phe.cbData;//mbi.RegionSize;
//else
// nUncommitted += mbi.RegionSize;
}
else
nCommittedPieces += phe.cbData;
*/
}
nOverhead += phe.cbOverhead;
}
CryLogAlways("* heap %8x: %6d (or ~%6d) K in use, %6d..%6d K uncommitted, %6d K overhead (%s)\n",
handles[i], nCommittedPieces / 1024, nCommitted / 1024, nUncommittedPieces / 1024, nUncommitted / 1024, nOverhead / 1024, hinfo);
nTotalC += nCommitted;
nTotalCP += nCommittedPieces;
nTotalUC += nUncommitted;
nTotalUCP += nUncommittedPieces;
totalo += nOverhead;
}
;
CryLogAlways("$6----------------- total in heaps: %d megs committed (win stats shows ~%d) (%d..%d uncommitted, %d k overhead) ---------------------", nTotalCP / 1024 / 1024, nTotalC / 1024 / 1024, nTotalUCP / 1024 / 1024, nTotalUC / 1024 / 1024, totalo / 1024);
#endif //WIN32
};
#ifdef WIN32
struct DumpHeap32Stats
{
DumpHeap32Stats()
: dwFree(0)
, dwMoveable(0)
, dwFixed(0)
, dwUnknown(0)
{
}
void operator += (const DumpHeap32Stats& right)
{
dwFree += right.dwFree;
dwMoveable += right.dwMoveable;
dwFixed += right.dwFixed;
dwUnknown += right.dwUnknown;
}
DWORD dwFree;
DWORD dwMoveable;
DWORD dwFixed;
DWORD dwUnknown;
};
static void DumpHeap32 (const HEAPLIST32& hl, DumpHeap32Stats& stats)
{
HEAPENTRY32 he;
memset (&he, 0, sizeof(he));
he.dwSize = sizeof(he);
if (Heap32First (&he, hl.th32ProcessID, hl.th32HeapID))
{
DumpHeap32Stats heap;
do
{
if (he.dwFlags & LF32_FREE)
{
heap.dwFree += he.dwBlockSize;
}
else
if (he.dwFlags & LF32_MOVEABLE)
{
heap.dwMoveable += he.dwBlockSize;
}
else
if (he.dwFlags & LF32_FIXED)
{
heap.dwFixed += he.dwBlockSize;
}
else
{
heap.dwUnknown += he.dwBlockSize;
}
} while (Heap32Next (&he));
CryLogAlways ("%08X %6d %6d %6d (%d)", hl.th32HeapID, heap.dwFixed / 0x400, heap.dwFree / 0x400, heap.dwMoveable / 0x400, heap.dwUnknown / 0x400);
stats += heap;
}
else
{
CryLogAlways ("%08X empty or invalid");
}
}
//////////////////////////////////////////////////////////////////////////
class CStringOrder
{
@@ -566,94 +262,6 @@ const char* GetModuleGroup (const char* szString)
#endif
//////////////////////////////////////////////////////////////////////////
void CSystem::DumpWinHeaps()
{
#ifdef WIN32
//
// Retrieve modules and log them; remember the process id
HANDLE hSnapshot;
hSnapshot = CreateToolhelp32Snapshot (TH32CS_SNAPMODULE, 0);
if (hSnapshot == INVALID_HANDLE_VALUE)
{
CryLogAlways ("Cannot get the module snapshot, error code %d", GetLastError());
return;
}
DWORD dwProcessID = GetCurrentProcessId();
MODULEENTRY32 me;
memset (&me, 0, sizeof(me));
me.dwSize = sizeof(me);
if (Module32First (hSnapshot, &me))
{
// the sizes of each module group
StringToSizeMap mapGroupSize;
DWORD dwTotalModuleSize = 0;
CryLogAlways ("base size module");
do
{
dwProcessID = me.th32ProcessID;
const char* szGroup = GetModuleGroup (me.szModule);
CryLogAlways ("%08X %8X %25s - %s", me.modBaseAddr, me.modBaseSize, me.szModule, azstricmp(szGroup, "Other") ? szGroup : "");
dwTotalModuleSize += me.modBaseSize;
AddSize (mapGroupSize, szGroup, me.modBaseSize);
} while (Module32Next(hSnapshot, &me));
CryLogAlways ("------------------------------------");
for (StringToSizeMap::iterator it = mapGroupSize.begin(); it != mapGroupSize.end(); ++it)
{
CryLogAlways (" %6.3f Mbytes - %s", double(it->second) / 0x100000, it->first);
}
CryLogAlways ("------------------------------------");
CryLogAlways (" %6.3f Mbytes - TOTAL", double(dwTotalModuleSize) / 0x100000);
CryLogAlways ("------------------------------------");
}
else
{
CryLogAlways ("No modules to dump");
}
CloseHandle (hSnapshot);
//
// Retrieve the heaps and dump each of them with a special function
hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPHEAPLIST, 0);
if (hSnapshot == INVALID_HANDLE_VALUE)
{
CryLogAlways ("Cannot get the heap LIST snapshot, error code %d", GetLastError());
return;
}
HEAPLIST32 hl;
memset (&hl, 0, sizeof(hl));
hl.dwSize = sizeof(hl);
CryLogAlways ("__Heap__ fixed free move (unknown)");
if (Heap32ListFirst (hSnapshot, &hl))
{
DumpHeap32Stats stats;
do
{
DumpHeap32 (hl, stats);
} while (Heap32ListNext (hSnapshot, &hl));
CryLogAlways ("-------------------------------------------------");
CryLogAlways ("$6 %6.3f %6.3f %6.3f (%.3f) Mbytes", double(stats.dwFixed) / 0x100000, double(stats.dwFree) / 0x100000, double(stats.dwMoveable) / 0x100000, double(stats.dwUnknown) / 0x100000);
CryLogAlways ("-------------------------------------------------");
}
else
{
CryLogAlways ("No heaps to dump");
}
CloseHandle(hSnapshot);
#endif
}
// Make system error message string
//////////////////////////////////////////////////////////////////////////
//! \return pointer to the null terminated error string or 0
@@ -739,8 +347,6 @@ void CSystem::FatalError(const char* format, ...)
assert(szBuffer[0] >= ' ');
// strcpy(szBuffer,szBuffer+1); // remove verbosity tag since it is not supported by ::MessageBox
LogSystemInfo();
OutputDebugString(szBuffer);
#ifdef WIN32
OnFatalError(szBuffer);
@@ -879,146 +485,6 @@ bool CSystem::ReLaunchMediaCenter()
}
#endif //defined(WIN32)
//////////////////////////////////////////////////////////////////////////
#if defined(WIN32)
void CSystem::LogSystemInfo()
{
//////////////////////////////////////////////////////////////////////
// Write the system informations to the log
//////////////////////////////////////////////////////////////////////
char szBuffer[1024];
char szProfileBuffer[128];
char szLanguageBuffer[64];
//char szCPUModel[64];
MEMORYSTATUSEX MemoryStatus;
MemoryStatus.dwLength = sizeof(MemoryStatus);
DEVMODE DisplayConfig;
OSVERSIONINFO OSVerInfo;
OSVerInfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
// log system language
GetLocaleInfo(LOCALE_SYSTEM_DEFAULT, LOCALE_SENGLANGUAGE, szLanguageBuffer, sizeof(szLanguageBuffer));
azsprintf(szBuffer, "System language: %s", szLanguageBuffer);
CryLogAlways(szBuffer);
// log Windows directory
GetWindowsDirectory(szBuffer, sizeof(szBuffer));
string str = "Windows Directory: \"";
str += szBuffer;
str += "\"";
CryLogAlways(str);
//////////////////////////////////////////////////////////////////////
// Send system time & date
//////////////////////////////////////////////////////////////////////
str = "Local time is ";
azstrtime(szBuffer);
str += szBuffer;
str += " ";
_strdate_s(szBuffer);
str += szBuffer;
azsprintf(szBuffer, ", system running for %lu minutes", GetTickCount() / 60000);
str += szBuffer;
CryLogAlways(str);
//////////////////////////////////////////////////////////////////////
// Send system memory status
//////////////////////////////////////////////////////////////////////
GlobalMemoryStatusEx(&MemoryStatus);
azsprintf(szBuffer, "%I64dMB physical memory installed, %I64dMB available, %I64dMB virtual memory installed, %ld percent of memory in use",
MemoryStatus.ullTotalPhys / 1048576 + 1,
MemoryStatus.ullAvailPhys / 1048576,
MemoryStatus.ullTotalVirtual / 1048576,
MemoryStatus.dwMemoryLoad);
CryLogAlways(szBuffer);
if (GetISystem()->GetIMemoryManager())
{
IMemoryManager::SProcessMemInfo memCounters;
GetISystem()->GetIMemoryManager()->GetProcessMemInfo(memCounters);
uint64 PagefileUsage = memCounters.PagefileUsage;
uint64 PeakPagefileUsage = memCounters.PeakPagefileUsage;
uint64 WorkingSetSize = memCounters.WorkingSetSize;
azsprintf(szBuffer, "PageFile usage: %I64dMB, Working Set: %I64dMB, Peak PageFile usage: %I64dMB,",
(uint64)PagefileUsage / (1024 * 1024),
(uint64)WorkingSetSize / (1024 * 1024),
(uint64)PeakPagefileUsage / (1024 * 1024));
CryLogAlways(szBuffer);
}
//////////////////////////////////////////////////////////////////////
// Send display settings
//////////////////////////////////////////////////////////////////////
EnumDisplaySettings(NULL, ENUM_CURRENT_SETTINGS, &DisplayConfig);
GetPrivateProfileString("boot.description", "display.drv",
"(Unknown graphics card)", szProfileBuffer, sizeof(szProfileBuffer),
"system.ini");
azsprintf(szBuffer, "Current display mode is %lux%lux%lu, %s",
DisplayConfig.dmPelsWidth, DisplayConfig.dmPelsHeight,
DisplayConfig.dmBitsPerPel, szProfileBuffer);
CryLogAlways(szBuffer);
//////////////////////////////////////////////////////////////////////
// Send input device configuration
//////////////////////////////////////////////////////////////////////
str = "";
// Detect the keyboard type
switch (GetKeyboardType(0))
{
case 1:
str = "IBM PC/XT (83-key)";
break;
case 2:
str = "ICO (102-key)";
break;
case 3:
str = "IBM PC/AT (84-key)";
break;
case 4:
str = "IBM enhanced (101/102-key)";
break;
case 5:
str = "Nokia 1050";
break;
case 6:
str = "Nokia 9140";
break;
case 7:
str = "Japanese";
break;
default:
str = "Unknown";
break;
}
// Any mouse attached ?
if (!GetSystemMetrics(SM_MOUSEPRESENT))
{
CryLogAlways(str + " keyboard and no mouse installed");
}
else
{
azsprintf(szBuffer, " keyboard and %i+ button mouse installed",
GetSystemMetrics(SM_CMOUSEBUTTONS));
CryLogAlways(str + szBuffer);
}
CryLogAlways("--------------------------------------------------------------------------------");
}
#else
void CSystem::LogSystemInfo()
{
}
#endif
#if (defined(WIN32) || defined(WIN64))
//////////////////////////////////////////////////////////////////////////
bool CSystem::GetWinGameFolder(char* szMyDocumentsPath, int maxPathSize)
Code/CryEngine/CrySystem/Tests/test_CrySystem.cpp
-7
View File
@@ -15,7 +15,6 @@
#include <AzCore/Memory/SystemAllocator.h>
#include <LegacyAllocator.h>
#include <System.h>
#include <CryMemoryManager.h>
namespace UnitTests
{
@@ -25,10 +24,6 @@ namespace UnitTests
public:
void SetUp() override
{
IMemoryManager* cryMemoryManager = nullptr;
CryGetIMemoryManagerInterface((void**)&cryMemoryManager);
AZ_Assert(cryMemoryManager, "Unable to resolve CryMemoryManager");
m_cryMemoryManager = AZ::Environment::CreateVariable<IMemoryManager*>("CryIMemoryManagerInterface", cryMemoryManager);
SSystemInitParams startupParams;
AZ::AllocatorInstance<AZ::LegacyAllocator>::Create();
AZ::AllocatorInstance<CryStringAllocator>::Create();
@@ -44,8 +39,6 @@ namespace UnitTests
CSystem* m_system = nullptr;
AZ::EnvironmentVariable<IMemoryManager*> m_cryMemoryManager;
};
TEST_F(CSystemUnitTests, ApplicationLogInstanceUnitTests)
Code/CryEngine/CrySystem/XML/XmlUtils.cpp
-1
View File
@@ -272,7 +272,6 @@ void CXmlUtils::OnSystemEvent(ESystemEvent event, [[maybe_unused]] UINT_PTR wpar
case ESYSTEM_EVENT_LEVEL_POST_UNLOAD:
case ESYSTEM_EVENT_LEVEL_LOAD_END:
g_pCXmlNode_PoolAlloc->FreeMemoryIfEmpty();
STLALLOCATOR_CLEANUP;
break;
}
}
Code/CryEngine/CrySystem/crysystem_files.cmake
-30
View File
@@ -19,13 +19,10 @@ set(FILES
ConsoleBatchFile.cpp
ConsoleHelpGen.cpp
CryAsyncMemcpy.cpp
GeneralMemoryHeap.cpp
HandlerBase.cpp
AsyncPakManager.cpp
Log.cpp
SystemRender.cpp
PhysRenderer.cpp
ResourceManager.cpp
ServerHandler.cpp
ServerThrottle.cpp
SyncLock.cpp
@@ -42,9 +39,7 @@ set(FILES
AutoDetectSpec.h
ClientHandler.h
HandlerBase.h
AsyncPakManager.h
PhysRenderer.h
ResourceManager.h
ServerHandler.h
ServerThrottle.h
SyncLock.h
@@ -58,7 +53,6 @@ set(FILES
ConsoleBatchFile.h
ConsoleHelpGen.h
CryWaterMark.h
GeneralMemoryHeap.h
Log.h
resource.h
SimpleStringPool.h
@@ -71,18 +65,6 @@ set(FILES
WindowsConsole.h
XConsole.h
XConsoleVariable.h
crash_face.bmp
ImageHandler.h
ImageHandler.cpp
MemoryAddressRange.cpp
PageMappingHeap.cpp
CustomMemoryHeap.cpp
MemoryManager.cpp
MTSafeAllocator.cpp
MemoryAddressRange.h
PageMappingHeap.h
MemoryManager.h
MTSafeAllocator.h
XML/SerializeXMLReader.cpp
XML/SerializeXMLWriter.cpp
XML/xml.cpp
@@ -126,18 +108,6 @@ set(FILES
ViewSystem/ViewSystem.h
ZStdDecompressor.h
ZStdDecompressor.cpp
StreamEngine/StreamAsyncFileRequest.cpp
StreamEngine/StreamAsyncFileRequest_Jobs.cpp
StreamEngine/StreamEngine.cpp
StreamEngine/StreamIOThread.cpp
StreamEngine/StreamReadStream.cpp
StreamEngine/AZRequestReadStream.cpp
StreamEngine/StreamAsyncFileRequest.h
StreamEngine/StreamEngine.h
StreamEngine/StreamIOThread.h
StreamEngine/StreamReadStream.h
StreamEngine/AZRequestReadStream.h
CrashHandler.rc
CrySystem_precompiled.cpp
CPUDetect.cpp
CPUDetect.h