birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
alexpete
2021-03-05 11:26:34 -08:00
commit a10351f38d
27091 changed files with 5521199 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/CryEngine/CrySystem/StreamEngine/AZRequestReadStream.cpp
+410
View File
@@ -0,0 +1,410 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Engine
#include "CrySystem_precompiled.h"
#include <ISystem.h>
#include <ILog.h>
#include "AZRequestReadStream.h"
#include <AzCore/std/parallel/semaphore.h>
#include <AzCore/IO/IStreamer.h>
#include <AzCore/Component/TickBus.h>
#include "StreamEngine.h"
AZRequestReadStream* AZRequestReadStream::Allocate(const EStreamTaskType tSource, const char* filename, IStreamCallback* callback,
const StreamReadParams* params)
{
//Once an async method is available to read file sizes this code should be removed:
// and the file size should be known before calling this method and pass it as a
// parameter to this method.
//REMOVE In the Future START.
AZ::IO::SizeType fileSize = 0;
if (params && params->nSize)
{
fileSize = params->nSize;
}
else
{
AZ::IO::FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
AZ::IO::Result res = fileIO->Size(filename, fileSize);
if (!res)
{
AZ_Error("AZRequestReadStream", false, "Failed to read file size of %s", filename);
return nullptr;
}
//REMOVE In the Future END.
}
auto streamer = AZ::Interface<AZ::IO::IStreamer>::Get();
AZRequestReadStream* retReq;
retReq = aznew AZRequestReadStream();
retReq->m_Type = tSource;
retReq->m_fileName = filename;
retReq->m_callback = callback;
retReq->m_fileSize = fileSize;
//REMARK: if params->pBuffer is NOT NULL, then retReq->m_buffer
//should become params->pBuffer, this is called stream-in-place.
//The only reason we are not doing this here is because
//platforms like Xenia support stream-in-place to WRITE ONLY buffers.
//Because there are no guarantees that low level streaming and decompression apis
//would treat the output buffer as WRITE ONLY, we still allocate the buffer and memcpy
//to params->pBuffer upon the completion callback being called.
//Once LY-98089 is complete/fixed, we should be able to safely
//set retReq->m_buffer = params->pBuffer and skip the memory allocation.
retReq->m_buffer = azmalloc(fileSize, streamer->GetRecommendations().m_memoryAlignment);
if (params)
{
retReq->m_params = *params;
}
return retReq;
}
//////////////////////////////////////////////////////////////////////////
AZRequestReadStream::AZRequestReadStream() : m_fileName(""), m_fileRequest(nullptr),
m_buffer(nullptr), m_Type(eStreamTaskTypeTexture),
m_callback(nullptr), m_fileSize(0), m_numBytesRead(0), m_isAsyncCallbackExecuted(false),
m_isSyncCallbackExecuted(false), m_isFileRequestComplete(false), m_isError(false), m_isFinished(false),
m_IOError(0)
{
AZStd::atomic_init<int>(&m_refCount, 0);
m_params = StreamReadParams();
}
//////////////////////////////////////////////////////////////////////////
AZRequestReadStream::~AZRequestReadStream()
{
azfree(m_buffer);
}
// tries to stop reading the stream; this is advisory and may have no effect
// all the callbacks will be called after this. If you just destructing object,
// dereference this object and it will automatically abort and release all associated resources.
void AZRequestReadStream::Abort()
{
{
CryAutoCriticalSection lock(m_callbackLock);
// Increase ref counting to avoid preliminary destruction
AZRequestReadStream_AutoPtr refCountLock(this);
if (m_isFileRequestComplete || m_isError)
{
// It is possible the file I/O request to be completed by AZ::IO::Streamer,
// but if the completion callback is deferred for the main thread then
// the stream is not finished. So, only if it is finished then
// it is safe to do nothing.
if (m_isFinished)
{
return;
}
}
m_isError = true;
m_IOError = ERROR_USER_ABORT;
m_isFileRequestComplete = true;
m_numBytesRead = 0;
if (m_fileRequest)
{
auto streamer = AZ::Interface<AZ::IO::IStreamer>::Get();
streamer->QueueRequest(streamer->Cancel(m_fileRequest));
}
// all the callbacks have to handle error cases and needs to be called anyway, even if the stream I/O is aborted
ExecuteAsyncCallback_CBLocked();
ExecuteSyncCallback_CBLocked();
m_callback = nullptr;
}
}
bool AZRequestReadStream::TryAbort()
{
// Increase ref counting to avoid preliminary destruction
AZRequestReadStream_AutoPtr refCountLock(this);
if (!m_callbackLock.TryLock())
{
return false;
}
if (m_isFileRequestComplete || m_isError)
{
// It is possible the file I/O request to be completed by AZ::IO::Streamer,
// but if the completion callback is deferred for the main thread then
// the stream is not finished. So, only if it is finished then
// it is safe to do nothing.
if (m_isFinished)
{
m_callbackLock.Unlock();
return false;
}
}
m_isError = true;
m_IOError = ERROR_USER_ABORT;
m_isFileRequestComplete = true;
m_numBytesRead = 0;
if (m_fileRequest)
{
auto streamer = AZ::Interface<AZ::IO::IStreamer>::Get();
streamer->QueueRequest(streamer->Cancel(m_fileRequest));
}
// all the callbacks have to handle error cases and needs to be called anyway, even if the stream I/O is aborted
ExecuteAsyncCallback_CBLocked();
ExecuteSyncCallback_CBLocked();
m_callback = nullptr;
m_callbackLock.Unlock();
return true;
}
// tries to raise the priority of the read; this is advisory and may have no effect
void AZRequestReadStream::SetPriority(EStreamTaskPriority ePriority)
{
CryAutoCriticalSection lock(m_callbackLock);
if (m_params.ePriority != ePriority)
{
m_params.ePriority = ePriority;
if (m_fileRequest)
{
AZ::Interface<AZ::IO::IStreamer>::Get()->RescheduleRequest(m_fileRequest, AZ::IO::IStreamerTypes::s_noDeadline,
CStreamEngine::CryStreamPriorityToAZStreamPriority(ePriority));
}
}
}
// unconditionally waits until the callback is called
// i.e. if the stream hasn't yet finish, it's guaranteed that the user-supplied callback
// is called before return from this function (unless no callback was specified)
void AZRequestReadStream::Wait(int maxWaitMillis)
{
// lock this object to avoid preliminary destruction
AZRequestReadStream_AutoPtr refCountLock(this);
if (!m_isFinished && !m_isError && !m_fileRequest)
{
AZ_Error("AZRequestReadStream", false, "Stream for file %s is unwaitable", m_fileName.c_str());
return;
}
if (maxWaitMillis > 0)
{
m_wait.try_acquire_for(AZStd::chrono::milliseconds(maxWaitMillis));
}
else
{
m_wait.acquire();
}
}
//////////////////////////////////////////////////////////////////////////
const char* AZRequestReadStream::GetErrorName() const
{
switch (m_IOError)
{
case ERROR_UNKNOWN_ERROR:
return "Unknown error";
case ERROR_UNEXPECTED_DESTRUCTION:
return "Unexpected destruction";
case ERROR_INVALID_CALL:
return "Invalid call";
case ERROR_CANT_OPEN_FILE:
return "Cannot open the file";
case ERROR_REFSTREAM_ERROR:
return "Refstream error";
case ERROR_OFFSET_OUT_OF_RANGE:
return "Offset out of range";
case ERROR_REGION_OUT_OF_RANGE:
return "Region out of range";
case ERROR_SIZE_OUT_OF_RANGE:
return "Size out of range";
case ERROR_CANT_START_READING:
return "Cannot start reading";
case ERROR_OUT_OF_MEMORY:
return "Out of memory";
case ERROR_ABORTED_ON_SHUTDOWN:
return "Aborted on shutdown";
case ERROR_OUT_OF_MEMORY_QUOTA:
return "Out of memory quota";
case ERROR_ZIP_CACHE_FAILURE:
return "ZIP cache failure";
case ERROR_USER_ABORT:
return "User aborted";
}
return "Unrecognized error";
}
int AZRequestReadStream::AddRef()
{
return m_refCount.fetch_add(1) + 1;
}
int AZRequestReadStream::Release()
{
int refCount = m_refCount.fetch_sub(1);
#ifndef _RELEASE
if (refCount < 1)
{
__debugbreak();
}
#endif
if (refCount == 1)
{
//UNUSUAL, yet necessary.
//Why "delete this"?
//So, AZRequestReadStream is a replacement of CReadStream. The original design of
//Cry Texture Mips Streaming makes use of CReadStream through IReadStreamPtr, which
//is a smart pointer design that calls AddRef() and Release() but never calls "delete",
//like AZStd::shared_ptr<> does. This means the original Cry design had a memory leak
//because it never called delete on IReadStream objects. If you look at the original
//code of CReadStream (StreamReadStream.cpp) , the static Allocate method has two paths
//to allocate memory, one used a stack based memory allocation hack, and the other path
//was doing a "new CReadStream". Using VS2017 debugger I found both paths to be used, but
//"delete" and hence the destructor of CReadStream is never called causing minor memory leaks.
//The best solution I found was to call "delete this" here and later when we chnage IReadStreamPtr
//for AZstd::smart_ptr then AddRef() and Release() won't be needed anymore and this "delete this"
//hack won't be necessary either.
delete this;
}
return refCount - 1;
}
//////////////////////////////////////////////////////////////////////////
void AZRequestReadStream::ExecuteAsyncCallback_CBLocked()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
if (!m_isAsyncCallbackExecuted && m_callback)
{
m_isAsyncCallbackExecuted = true;
m_callback->StreamAsyncOnComplete(this, m_IOError);
}
}
void AZRequestReadStream::ExecuteSyncCallback_CBLocked()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
if (!m_isSyncCallbackExecuted && m_callback && (0 == (m_params.nFlags & IStreamEngine::FLAGS_NO_SYNC_CALLBACK)))
{
m_isSyncCallbackExecuted = true;
AZRequestReadStream_AutoPtr refCountLock(this); // Stream can be freed inside the callback!
m_callback->StreamOnComplete(this, m_IOError);
m_isFinished = true;
FreeTemporaryMemory();
}
}
//////////////////////////////////////////////////////////////////////////
void AZRequestReadStream::FreeTemporaryMemory()
{
// Make sure m_buffer is not freed if the file request is still in flight, as Streamer can still write to m_buffer in that case
if (!m_fileRequest || AZ::Interface<AZ::IO::IStreamer>::Get()->HasRequestCompleted(m_fileRequest))
{
azfree(m_buffer);
m_buffer = nullptr;
m_numBytesRead = 0;
}
}
//////////////////////////////////////////////////////////////////////////
void AZRequestReadStream::OnRequestComplete(AZ::IO::SizeType numBytesRead, [[maybe_unused]] void* buffer, AZ::IO::IStreamerTypes::RequestStatus requestState)
{
CryAutoCriticalSection lock(m_callbackLock);
if (!m_isFileRequestComplete)
{
switch (requestState)
{
case AZ::IO::IStreamerTypes::RequestStatus::Completed:
m_IOError = 0;
m_numBytesRead = static_cast<uint32>(numBytesRead);
m_isError = false;
if (m_params.pBuffer)
{
//In some systems like Xenia, streaming-in-place is supported. The caveat
//is that in Xenia's case, the destination buffer is write-only. This is why
//a final memcpy must be done here until support is added to AZ::IO::Streamer API
//to decompress/load data into write-only buffers. SEE: LY-98089
AZ_Assert(m_params.pBuffer != m_buffer, "Streaming-In-Place requires destination and source buffers to be different");
memcpy(m_params.pBuffer, m_buffer, numBytesRead);
}
break;
case AZ::IO::IStreamerTypes::RequestStatus::Canceled:
m_IOError = ERROR_USER_ABORT;
m_numBytesRead = 0;
m_isError = true;
break;
default:
m_IOError = ERROR_UNKNOWN_ERROR;
m_numBytesRead = 0;
m_isError = true;
break;
}
ExecuteAsyncCallback_CBLocked();
m_isFileRequestComplete = true;
if (m_params.nFlags & IStreamEngine::FLAGS_NO_SYNC_CALLBACK)
{
// We do not need FileRequest here anymore, and not its temporary memory.
m_fileRequest = nullptr;
m_isFinished = true;
}
else
{
//The completion must be triggered from MainThread. (Typically only happens when loading Terrain Macro Textures
AddRef();
AZ::SystemTickBus::QueueFunction([this] {
RequestCompleteOnMainThread();
});
}
}
m_wait.release();
}
void AZRequestReadStream::RequestCompleteOnMainThread()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
// call asynchronous callback function if needed synchronously
{
CryAutoCriticalSection lock(m_callbackLock);
ExecuteSyncCallback_CBLocked();
}
//Always called because before enqueuing this call was called AddRef()
Release();
}
Code/CryEngine/CrySystem/StreamEngine/AZRequestReadStream.h
+151
View File
@@ -0,0 +1,151 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Description : An IReadStream implementation designed to work with AZ::IO::Streamer
// instead of CStreamEngine.
#pragma once
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/Memory/SystemAllocator.h>
#include <AzCore/std/parallel/semaphore.h>
#include <AzCore/std/smart_ptr/shared_ptr.h>
#include "IStreamEngine.h"
namespace AZ
{
namespace IO
{
class Request;
}
}
//This class is a wrapper of AZ::IO::Request so Cry Classes can use AZ::IO::Streamer.
//Basicallythis replaces CReadStream.
class AZRequestReadStream
: public IReadStream
{
public:
AZ_CLASS_ALLOCATOR(AZRequestReadStream, AZ::SystemAllocator, 0);
static AZRequestReadStream* Allocate(const EStreamTaskType tSource, const char* filename, IStreamCallback* callback,
const StreamReadParams* params);
int AddRef() override;
int Release() override;
DWORD_PTR GetUserData() override {return m_params.dwUserData; }
// set user defined data into stream's params
void SetUserData(DWORD_PTR userData) override { m_params.dwUserData = userData; };
// returns true if the file read was not successful.
bool IsError() override { return m_isError; };
// returns true if the file read was completed (successfully or unsuccessfully)
// check IsError to check if the whole requested file (piece) was read
bool IsFinished() override { return m_isFinished; };
// returns the number of bytes read so far (the whole buffer size if IsFinished())
unsigned int GetBytesRead([[maybe_unused]] bool bWait) override { return static_cast<unsigned int>(m_numBytesRead); };
// returns the buffer into which the data has been or will be read
// at least GetBytesRead() bytes in this buffer are guaranteed to be already read
const void* GetBuffer() override { return m_buffer; };
// tries to stop reading the stream; this is advisory and may have no effect
// but the callback will not be called after this. If you just destructing object,
// dereference this object and it will automatically abort and release all associated resources.
void Abort() override;
bool TryAbort() override;
// unconditionally waits until the callback is called
// i.e. if the stream hasn't yet finish, it's guaranteed that the user-supplied callback
// is called before return from this function (unless no callback was specified)
void Wait(int maxWaitMillis = -1) override;
const StreamReadParams& GetParams() const override {return m_params; }
const EStreamTaskType GetCallerType() const override { return m_Type; }
//We must define this one. But it is never used in the context of AZ::IO::Streamer.
//Legacy Cry StreamEngine stuff.
EStreamSourceMediaType GetMediaType() const override { return EStreamSourceMediaType::eStreamSourceTypeUnknown; }
// return pointer to callback routine(can be NULL)
IStreamCallback* GetCallback() const override { return m_callback; };
// return IO error #
unsigned GetError() const override { return m_IOError; };
// Returns IO error name
const char* GetErrorName() const override;
// return stream name
const char* GetName() const override { return m_fileName.c_str(); };
void FreeTemporaryMemory() override;
// tries to raise the priority of the read; this is advisory and may have no effect
void SetPriority(EStreamTaskPriority EPriority);
uint64 GetPriority() const { return m_params.ePriority; };
void* GetFileReadBuffer() { return m_buffer; } //GetBuffer from IReadStream is "const void *"
AZStd::size_t GetFileSize() { return m_fileSize; }
void SetFileRequest(AZ::IO::FileRequestPtr request) { m_fileRequest = AZStd::move(request); }
AZ::IO::FileRequestPtr GetFileRequest() { return m_fileRequest; }
void OnRequestComplete(AZ::IO::SizeType numBytesRead, void* buffer, AZ::IO::IStreamerTypes::RequestStatus requestState);
private:
AZRequestReadStream();
virtual ~AZRequestReadStream();
// call the async callback
void ExecuteAsyncCallback_CBLocked();
void ExecuteSyncCallback_CBLocked();
void RequestCompleteOnMainThread();
AZStd::atomic_int m_refCount;
CryCriticalSection m_callbackLock;
StreamReadParams m_params;
AZStd::semaphore m_wait;
CryStringLocal m_fileName;
AZ::IO::FileRequestPtr m_fileRequest;
// Bytes actually read from media.
void* m_buffer;
// the type of the task
EStreamTaskType m_Type;
// the initial data from the user
// the callback; may be NULL
IStreamCallback* m_callback;
AZ::IO::SizeType m_fileSize; //Expected number of bytes to be read.
AZ::IO::SizeType m_numBytesRead; //On a successful read m_nBytesRead == m_fileSize;
bool m_isAsyncCallbackExecuted;
bool m_isSyncCallbackExecuted;
bool m_isFileRequestComplete;
bool m_isError;
bool m_isFinished;
unsigned int m_IOError;
};
TYPEDEF_AUTOPTR(AZRequestReadStream);
Code/CryEngine/CrySystem/StreamEngine/StreamAsyncFileRequest.cpp
+1100
View File
File diff suppressed because it is too large Load Diff
Code/CryEngine/CrySystem/StreamEngine/StreamAsyncFileRequest.h
+607
View File
@@ -0,0 +1,607 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Thread for IO
#ifndef CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMASYNCFILEREQUEST_H
#define CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMASYNCFILEREQUEST_H
#pragma once
#include <IStreamEngineDefs.h>
#include <AzCore/Jobs/LegacyJobExecutor.h>
#include "TimeValue.h"
#define STREAMENGINE_LL_ALIGN _MS_ALIGN(MEMORY_ALLOCATION_ALIGNMENT)
class CStreamEngine;
class CAsyncIOFileRequest;
struct z_stream_s;
class CStreamingIOThread;
namespace AZ::IO
{
struct CCachedFileData;
}
class CCryFile;
struct SStreamJobEngineState;
class CMTSafeHeap;
class CAsyncIOFileRequest_TransferPtr;
struct SStreamEngineTempMemStats;
#ifdef SUPPORT_RSA_AND_STREAMCIPHER_PAK_ENCRYPTION //Could check for INCLUDE_LIBTOMCRYPT here, but only decryption is implemented here, not signing
#include "CryTomcrypt.h"
#endif
#if !defined(USE_EDGE_ZLIB)
// Prevent compilation conflicts - zconf.h (included by zlib.h) defines WINDOWS and WIN32 - those
// definitions conflict with CryEngine's definitions.
# if defined(CRY_TMP_DEFINED_WINDOWS) || defined(CRY_TMP_DEFINED_WIN32)
# error CRY_TMP_DEFINED_WINDOWS and/or CRY_TMP_DEFINED_WIN32 already defined
# endif
# if defined(WINDOWS)
# define CRY_TMP_DEFINED_WINDOWS 1
# endif
# if defined(WIN32)
# define CRY_TMP_DEFINED_WIN32 1
# endif
# include <zlib.h>
# if !defined(CRY_TMP_DEFINED_WINDOWS)
# undef WINDOWS
#endif
# undef CRY_TMP_DEFINED_WINDOWS
# if !defined(CRY_TMP_DEFINED_WIN32)
# undef WIN32
# endif
# undef CRY_TMP_DEFINED_WIN32
// Undefine macros defined in zutil.h to prevent compilation errors in 'steamclientpublic.h', 'OVR_Math.h' etc.
# undef Assert
# undef Trace
# undef Tracev
# undef Tracevv
# undef Tracec
# undef Tracecv
#endif // !defined(USE_EDGE_ZLIB)
namespace AZ::IO::ZipDir {
struct UncompressLookahead;
}
struct IAsyncIOFileCallback
{
virtual ~IAsyncIOFileCallback(){}
// Asynchronous finished event.
// Must be thread safe, can be called from a different thread.
virtual void OnAsyncFinished(CAsyncIOFileRequest* pFileRequest) = 0;
};
struct SStreamPageHdr
{
explicit SStreamPageHdr(int size)
: nRefs()
, nSize(size)
{}
volatile int nRefs;
int nSize;
};
struct SStreamJobQueue
{
enum
{
MaxJobs = 256,
};
struct Job
{
void* pSrc;
SStreamPageHdr* pSrcHdr;
uint32 nOffs;
uint32 nBytes : 31;
uint32 bLast : 1;
};
SStreamJobQueue()
: m_sema(MaxJobs, MaxJobs)
{
m_nQueueLen = 0;
m_nPush = 0;
m_nPop = 0;
memset(m_jobs, 0, sizeof(m_jobs));
}
void Flush(SStreamEngineTempMemStats& tms);
int Push(void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nOffs, uint32 nBytes, bool bLast);
int Pop();
CryFastSemaphore m_sema;
Job m_jobs[MaxJobs];
volatile int m_nQueueLen;
volatile int m_nPush;
volatile int m_nPop;
};
// This class represent a request to read some file from disk asynchronously via one of the IO threads.
class CAsyncIOFileRequest
{
public:
enum EStatus
{
eStatusNotReady,
eStatusInFileQueue,
eStatusFailed,
eStatusUnzipComplete,
eStatusDone,
};
enum
{
BUFFER_ALIGNMENT = 128,
WINDOW_SIZE = 1 << 15,
#if defined(ANDROID)
STREAMING_PAGE_SIZE = (128 * 1024),
#else
STREAMING_PAGE_SIZE = (1 * 1024 * 1024),
#endif
#if defined(ANDROID)
STREAMING_BLOCK_SIZE = (64 * 1024),
#else
STREAMING_BLOCK_SIZE = (32 * 1024),
#endif
};
public:
static CAsyncIOFileRequest* Allocate(EStreamTaskType eType);
static void Flush();
public:
void AddRef();
int Release();
public:
void Init(EStreamTaskType eType);
void Finalize();
void Reset();
ILINE bool IsCancelled() const { return m_nError == ERROR_USER_ABORT; }
ILINE bool HasFailed() const { return m_nError != 0; }
void Failed(uint32 nError)
{
CryInterlockedCompareExchange(reinterpret_cast<volatile LONG*>(&m_nError), nError, 0);
}
uint32 OpenFile(CCryFile& file);
uint32 ReadFile(CStreamingIOThread* pIOThread);
uint32 ReadFileResume(CStreamingIOThread* pIOThread);
uint32 ReadFileInPages(CStreamingIOThread* pIOThread, CCryFile& file);
uint32 ReadFileCheckPreempt(CStreamingIOThread* pIOThread);
uint32 ConfigureRead(AZ::IO::CCachedFileData* pFileData);
bool CanReadInPages();
uint32 AllocateOutput(AZ::IO::CCachedFileData* pZipEntry);
unsigned char* AllocatePage(size_t sz, bool bOnlyPakMem, SStreamPageHdr*& pHdrOut);
static void JobFinalize_Read(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState);
uint32 PushDecompressPage(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nBytes, bool bLast);
uint32 PushDecompressBlock(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nOffs, uint32 nBytes, bool bLast);
static void JobStart_Decompress(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState, int nSlot);
void DecompressBlockEntry(SStreamJobEngineState engineState, int nJob);
#if defined(STREAMENGINE_SUPPORT_DECRYPT)
uint32 PushDecryptPage(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nBytes, bool bLast);
uint32 PushDecryptBlock(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nOffs, uint32 nBytes, bool bLast);
static void JobStart_Decrypt(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState, int nSlot);
void DecryptBlockEntry(SStreamJobEngineState engineState, int nJob);
#endif //STREAMENGINE_SUPPORT_DECRYPT
void Cancel();
bool TryCancel();
void SyncWithDecrypt();
void SyncWithDecompress();
void ComputeSortKey(uint64 nCurrentKeyInProgress);
void SetPriority(EStreamTaskPriority estp);
void BumpSweep();
void FreeBuffer();
bool IgnoreOutofTmpMem() const;
CStreamEngine* GetStreamEngine();
EStreamSourceMediaType GetMediaType();
private:
void* operator new (size_t sz);
void operator delete(void* p);
private:
static void JobFinalize_Decompress(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState);
static void JobFinalize_Decrypt(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState);
static void JobFinalize_Transfer(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState);
private:
void JobFinalize_Buffer(const SStreamJobEngineState& engineState);
void JobFinalize_Validate(const SStreamJobEngineState& engineState);
private:
CAsyncIOFileRequest();
~CAsyncIOFileRequest();
public:
static volatile int s_nLiveRequests;
static SLockFreeSingleLinkedListHeader s_freeRequests;
public:
// Must be first
STREAMENGINE_LL_ALIGN SLockFreeSingleLinkedListEntry m_nextFree;
volatile int m_nRefCount;
// Locks to be held whilst the file is being read, and an external memory buffer is in use
// (to ensure that if cancelled, the stream engine doesn't write to the external buffer)
// Separate locks for read and decomp as they can overlap (block decompress)
// Cancel() must acquire both
CryCriticalSection m_externalBufferLockRead;
CryCriticalSection m_externalBufferLockDecompress;
#if defined(STREAMENGINE_SUPPORT_DECRYPT)
CryCriticalSection m_externalBufferLockDecrypt;
#endif //STREAMENGINE_SUPPORT_DECRYPT
CryStringLocal m_strFileName;
string m_pakFile;
#ifdef SUPPORT_RSA_AND_STREAMCIPHER_PAK_ENCRYPTION
string m_decryptionCTRInitialisedAgainst;
#endif //SUPPORT_RSA_AND_STREAMCIPHER_PAK_ENCRYPTION
// If request come from stream, it will be not 0.
IReadStreamPtr m_pReadStream;
#if defined(STREAMENGINE_SUPPORT_DECRYPT)
AZStd::unique_ptr<AZ::LegacyJobExecutor> m_decryptJobExecutor;
#endif //STREAMENGINE_SUPPORT_DECRYPT
AZStd::unique_ptr<AZ::LegacyJobExecutor> m_decompJobExecutor;
// Only POD data should exist beyond this point - will be memsetted to 0 on Reset !
uint64 m_nSortKey;
EStreamTaskPriority m_ePriority;
EStreamSourceMediaType m_eMediaType;
EStreamTaskType m_eType;
volatile EStatus m_status;
volatile uint32 m_nError;
uint32 m_nRequestedOffset;
uint32 m_nRequestedSize;
// the file size, or 0 if the file couldn't be opened
uint32 m_nFileSize;
uint32 m_nFileSizeCompressed;
void* m_pMemoryBuffer;
uint32 m_nMemoryBufferSize;
volatile int m_nMemoryBufferUsers;
void* m_pExternalMemoryBuffer;
void* m_pOutputMemoryBuffer;
void* m_pReadMemoryBuffer;
uint32 m_nReadMemoryBufferSize;
uint32 m_bCompressedBuffer : 1;
uint32 m_bEncryptedBuffer : 1;
uint32 m_bStatsUpdated : 1;
uint32 m_bStreamInPlace : 1;
uint32 m_bWriteOnlyExternal : 1;
uint32 m_bSortKeyComputed : 1;
uint32 m_bOutputAllocated : 1;
uint32 m_bReadBegun : 1;
// Actual size of the data on the media.
uint32 m_nSizeOnMedia;
int64 m_nDiskOffset;
int32 m_nReadHeadOffsetKB; // Offset of the Read Head when reading from media.
int32 m_nTimeGroup;
int32 m_nSweep;
IAsyncIOFileCallback* m_pCallback;
//
// Block based streaming
//
uint32 m_nPageReadStart;
uint32 m_nPageReadCurrent;
uint32 m_nPageReadEnd;
volatile uint32 m_nBytesDecompressed;
volatile uint32 m_nBytesDecrypted;
uint32 m_crc32FromHeader;
volatile LONG m_nFinalised;
z_stream_s* m_pZlibStream;
AZ::IO::ZipDir::UncompressLookahead* m_pLookahead;
SStreamJobQueue* m_pDecompQueue;
#if defined(STREAMENGINE_SUPPORT_DECRYPT)
SStreamJobQueue* m_pDecryptQueue;
#endif //STREAMENGINE_SUPPORT_DECRYPT
#ifdef SUPPORT_RSA_AND_STREAMCIPHER_PAK_ENCRYPTION
symmetric_CTR* m_pDecryptionCTR;
#endif //SUPPORT_RSA_AND_STREAMCIPHER_PAK_ENCRYPTION
#ifdef STREAMENGINE_ENABLE_STATS
// Time that read operation took.
CTimeValue m_readTime;
CTimeValue m_unzipTime;
CTimeValue m_verifyTime;
CTimeValue m_decryptTime;
CTimeValue m_startTime;
CTimeValue m_completionTime;
uint32 m_nReadCounter;
#endif
};
TYPEDEF_AUTOPTR(CAsyncIOFileRequest);
struct SStreamRequestQueue
{
CryCriticalSection m_lock;
std::vector<CAsyncIOFileRequest*> m_requests;
CryEvent m_awakeEvent;
SStreamRequestQueue();
~SStreamRequestQueue();
void Reset();
bool IsEmpty() const;
// Transfers ownership (rather than shares ownership) to the queue
void TransferRequest(CAsyncIOFileRequest_TransferPtr& pReq);
bool TryPopRequest(CAsyncIOFileRequest_AutoPtr& pOut);
private:
SStreamRequestQueue(const SStreamRequestQueue&);
SStreamRequestQueue& operator = (const SStreamRequestQueue&);
};
#if defined(STREAMENGINE_ENABLE_STATS)
struct SStreamEngineDecompressStats
{
uint64 m_nTotalBytesUnziped;
uint64 m_nTempBytesUnziped;
uint64 m_nTotalBytesDecrypted;
uint64 m_nTempBytesDecrypted;
uint64 m_nTotalBytesVerified;
uint64 m_nTempBytesVerified;
CTimeValue m_totalUnzipTime;
CTimeValue m_tempUnzipTime;
CTimeValue m_totalDecryptTime;
CTimeValue m_tempDecryptTime;
CTimeValue m_totalVerifyTime;
CTimeValue m_tempVerifyTime;
};
#endif
class CAsyncIOFileRequest_TransferPtr
{
public:
explicit CAsyncIOFileRequest_TransferPtr(CAsyncIOFileRequest* p)
: m_p(p)
{
}
~CAsyncIOFileRequest_TransferPtr()
{
if (m_p)
{
m_p->Release();
}
}
CAsyncIOFileRequest* operator -> () { return m_p; }
CAsyncIOFileRequest& operator * () { return *m_p; }
const CAsyncIOFileRequest* operator -> () const { return m_p; }
const CAsyncIOFileRequest& operator * () const { return *m_p; }
operator bool () const {
return m_p != NULL;
}
CAsyncIOFileRequest* Relinquish()
{
CAsyncIOFileRequest* p = m_p;
m_p = NULL;
return p;
}
CAsyncIOFileRequest_TransferPtr& operator = (CAsyncIOFileRequest* p)
{
#ifndef _RELEASE
if (m_p)
{
__debugbreak();
}
#endif
m_p = p;
return *this;
}
private:
CAsyncIOFileRequest_TransferPtr(const CAsyncIOFileRequest_TransferPtr&);
CAsyncIOFileRequest_TransferPtr& operator = (const CAsyncIOFileRequest_TransferPtr&);
private:
CAsyncIOFileRequest* m_p;
};
class CStreamEngineWakeEvent
{
public:
CStreamEngineWakeEvent()
: m_state(0)
{
}
void Set()
{
volatile LONG oldState, newState;
bool bSignalInner;
do
{
bSignalInner = false;
oldState = m_state;
newState = oldState | 0x80000000;
if (oldState & 0x7fffffff)
{
bSignalInner = true;
}
}
while (CryInterlockedCompareExchange(&m_state, newState, oldState) != oldState);
if (bSignalInner)
{
m_innerEvent.Set();
}
}
bool Wait(uint32 timeout = 0)
{
bool bTimedOut = false;
bool bAcquiredSignal = false;
while (!bTimedOut && !bAcquiredSignal)
{
volatile long oldState, newState;
do
{
bAcquiredSignal = false;
oldState = m_state;
if (oldState & 0x80000000)
{
// Signalled
newState = oldState & 0x7fffffff;
bAcquiredSignal = true;
}
else
{
newState = oldState + 1;
}
}
while (CryInterlockedCompareExchange(&m_state, newState, oldState) != oldState);
if (!bAcquiredSignal)
{
if (!timeout)
{
m_innerEvent.Wait();
}
else
{
bTimedOut = !m_innerEvent.Wait(timeout);
}
if (!bTimedOut)
{
m_innerEvent.Reset();
}
do
{
bAcquiredSignal = false;
oldState = m_state;
if (!bTimedOut && (oldState & 0x80000000))
{
newState = (oldState & 0x7fffffff) - 1;
bAcquiredSignal = true;
}
else
{
newState = oldState - 1;
}
}
while (CryInterlockedCompareExchange(&m_state, newState, oldState) != oldState);
}
}
return bAcquiredSignal;
}
private:
CStreamEngineWakeEvent(const CStreamEngineWakeEvent&);
CStreamEngineWakeEvent& operator = (const CStreamEngineWakeEvent&);
private:
volatile LONG m_state;
CryEvent m_innerEvent;
};
struct SStreamEngineTempMemStats
{
enum
{
MaxWakeEvents = 8,
};
SStreamEngineTempMemStats()
{
memset(this, 0, sizeof(*this));
}
void* TempAlloc(CMTSafeHeap* pHeap, size_t nSize, const char* szDbgSource, bool bFallBackToMalloc = true, bool bUrgent = false, uint32 align = 0);
void TempFree(CMTSafeHeap* pHeap, const void* p, size_t nSize);
void ReportTempMemAlloc(uint32 nSizeAlloc, uint32 nSizeFree, bool bTriggerWake);
volatile LONG m_nTempAllocatedMemory;
volatile LONG m_nTempAllocatedMemoryFrameMax;
int m_nTempMemoryBudget;
CStreamEngineWakeEvent* m_wakeEvents[MaxWakeEvents];
int m_nWakeEvents;
};
struct SStreamJobEngineState
{
std::vector<SStreamRequestQueue*>* pReportQueues;
#if defined(STREAMENGINE_ENABLE_STATS)
SStreamEngineStatistics* pStats;
SStreamEngineDecompressStats* pDecompressStats;
#endif
SStreamEngineTempMemStats* pTempMem;
CMTSafeHeap* pHeap;
};
#endif // CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMASYNCFILEREQUEST_H
Code/CryEngine/CrySystem/StreamEngine/StreamAsyncFileRequest_Jobs.cpp
+803
View File
@@ -0,0 +1,803 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySystem_precompiled.h"
#include <AzCore/Debug/Profiler.h>
#include <CryPath.h>
#include "StreamAsyncFileRequest.h"
#if defined(STREAMENGINE_SUPPORT_DECRYPT)
#include "ZipEncrypt.h"
#include "StreamEngine.h"
#endif //STREAMENGINE_SUPPORT_DECRYPT
#include "MTSafeAllocator.h"
namespace AZ::IO::ZipDir::ZipDirStructuresInternal
{
extern void ZlibInflateElementPartial_Impl(
int* pReturnCode, z_stream* pZStream, ZipDir::UncompressLookahead* pLookahead,
uint8_t* pOutput, size_t nOutputLen, bool bOutputWriteOnly,
const uint8_t* pInput, size_t nInputLen, size_t* pTotalOut);
}
#ifdef STREAMENGINE_ENABLE_LISTENER
#include "IStreamEngine.h"
class NotifyListener
{
public:
NotifyListener(IStreamEngineListener* pL, CAsyncIOFileRequest* pReq)
: m_pL(pL)
, m_pReq(pReq)
, m_bInProgress(false) {}
virtual ~NotifyListener() {}
protected:
IStreamEngineListener* m_pL;
CAsyncIOFileRequest* m_pReq;
bool m_bInProgress;
};
class NotifyListenerInflate
: NotifyListener
{
public:
NotifyListenerInflate(IStreamEngineListener* pL, CAsyncIOFileRequest* pReq)
: NotifyListener(pL, pReq)
{
if (m_pL)
{
m_pL->OnStreamBeginInflate(m_pReq);
m_bInProgress = true;
}
}
~NotifyListenerInflate()
{
End();
}
void End()
{
if (m_bInProgress)
{
m_pL->OnStreamEndInflate(m_pReq);
m_bInProgress = false;
}
}
};
class NotifyListenerDecrypt
: NotifyListener
{
public:
NotifyListenerDecrypt(IStreamEngineListener* pL, CAsyncIOFileRequest* pReq)
: NotifyListener(pL, pReq)
{
if (m_pL)
{
m_pL->OnStreamBeginDecrypt(m_pReq);
m_bInProgress = true;
}
}
~NotifyListenerDecrypt()
{
End();
}
void End()
{
if (m_bInProgress)
{
m_pL->OnStreamEndDecrypt(m_pReq);
m_bInProgress = false;
}
}
};
#endif
#if defined(STREAMENGINE_ENABLE_STATS)
#define STREAMENGINE_ENABLE_TIMING
#endif
//#define STREAM_DECOMPRESS_TRACE(...) OutputDebugString(AZStd::string::format(__VA_ARGS__).c_str());
#define STREAM_DECOMPRESS_TRACE(...)
void SStreamJobQueue::Flush(SStreamEngineTempMemStats& tms)
{
extern CMTSafeHeap* g_pPakHeap;
for (int c = m_nQueueLen, i = m_nPop % MaxJobs; c; --c, i = (i + 1) % MaxJobs)
{
Job& j = m_jobs[i];
if (j.pSrcHdr && CryInterlockedDecrement(&j.pSrcHdr->nRefs) == 0)
{
tms.TempFree(g_pPakHeap, j.pSrc, j.pSrcHdr->nSize);
}
j.pSrc = NULL;
}
}
int SStreamJobQueue::Push(void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nOffs, uint32 nBytes, bool bLast)
{
m_sema.Acquire();
int nSlot = (m_nPush++) % MaxJobs;
Job& j = m_jobs[nSlot];
j.pSrc = pSrc;
j.pSrcHdr = pSrcHdr;
j.nOffs = nOffs;
j.nBytes = nBytes;
j.bLast = (uint32)bLast;
bool bStartNext = CryInterlockedIncrement(&m_nQueueLen) == 1;
return bStartNext ? nSlot : -1;
}
int SStreamJobQueue::Pop()
{
int nSlot = (++m_nPop) % MaxJobs;
bool bStartNext = CryInterlockedDecrement(&m_nQueueLen) > 0;
m_sema.Release();
return bStartNext ? nSlot : -1;
}
void CAsyncIOFileRequest::AddRef()
{
int nRef = CryInterlockedIncrement(&m_nRefCount);
STREAM_DECOMPRESS_TRACE("[StreamDecompress],AddRef,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nRef);
}
int CAsyncIOFileRequest::Release()
{
int nRef = CryInterlockedDecrement(&m_nRefCount);
STREAM_DECOMPRESS_TRACE("[StreamDecompress],Release,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nRef);
#ifndef _RELEASE
if (nRef < 0)
{
__debugbreak();
}
#endif
if (nRef == 0)
{
Finalize();
CryInterlockedPushEntrySList(s_freeRequests, m_nextFree);
}
return nRef;
}
void CAsyncIOFileRequest::DecompressBlockEntry(SStreamJobEngineState engineState, int nJob)
{
AZ_PROFILE_FUNCTION(AZ::Debug::ProfileCategory::System);
STREAM_DECOMPRESS_TRACE("[StreamDecompress],DecompressBlockEntry,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nJob);
CAsyncIOFileRequest_TransferPtr pSelf(this);
SStreamJobQueue::Job& job = m_pDecompQueue->m_jobs[nJob];
void* pSrc = job.pSrc;
SStreamPageHdr* const pSrcHdr = job.pSrcHdr;
const uint32 nOffs = job.nOffs;
const uint32 nBytes = job.nBytes;
const bool bLast = job.bLast;
const bool bFailed = HasFailed();
if (!bFailed)
{
#if defined(STREAMENGINE_ENABLE_TIMING)
LARGE_INTEGER liStart;
QueryPerformanceCounter(&liStart);
#endif
//printf("Inflate: %s Avail in: %d, Avail Out: %d, Next In: 0x%p, Next Out: 0x%p\n", m_strFileName.c_str(), m_pZlibStream->avail_in, m_pZlibStream->avail_out, m_pZlibStream->next_in, m_pZlibStream->next_out);
#ifdef STREAMENGINE_ENABLE_LISTENER
NotifyListenerInflate inflateListener(gEnv->pSystem->GetStreamEngine()->GetListener(), this);
#endif
size_t nBytesDecomped = m_nBytesDecompressed;
STREAM_DECOMPRESS_TRACE ("[StreamDecompress],ZlibInflateElementPartial_Impl,0x%x,%s,0x%p,%i,0x%p,%i,%i\n",
CryGetCurrentThreadId(),
m_strFileName.c_str(),
(uint8_t*)m_pReadMemoryBuffer + nBytesDecomped,
m_nFileSize - nBytesDecomped,
(uint8_t*)pSrc + nOffs,
nBytes,
nBytesDecomped);
int readStatus = Z_OK;
{
CryOptionalAutoLock<CryCriticalSection> decompLock(m_externalBufferLockDecompress, m_pExternalMemoryBuffer != NULL);
AZ::IO::ZipDir::ZipDirStructuresInternal::ZlibInflateElementPartial_Impl(
&readStatus,
m_pZlibStream,
m_pLookahead,
(uint8_t*)m_pReadMemoryBuffer + nBytesDecomped,
m_nFileSize - nBytesDecomped,
m_bWriteOnlyExternal,
(uint8_t*)pSrc + nOffs,
nBytes,
&nBytesDecomped
);
}
m_nBytesDecompressed = nBytesDecomped;
//inform listen, so aysnc callback does not overlap
#ifdef STREAMENGINE_ENABLE_LISTENER
inflateListener.End();
#endif
if (readStatus == Z_OK || readStatus == Z_STREAM_END)
{
#if defined(STREAMENGINE_ENABLE_TIMING)
LARGE_INTEGER liEnd, liFreq;
QueryPerformanceCounter(&liEnd);
QueryPerformanceFrequency(&liFreq);
m_unzipTime += CTimeValue((int64)((liEnd.QuadPart - liStart.QuadPart) * CTimeValue::TIMEVALUE_PRECISION / liFreq.QuadPart));
#endif
}
else
{
#ifndef _RELEASE
AZ_Assert(false, "Decomp Error: %s : %s\n", m_strFileName.c_str(), m_pZlibStream ? m_pZlibStream->msg : "m_pZlibStream == NULL, no message available");
#endif
Failed(ERROR_DECOMPRESSION_FAIL);
}
}
if (pSrcHdr)
{
if (CryInterlockedDecrement(&pSrcHdr->nRefs) == 0)
{
engineState.pTempMem->TempFree(engineState.pHeap, pSrc, pSrcHdr->nSize);
}
}
job.pSrc = NULL;
int nPopSlot = m_pDecompQueue->Pop();
// job is no longer valid
if (HasFailed() || bLast)
{
JobFinalize_Decompress(pSelf, engineState);
}
else if (nPopSlot >= 0)
{
// Chain start the next job, we're responsible for it.
STREAM_DECOMPRESS_TRACE("[StreamDecompress],Chaining,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nPopSlot);
JobStart_Decompress(pSelf, engineState, nPopSlot);
}
#if defined(STREAMENGINE_ENABLE_STATS)
CryInterlockedDecrement(&engineState.pStats->nCurrentDecompressCount);
#endif
}
#if defined(STREAMENGINE_SUPPORT_DECRYPT)
//////////////////////////////////////////////////////////////////////////
void CAsyncIOFileRequest::DecryptBlockEntry(SStreamJobEngineState engineState, int nJob)
{
AZ_PROFILE_FUNCTION(AZ::Debug::ProfileCategory::System);
STREAM_DECOMPRESS_TRACE("[StreamDecrypt],DecryptBlockEntry,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nJob);
SStreamJobQueue::Job& job = m_pDecryptQueue->m_jobs[nJob];
void* const pSrc = job.pSrc;
SStreamPageHdr* const pSrcHdr = job.pSrcHdr;
const uint32 nOffs = job.nOffs;
const uint32 nBytes = job.nBytes;
const bool bLast = job.bLast;
const bool bFailed = HasFailed();
const bool bCompressed = m_bCompressedBuffer;
CAsyncIOFileRequest_TransferPtr pSelf(this);
bool decryptOK = false;
if (!bFailed)
{
#if defined(STREAMENGINE_ENABLE_TIMING)
LARGE_INTEGER liStart;
QueryPerformanceCounter(&liStart);
#endif
//printf("Inflate: %s Avail in: %d, Avail Out: %d, Next In: 0x%p, Next Out: 0x%p\n", m_strFileName.c_str(), m_pZlibStream->avail_in, m_pZlibStream->avail_out, m_pZlibStream->next_in, m_pZlibStream->next_out);
#ifdef STREAMENGINE_ENABLE_LISTENER
NotifyListenerDecrypt decryptListener(gEnv->pSystem->GetStreamEngine()->GetListener(), this);
#endif
unsigned long nBytesDecrypted = m_nBytesDecrypted;
uint8_t* pData = (uint8_t*)pSrc + nOffs;
//if (reinterpret_cast<UINT_PTR>(m_pExternalMemoryBuffer) < 0xc0000000 || reinterpret_cast<UINT_PTR>(m_pExternalMemoryBuffer) >= 0xd0000000)
{
CryOptionalAutoLock<CryCriticalSection> decryptLock(m_externalBufferLockDecrypt, m_pExternalMemoryBuffer != NULL);
if (0)
{
//Intentionally empty
}
#ifdef SUPPORT_RSA_AND_STREAMCIPHER_PAK_ENCRYPTION
else if (m_pDecryptionCTR)
{
STREAM_DECOMPRESS_TRACE ("[StreamDecrypt],ZipEncrypt::DecryptBufferWithStreamCipher,0x%x,%s,0x%p,%i,0x%p,%i,%i\n",
CryGetCurrentThreadId(),
m_strFileName.c_str(),
pData,
m_nFileSize - nBytesDecrypted,
(uint8_t*)pSrc + nOffs,
nBytes,
nBytesDecrypted);
decryptOK = ZipEncrypt::DecryptBufferWithStreamCipher(
pData, //In
pData, //Out - same = decrypt in place
nBytes,
m_pDecryptionCTR);
nBytesDecrypted += decryptOK ? nBytes : 0;
}
#endif
else
{
//Should never get here, this should have been checked in the prep functions
CryFatalError("Invalid encryption technique in streaming engine");
}
}
m_nBytesDecrypted = nBytesDecrypted;
//inform listen, so aysnc callback does not overlap
#ifdef STREAMENGINE_ENABLE_LISTENER
decryptListener.End();
#endif
if (decryptOK)
{
#if defined(STREAMENGINE_ENABLE_TIMING)
LARGE_INTEGER liEnd, liFreq;
QueryPerformanceCounter(&liEnd);
QueryPerformanceFrequency(&liFreq);
m_decryptTime += CTimeValue((int64)((liEnd.QuadPart - liStart.QuadPart) * CTimeValue::TIMEVALUE_PRECISION / liFreq.QuadPart));
#endif
}
else
{
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_ERROR_DBGBRK, "Decrypt Error: %s\n", m_strFileName.c_str());
Failed(ERROR_DECRYPTION_FAIL);
}
}
// FIXME later - if we end up here with a uncompressed request, that is not in-place, this won't copy
// to the output. Not currently an issue given how ConfigureRead sets up m_bStreamInPlace, but may be in
// future.
if (!decryptOK || !bCompressed) // Inverse of push condition below
{
if (pSrcHdr)
{
if (CryInterlockedDecrement(&pSrcHdr->nRefs) == 0)
{
engineState.pTempMem->TempFree(engineState.pHeap, pSrc, pSrcHdr->nSize);
}
}
}
int nPopSlot = m_pDecryptQueue->Pop();
// job is no longer valid
if (decryptOK && bCompressed)
{
PushDecompressBlock(engineState, pSrc, pSrcHdr, nOffs, nBytes, bLast);
if (pSrcHdr)
{
if (CryInterlockedDecrement(&pSrcHdr->nRefs) == 0)
{
engineState.pTempMem->TempFree(engineState.pHeap, pSrc, pSrcHdr->nSize);
}
}
}
if (HasFailed() || bLast)
{
JobFinalize_Decrypt(pSelf, engineState);
}
else if (nPopSlot >= 0)
{
// Chain start the next job, we're responsible for it.
STREAM_DECOMPRESS_TRACE("[StreamDecrypt],Chaining,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nPopSlot);
JobStart_Decrypt(pSelf, engineState, nPopSlot);
}
#if defined(STREAMENGINE_ENABLE_STATS)
CryInterlockedDecrement(&engineState.pStats->nCurrentDecryptCount);
#endif
}
#endif //STREAMENGINE_SUPPORT_DECRYPT
//////////////////////////////////////////////////////////////////////////
uint32 CAsyncIOFileRequest::PushDecompressPage(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nBytes, bool bLast)
{
uint32 nError = 0;
for (uint32 nBlockPos = 0; !nError && (nBlockPos < nBytes); nBlockPos += STREAMING_BLOCK_SIZE)
{
bool bLastBlock = (nBlockPos + STREAMING_BLOCK_SIZE) >= nBytes;
uint32 nBlockSize = min(nBytes - nBlockPos, (uint32)STREAMING_BLOCK_SIZE);
nError = PushDecompressBlock(engineState, pSrc, pSrcHdr, nBlockPos, nBlockSize, bLast && bLastBlock);
}
return nError;
}
uint32 CAsyncIOFileRequest::PushDecompressBlock(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nOffs, uint32 nBytes, bool bLast)
{
uint32 nError = m_nError;
if (!nError)
{
if (pSrcHdr)
{
CryInterlockedIncrement(&pSrcHdr->nRefs);
}
int nPushJob = m_pDecompQueue->Push(pSrc, pSrcHdr, nOffs, nBytes, bLast);
if (nPushJob >= 0)
{
STREAM_DECOMPRESS_TRACE("[StreamDecompress],PushDecompressBlock,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nPushJob);
AddRef();
CAsyncIOFileRequest_TransferPtr pSelf(this);
JobStart_Decompress(pSelf, engineState, nPushJob);
}
}
return nError;
}
void CAsyncIOFileRequest::JobStart_Decompress(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState, int nJob)
{
STREAM_DECOMPRESS_TRACE("[StreamDecompress],QueueDecompressBlockAppend,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), pSelf->m_strFileName.c_str(), &pSelf, nJob);
#if defined(STREAMENGINE_ENABLE_STATS)
CryInterlockedIncrement(&engineState.pStats->nCurrentDecompressCount);
#endif
CAsyncIOFileRequest* request = pSelf.Relinquish();
if (!request->m_decompJobExecutor)
{
request->m_decompJobExecutor = AZStd::make_unique<AZ::LegacyJobExecutor>();
}
request->m_decompJobExecutor->StartJob([request, engineState, nJob]()
{
request->DecompressBlockEntry(engineState, nJob);
}); // Legacy JobManager priority: eStreamPriority
}
#if defined(STREAMENGINE_SUPPORT_DECRYPT)
uint32 CAsyncIOFileRequest::PushDecryptPage(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nBytes, bool bLast)
{
uint32 nError = 0;
for (uint32 nBlockPos = 0; !nError && (nBlockPos < nBytes); nBlockPos += STREAMING_BLOCK_SIZE)
{
bool bLastBlock = (nBlockPos + STREAMING_BLOCK_SIZE) >= nBytes;
uint32 nBlockSize = min(nBytes - nBlockPos, (uint32)STREAMING_BLOCK_SIZE);
nError = PushDecryptBlock(engineState, pSrc, pSrcHdr, nBlockPos, nBlockSize, bLast && bLastBlock);
}
return nError;
}
uint32 CAsyncIOFileRequest::PushDecryptBlock(const SStreamJobEngineState& engineState, void* pSrc, SStreamPageHdr* pSrcHdr, uint32 nOffs, uint32 nBytes, bool bLast)
{
uint32 nError = m_nError;
if (!nError)
{
if (pSrcHdr)
{
CryInterlockedIncrement(&pSrcHdr->nRefs);
}
int nPushJob = m_pDecryptQueue->Push(pSrc, pSrcHdr, nOffs, nBytes, bLast);
if (nPushJob >= 0)
{
STREAM_DECOMPRESS_TRACE("[StreamDecrypt],PushDecryptBlock,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), m_strFileName.c_str(), this, nPushJob);
AddRef();
CAsyncIOFileRequest_TransferPtr pSelf(this);
JobStart_Decrypt(pSelf, engineState, nPushJob);
}
}
return nError;
}
void CAsyncIOFileRequest::JobStart_Decrypt(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState, int nJob)
{
STREAM_DECOMPRESS_TRACE("[StreamDecrypt],QueueDecryptBlockAppend,0x%x,%s,0x%p,%i\n", CryGetCurrentThreadId(), pSelf->m_strFileName.c_str(), &pSelf, nJob);
#if defined(STREAMENGINE_ENABLE_STATS)
CryInterlockedIncrement(&engineState.pStats->nCurrentDecryptCount);
#endif
CAsyncIOFileRequest* request = pSelf.Relinquish();
if (!request->m_decryptJobExecutor)
{
request->m_decryptJobExecutor = AZStd::make_unique<AZ::LegacyJobExecutor>();
}
request->m_decryptJobExecutor->StartJob([request, engineState, nJob]()
{
request->DecryptBlockEntry(engineState, nJob);
}); // Legacy JobManager priority: eStreamPriority
}
#endif //STREAMENGINE_SUPPORT_DECRYPT
//////////////////////////////////////////////////////////////////////////
void CAsyncIOFileRequest::JobFinalize_Read(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState)
{
if ((!pSelf->m_bCompressedBuffer && !pSelf->m_bEncryptedBuffer) || pSelf->HasFailed())
{
JobFinalize_Transfer(pSelf, engineState);
}
}
void CAsyncIOFileRequest::JobFinalize_Decompress(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState)
{
STREAM_DECOMPRESS_TRACE("[StreamDecompress],FinalizeDecompress,0x%x,%s,0x%p,0x%p,0x%p,0x%p\n", CryGetCurrentThreadId(), pSelf->m_strFileName.c_str(), &pSelf, &engineState, engineState.pStats, engineState.pDecompressStats);
CAsyncIOFileRequest* pReq = &*pSelf;
if (!pReq->HasFailed())
{
// Handle reads of subsections of a compressed file, by copying the section to the output
uint8_t* pDst = (uint8_t*)pReq->m_pOutputMemoryBuffer;
uint8_t* pSrc = (uint8_t*)pReq->m_pReadMemoryBuffer + pReq->m_nRequestedOffset;
if (pDst != pSrc)
{
memmove(pReq->m_pOutputMemoryBuffer, pSrc, pReq->m_nRequestedSize);
}
pReq->JobFinalize_Validate(engineState);
}
pReq->JobFinalize_Buffer(engineState);
#if defined(STREAMENGINE_ENABLE_STATS) && defined(STREAMENGINE_ENABLE_TIMING)
if (pReq->m_unzipTime.GetValue() != 0)
{
engineState.pDecompressStats->m_nTotalBytesUnziped += pReq->m_nFileSize;
engineState.pDecompressStats->m_totalUnzipTime += pReq->m_unzipTime;
engineState.pDecompressStats->m_nTempBytesUnziped += pReq->m_nFileSize;
engineState.pDecompressStats->m_tempUnzipTime += pReq->m_unzipTime;
}
#endif
JobFinalize_Transfer(pSelf, engineState);
}
void CAsyncIOFileRequest::JobFinalize_Decrypt(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState)
{
STREAM_DECOMPRESS_TRACE("[StreamDecompress],FinalizeDecompress,0x%x,%s,0x%p,0x%p,0x%p,0x%p\n", CryGetCurrentThreadId(), pSelf->m_strFileName.c_str(), &pSelf, &engineState, engineState.pStats, engineState.pDecompressStats);
CAsyncIOFileRequest* pReq = &*pSelf;
const bool bCompressed = pReq->m_bCompressedBuffer;
const bool bFailed = pReq->HasFailed();
if (!bCompressed && !bFailed)
{
pReq->JobFinalize_Validate(engineState);
}
pReq->JobFinalize_Buffer(engineState);
#if defined(STREAMENGINE_ENABLE_STATS) && defined(STREAMENGINE_ENABLE_TIMING)
if (pReq->m_decryptTime.GetValue() != 0)
{
engineState.pDecompressStats->m_nTotalBytesDecrypted += pReq->m_nFileSize;
engineState.pDecompressStats->m_totalDecryptTime += pReq->m_decryptTime;
engineState.pDecompressStats->m_nTempBytesDecrypted += pReq->m_nFileSize;
engineState.pDecompressStats->m_tempDecryptTime += pReq->m_decryptTime;
}
#endif
if (!bCompressed || pReq->HasFailed())
{
JobFinalize_Transfer(pSelf, engineState);
}
}
void CAsyncIOFileRequest::JobFinalize_Buffer(const SStreamJobEngineState& engineState)
{
if (CryInterlockedDecrement(&m_nMemoryBufferUsers) == 0)
{
z_stream_s* pZlib = m_pZlibStream;
if (pZlib)
{
//if the stream was cancelled in flight, inform zlib to free internal allocs
if (pZlib->state)
{
inflateEnd(pZlib);
}
m_pZlibStream = NULL;
}
if (m_pMemoryBuffer)
{
engineState.pTempMem->TempFree(engineState.pHeap, m_pMemoryBuffer, m_nMemoryBufferSize);
m_pMemoryBuffer = NULL;
m_nMemoryBufferSize = 0;
}
}
}
void CAsyncIOFileRequest::JobFinalize_Validate([[maybe_unused]] const SStreamJobEngineState& engineState)
{
#if defined(SKIP_CHECKSUM_FROM_OPTICAL_MEDIA)
if (m_eMediaType != eStreamSourceTypeDisc)
#endif //SKIP_CHECKSUM_FROM_OPTICAL_MEDIA
{
CryOptionalAutoLock<CryCriticalSection> readLock(m_externalBufferLockRead, m_pExternalMemoryBuffer != NULL);
if (!HasFailed())
{
if (m_crc32FromHeader != 0 && m_nPageReadStart == 0 && m_nRequestedSize == m_nFileSize) //Compute the CRC32 if appropriate.
{
#if defined(STREAMENGINE_ENABLE_TIMING)
LARGE_INTEGER liStart, liEnd, liFreq;
QueryPerformanceCounter(&liStart);
#endif //STREAMENGINE_ENABLE_TIMING
uint32 nCRC32 = crc32(0, (uint8_t*)m_pReadMemoryBuffer + m_nPageReadStart, m_nRequestedSize);
#if defined(STREAMENGINE_ENABLE_TIMING)
QueryPerformanceCounter(&liEnd);
QueryPerformanceFrequency(&liFreq);
m_verifyTime += CTimeValue((int64)((liEnd.QuadPart - liStart.QuadPart) * CTimeValue::TIMEVALUE_PRECISION / liFreq.QuadPart));
engineState.pDecompressStats->m_nTotalBytesVerified += m_nFileSize;
engineState.pDecompressStats->m_totalVerifyTime += m_verifyTime;
engineState.pDecompressStats->m_nTempBytesVerified += m_nFileSize;
engineState.pDecompressStats->m_tempVerifyTime += m_verifyTime;
#endif //STREAMENGINE_ENABLE_TIMING
if (m_crc32FromHeader != nCRC32)
{
//The contents of this file don't match what the header expects
#if !defined(_RELEASE)
CryWarning(VALIDATOR_MODULE_SYSTEM, VALIDATOR_ERROR_DBGBRK, "Streaming Engine Failed to verify a file (%s). Computed CRC32 %d does not match stored CRC32 %d", m_strFileName.c_str(), nCRC32, m_crc32FromHeader);
#endif //!_RELEASE
Failed(ERROR_VERIFICATION_FAIL);
}
}
}
}
}
void CAsyncIOFileRequest::JobFinalize_Transfer(CAsyncIOFileRequest_TransferPtr& pSelf, const SStreamJobEngineState& engineState)
{
STREAM_DECOMPRESS_TRACE("[StreamDecompress],FinalizeTransform,0x%x,%s,0x%p,0x%p,0x%p,0x%p\n", CryGetCurrentThreadId(), pSelf->m_strFileName.c_str(), &pSelf, &engineState, engineState.pStats, engineState.pDecompressStats);
if (CryInterlockedCompareExchange(&pSelf->m_nFinalised, 1, 0) == 0)
{
#if defined(STREAMENGINE_ENABLE_STATS)
CryInterlockedIncrement(&engineState.pStats->nCurrentAsyncCount);
#endif
#if defined(STREAMENGINE_ENABLE_TIMING)
pSelf->m_completionTime = gEnv->pTimer->GetAsyncTime();
#endif
int nCallbackThreads = engineState.pReportQueues->size();
EStreamTaskType eType = pSelf->m_eType;
if (nCallbackThreads > 1 && eType == eStreamTaskTypeGeometry)
{
// If we have more then 1 call back threads, use this one for geometry only.
(*engineState.pReportQueues)[1]->TransferRequest(pSelf);
}
else if (nCallbackThreads > 2 && eType == eStreamTaskTypeTexture)
{
// If we have more then 1 call back threads, use this one for textures only.
(*engineState.pReportQueues)[2]->TransferRequest(pSelf);
}
else if (nCallbackThreads > 3 && eType == eStreamTaskTypeMergedMesh)
{
// If we have more then 3 call back threads, use this one for merged meshes only.
(*engineState.pReportQueues)[3]->TransferRequest(pSelf);
}
else if (nCallbackThreads > 0)
{
(*engineState.pReportQueues)[0]->TransferRequest(pSelf);
}
else
{
__debugbreak();
}
}
}
//////////////////////////////////////////////////////////////////////////
void SStreamRequestQueue::TransferRequest(CAsyncIOFileRequest_TransferPtr& pRequest)
{
{
CryAutoLock<CryCriticalSection> l(m_lock);
m_requests.push_back(pRequest.Relinquish());
}
m_awakeEvent.Set();
}
//////////////////////////////////////////////////////////////////////////
void SStreamEngineTempMemStats::TempFree(CMTSafeHeap* pHeap, const void* p, size_t nSize)
{
#if MTSAFE_USE_GENERAL_HEAP
bool bInGenHeap = pHeap->IsInGeneralHeap(p);
#else
bool bInGenHeap = false;
#endif
pHeap->FreeTemporary(const_cast<void*>(p));
ReportTempMemAlloc(0, bInGenHeap ? 0 : nSize, true);
}
void SStreamEngineTempMemStats::ReportTempMemAlloc(uint32 nSizeAlloc, uint32 nSizeFree, bool bTriggerWake)
{
int nAdd = (int)nSizeAlloc - (int)nSizeFree;
int const nOldSize = CryInterlockedExchangeAdd(&m_nTempAllocatedMemory, nAdd);
int const nNewSize = nOldSize + nAdd;
LONG nNewMax = 0;
LONG nOldMax = 0;
do
{
nOldMax = m_nTempAllocatedMemoryFrameMax;
nNewMax = (LONG)max((int)nNewSize, (int)nOldMax);
}
while (CryInterlockedCompareExchange(&m_nTempAllocatedMemoryFrameMax, nNewMax, nOldMax) != nOldMax);
if (bTriggerWake)
{
for (int i = 0, c = m_nWakeEvents; i != c; ++i)
{
m_wakeEvents[i]->Set();
}
}
}
Code/CryEngine/CrySystem/StreamEngine/StreamEngine.cpp
+1722
View File
File diff suppressed because it is too large Load Diff
Code/CryEngine/CrySystem/StreamEngine/StreamEngine.h
+241
View File
@@ -0,0 +1,241 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Engine
#ifndef CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMENGINE_H
#define CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMENGINE_H
#pragma once
#include "IStreamEngine.h"
#include "ISystem.h"
#include "TimeValue.h"
#include <CryThread.h>
#include "StreamIOThread.h"
#include "StreamReadStream.h"
#include <AzFramework/Input/Events/InputChannelEventListener.h>
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/std/chrono/clocks.h>
#include <AzCore/std/containers/queue.h>
enum EIOThread
{
eIOThread_HDD = 0,
eIOThread_Optical = 1,
eIOThread_InMemory = 2,
eIOThread_Last = 3,
};
//////////////////////////////////////////////////////////////////////////
class CStreamEngine
: public IStreamEngine
, public ISystemEventListener
, public AzFramework::InputChannelEventListener
{
public:
CStreamEngine();
~CStreamEngine();
void Shutdown();
// This is called to cancel all pending requests, without sending callbacks.
void CancelAll();
//Helper added to aid in migration from Cry's CStreamEngine to AZ::IO::Streamer
static AZ::IO::IStreamerTypes::Priority CryStreamPriorityToAZStreamPriority(EStreamTaskPriority cryPriority);
static AZStd::chrono::milliseconds AZDeadlineFromReadParams(const StreamReadParams& params);
//////////////////////////////////////////////////////////////////////////
// IStreamEngine interface
//////////////////////////////////////////////////////////////////////////
IReadStreamPtr StartRead (const EStreamTaskType tSource, const char* szFile, IStreamCallback* pCallback, const StreamReadParams* pParams = NULL);
size_t StartBatchRead(IReadStreamPtr* pStreamsOut, const StreamReadBatchParams* pReqs, size_t numReqs, AZStd::function<void()>* preRequestCallback = nullptr);
void BeginReadGroup();
void EndReadGroup();
bool IsStreamDataOnHDD() const { return m_bStreamDataOnHDD; }
void SetStreamDataOnHDD(bool bFlag) { m_bStreamDataOnHDD = bFlag; }
void Update();
void UpdateAndWait(bool bAbortAll = false);
void Update(uint32 nUpdateTypesBitmask);
void GetMemoryStatistics(ICrySizer* pSizer);
#if defined(STREAMENGINE_ENABLE_STATS)
SStreamEngineStatistics& GetStreamingStatistics();
void ClearStatistics();
void GetBandwidthStats(EStreamTaskType type, float* bandwidth);
#endif
void GetStreamingOpenStatistics(SStreamEngineOpenStats& openStatsOut);
const char* GetStreamTaskTypeName(EStreamTaskType type);
SStreamJobEngineState GetJobEngineState();
SStreamEngineTempMemStats& GetTempMemStats() { return m_tempMem; }
// Will pause or unpause streaming of specified by mask data types
void PauseStreaming(bool bPause, uint32 nPauseTypesBitmask);
// Pause/resumes any IO active from the streaming engine
void PauseIO(bool bPause);
uint32 GetPauseMask() const { return m_nPausedDataTypesMask; }
#if defined(STREAMENGINE_ENABLE_LISTENER)
void SetListener(IStreamEngineListener* pListener);
IStreamEngineListener* GetListener();
#endif
//////////////////////////////////////////////////////////////////////////
// updates the job priority of an IO job into the IOQueue while maintaining order in the queue
void UpdateJobPriority(IReadStreamPtr pJobStream);
void ReportAsyncFileRequestComplete(CAsyncIOFileRequest_AutoPtr pFileRequest);
void AbortJob(CReadStream* pStream);
// Dispatches synchrnous callbacks, free temporary memory hold for callbacks.
void MainThread_FinalizeIOJobs();
void MainThread_FinalizeIOJobs(uint32 type);
void* TempAlloc(size_t nSize, const char* szDbgSource, bool bFallBackToMalloc = true, bool bUrgent = false, uint32 align = 0);
void TempFree(void* p, size_t nSize);
uint32 GetCurrentTempMemorySize() const { return m_tempMem.m_nTempAllocatedMemory; }
void FlagTempMemOutOfBudget()
{
#ifdef STREAMENGINE_ENABLE_STATS
m_bTempMemOutOfBudget = true;
#endif
}
//////////////////////////////////////////////////////////////////////////
// AzFramework::InputChannelEventListener
//////////////////////////////////////////////////////////////////////////
bool OnInputChannelEventFiltered(const AzFramework::InputChannel& inputChannel) override;
//////////////////////////////////////////////////////////////////////////
bool StartFileRequest(CAsyncIOFileRequest* pFileRequest);
void SignalToStartWork(EIOThread e, bool bForce);
private:
void StartThreads();
void StopThreads();
void ResumePausedStreams_PauseLocked();
#if defined(STREAMENGINE_ENABLE_STATS)
// add job to current statistics
void UpdateStatistics(CReadStream* pReadStream);
void DrawStatistics();
#endif
void QueueRequestCompleteJob(class AZRequestReadStream* stream, AZ::IO::SizeType numBytesRead, void* buffer,
AZ::IO::IStreamerTypes::RequestStatus requestState);
//////////////////////////////////////////////////////////////////////////
// ISystemEventListener
//////////////////////////////////////////////////////////////////////////
virtual void OnSystemEvent(ESystemEvent event, UINT_PTR wparam, UINT_PTR lparam);
//////////////////////////////////////////////////////////////////////////
private:
//////////////////////////////////////////////////////////////////////////
CryMT::set<CReadStream_AutoPtr> m_streams;
CryMT::vector<CReadStream_AutoPtr> m_finishedStreams;
std::vector<CReadStream_AutoPtr> m_tempFinishedStreams;
CryCriticalSection m_pendingRequestCompletionsLock;
AZStd::queue<AZ::Job*> m_pendingRequestCompletions;
// 2 IO threads.
_smart_ptr<CStreamingIOThread> m_pThreadIO[eIOThread_Last];
std::vector<_smart_ptr<CStreamingWorkerThread> > m_asyncCallbackThreads;
std::vector<SStreamRequestQueue*> m_asyncCallbackQueues;
CryCriticalSection m_pausedLock;
std::vector<CReadStream_AutoPtr> m_pausedStreams;
volatile uint32 m_nPausedDataTypesMask;
bool m_bStreamDataOnHDD;
bool m_bUseOpticalDriveThread;
//////////////////////////////////////////////////////////////////////////
// Streaming statistics.
//////////////////////////////////////////////////////////////////////////
#ifdef STREAMENGINE_ENABLE_LISTENER
IStreamEngineListener* m_pListener;
#endif
#ifdef STREAMENGINE_ENABLE_STATS
SStreamEngineStatistics m_Statistics;
SStreamEngineDecompressStats m_decompressStats;
CTimeValue m_TimeOfLastReset;
CTimeValue m_TimeOfLastUpdate;
CryCriticalSection m_csStats;
std::vector<CAsyncIOFileRequest_AutoPtr> m_statsRequestList;
struct SExtensionInfo
{
SExtensionInfo()
: m_fTotalReadTime(0.0f)
, m_nTotalRequests(0)
, m_nTotalReadSize(0)
, m_nTotalRequestSize(0)
{
}
float m_fTotalReadTime;
size_t m_nTotalRequests;
uint64 m_nTotalReadSize;
uint64 m_nTotalRequestSize;
};
typedef std::map<string, SExtensionInfo> TExtensionInfoMap;
TExtensionInfoMap m_PerExtensionInfo;
//////////////////////////////////////////////////////////////////////////
// Used to calculate unzip/decrypt/verify bandwidth for statistics.
uint32 m_nUnzipBandwidth;
uint32 m_nUnzipBandwidthAverage;
uint32 m_nDecryptBandwidth;
uint32 m_nDecryptBandwidthAverage;
uint32 m_nVerifyBandwidth;
uint32 m_nVerifyBandwidthAverage;
CTimeValue m_nLastBandwidthUpdateTime;
bool m_bStreamingStatsPaused;
bool m_bInputCallback;
bool m_bTempMemOutOfBudget;
//////////////////////////////////////////////////////////////////////////
#endif
SStreamEngineOpenStats m_OpenStatistics;
bool m_bShutDown;
volatile int m_nBatchMode;
// Memory currently allocated by streaming engine for temporary storage.
SStreamEngineTempMemStats m_tempMem;
};
#endif // CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMENGINE_H
Code/CryEngine/CrySystem/StreamEngine/StreamIOThread.cpp
+815
View File
@@ -0,0 +1,815 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Thread for IO
#include "CrySystem_precompiled.h"
#include "StreamIOThread.h"
#include "StreamEngine.h"
#include "../System.h"
extern SSystemCVars g_cvars;
//#pragma("control %push O=0") // to disable optimization
//////////////////////////////////////////////////////////////////////////
CStreamingIOThread::CStreamingIOThread(CStreamEngine* pStreamEngine, EStreamSourceMediaType mediaType, const char* name)
{
m_pStreamEngine = pStreamEngine;
m_bCancelThreadRequest = false;
m_bNeedSorting = false;
m_bNeedReset = false;
m_bNewRequests = false;
m_name = name;
m_eMediaType = mediaType;
m_nFallbackMTs = 0;
m_iUrgentRequests = 0;
m_bPaused = false;
m_bAbortReads = false;
m_nReadCounter = 0;
m_nStreamingCPU = -1;
Start((unsigned)(1 << g_cvars.sys_streaming_cpu), name);
}
CStreamingIOThread::~CStreamingIOThread()
{
Cancel();
Stop();
WaitForThread();
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::AddRequest(CAsyncIOFileRequest* pRequest, bool bStartImmidietly)
{
pRequest->AddRef(); // Acquire ownership on file request.
pRequest->m_status = CAsyncIOFileRequest::eStatusInFileQueue;
if (pRequest->m_eMediaType != eStreamSourceTypeMemory)
{
pRequest->m_eMediaType = m_eMediaType;
}
// does this ignore the tmp out of memory
if (pRequest->IgnoreOutofTmpMem())
{
CryInterlockedIncrement(&m_iUrgentRequests);
}
m_newFileRequests.push_back(pRequest);
if (bStartImmidietly)
{
READ_WRITE_BARRIER
m_bNewRequests = true;
m_awakeEvent.Set();
}
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::SignalStartWork(bool bForce)
{
if (!m_newFileRequests.empty() || bForce)
{
READ_WRITE_BARRIER
m_bNewRequests = true;
m_awakeEvent.Set();
}
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::Pause(bool bPause)
{
m_bPaused = bPause;
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::Run()
{
SetName(m_name);
CTimeValue t0 = gEnv->pTimer->GetAsyncTime();
m_nLastReadDiskOffset = 0;
//
// Main thread loop
while (!m_bCancelThreadRequest)
{
if (m_nStreamingCPU != g_cvars.sys_streaming_cpu)
{
m_nStreamingCPU = g_cvars.sys_streaming_cpu;
#if defined(AZ_RESTRICTED_PLATFORM)
#undef AZ_RESTRICTED_SECTION
#define STREAMIOTHREAD_CPP_SECTION_1 1
#define STREAMIOTHREAD_CPP_SECTION_2 2
#endif
#if defined(AZ_RESTRICTED_PLATFORM)
#define AZ_RESTRICTED_SECTION STREAMIOTHREAD_CPP_SECTION_1
#include AZ_RESTRICTED_FILE(StreamEngine/StreamIOThread_cpp)
#endif
}
if (m_bNewRequests || !m_newFileRequests.empty())
{
READ_WRITE_BARRIER
ProcessNewRequests();
}
else
{
#if defined(_RELEASE)
m_awakeEvent.Wait();
#elif defined(STREAMENGINE_ENABLE_STATS)
// compute max time to wait - revive thread every second at least once to update stats
bool bWaiting = true;
while (bWaiting)
{
CTimeValue t1 = gEnv->pTimer->GetAsyncTime();
CTimeValue deltaT = t1 - t0;
uint64 msec = deltaT.GetMilliSecondsAsInt64();
if (msec < 1000)
{
bWaiting = !m_awakeEvent.Wait(1000 - (uint32)msec);
}
if (bWaiting)
{
// update the delta time again
t1 = gEnv->pTimer->GetAsyncTime();
deltaT = t1 - t0;
m_InMemoryStats.Update(deltaT);
m_NotInMemoryStats.Update(deltaT);
t0 = t1;
}
}
#endif
}
if (m_bNeedReset)
{
ProcessReset();
}
bool bIsOOM = false;
while (!m_bCancelThreadRequest && !m_fileRequestQueue.empty())
{
CAsyncIOFileRequest_TransferPtr pFileRequest(m_fileRequestQueue.back());
m_fileRequestQueue.pop_back();
assert (&*pFileRequest);
if (pFileRequest->HasFailed())
{
// check if request was high prio, then decr open count
if (pFileRequest->IgnoreOutofTmpMem())
{
CryInterlockedDecrement(&m_iUrgentRequests);
}
CAsyncIOFileRequest::JobFinalize_Read(pFileRequest, m_pStreamEngine->GetJobEngineState());
continue;
}
//////////////////////////////////////////////////////////////////////////
// When temporary memory goes out of budget we must loop here and wait until previous file requests are finished and free up memory.
// Only allow processing of requests which are flagged for processing when out of tmp memory
//////////////////////////////////////////////////////////////////////////
if (bIsOOM && !m_bCancelThreadRequest)
{
m_pStreamEngine->FlagTempMemOutOfBudget();
if (m_iUrgentRequests > 0)
{
if (m_bNewRequests || !m_newFileRequests.empty())
{
READ_WRITE_BARRIER
ProcessNewRequests();
}
// readd the current request
m_fileRequestQueue.push_back(pFileRequest.Relinquish());
// search for the first request which ignores the current out of mem state
// Search for next highest priority request
std::vector<CAsyncIOFileRequest*>::reverse_iterator rit;
for (rit = m_fileRequestQueue.rbegin(); rit != m_fileRequestQueue.rend(); ++rit)
{
if ((*rit)->IgnoreOutofTmpMem())
{
pFileRequest = *rit;
std::vector<CAsyncIOFileRequest*>::iterator it(rit.base());
--it;
m_fileRequestQueue.erase(it);
break;
}
}
}
else
{
// read the current request
m_fileRequestQueue.push_back(pFileRequest.Relinquish());
}
}
// Simply let the io thread sleep when paused before doing any actual IO
while (m_bPaused)
{
CrySleep(10);
}
// If at this point, the filerequest is zero, the above prioritization of
// urgent requests couldn't find a new task to displace the current
// one. As the current one had been pushed back previously, we can safely
// assume that restarting the loop will grab it again (eventually).
if (!pFileRequest)
{
break;
}
// check if request was high prio, then decr open count
if (pFileRequest->IgnoreOutofTmpMem())
{
CryInterlockedDecrement(&m_iUrgentRequests);
}
bIsOOM = false;
uint32 nSizeOnMedia = pFileRequest->m_nSizeOnMedia;
uint32 nError = 0;
// Handle file request.
if (m_bAbortReads)
{
nError = ERROR_ABORTED_ON_SHUTDOWN;
}
else if (pFileRequest->m_bReadBegun)
{
nError = pFileRequest->ReadFileResume(this);
}
else
{
nError = pFileRequest->ReadFile(this);
}
#ifdef STREAMENGINE_ENABLE_STATS
pFileRequest->m_nReadCounter = m_nReadCounter++;
#endif
if (nError == 0)
{
if (pFileRequest->m_eMediaType != eStreamSourceTypeMemory)
{
pFileRequest->m_nReadHeadOffsetKB = (int32)(((int64)pFileRequest->m_nDiskOffset - m_nLastReadDiskOffset) >> 10); // in KB
m_nLastReadDiskOffset = pFileRequest->m_nDiskOffset + pFileRequest->m_nSizeOnMedia;
#ifdef STREAMENGINE_ENABLE_STATS
m_NotInMemoryStats.m_nTempReadOffset += abs(pFileRequest->m_nReadHeadOffsetKB);
m_NotInMemoryStats.m_nTotalReadOffset += abs(pFileRequest->m_nReadHeadOffsetKB);
m_NotInMemoryStats.m_nTempRequestCount++;
// Calc IO bandwidth only for non memory files.
m_NotInMemoryStats.m_nTempBytesRead += pFileRequest->m_nSizeOnMedia;
m_NotInMemoryStats.m_TempReadTime += pFileRequest->m_readTime;
#endif
}
else
{
#ifdef STREAMENGINE_ENABLE_STATS
m_InMemoryStats.m_nTempRequestCount++;
// Calc IO bandwidth only for in memory files.
m_InMemoryStats.m_nTempBytesRead += pFileRequest->m_nSizeOnMedia;
m_InMemoryStats.m_TempReadTime += pFileRequest->m_readTime;
#endif
}
CAsyncIOFileRequest::JobFinalize_Read(pFileRequest, m_pStreamEngine->GetJobEngineState());
}
else
{
switch (nError)
{
case ERROR_OUT_OF_MEMORY:
bIsOOM = true;
pFileRequest->SetPriority(estpPreempted);
if (pFileRequest->IgnoreOutofTmpMem())
{
CryInterlockedIncrement(&m_iUrgentRequests);
}
m_fileRequestQueue.push_back(pFileRequest.Relinquish());
m_bNewRequests = true;
break;
case ERROR_PREEMPTED:
pFileRequest->SetPriority(estpPreempted);
if (pFileRequest->IgnoreOutofTmpMem())
{
CryInterlockedIncrement(&m_iUrgentRequests);
}
m_fileRequestQueue.push_back(pFileRequest.Relinquish());
m_bNewRequests = true;
break;
case ERROR_MISSCHEDULED:
// Request tried to read a file that has changed media type. Reset the sort key
// and reschedule.
pFileRequest->m_bSortKeyComputed = 0;
AddRequest(&*pFileRequest, false);
break;
default:
pFileRequest->SyncWithDecrypt();
pFileRequest->SyncWithDecompress();
pFileRequest->Failed(nError);
CAsyncIOFileRequest::JobFinalize_Read(pFileRequest, m_pStreamEngine->GetJobEngineState());
break;
}
}
//////////////////////////////////////////////////////////////////////////
if (m_bNewRequests)
{
READ_WRITE_BARRIER
ProcessNewRequests();
}
if (m_bNeedReset)
{
ProcessReset();
}
if (m_bNeedSorting)
{
SortRequests();
}
//////////////////////////////////////////////////////////////////////////
#ifdef STREAMENGINE_ENABLE_STATS
if (g_cvars.sys_streaming_max_bandwidth != 0)
{
CTimeValue t1 = gEnv->pTimer->GetAsyncTime();
CTimeValue deltaT = t1 - t0;
// Sleep in case we are streaming too fast.
const float fTheoreticalReadTime = float(nSizeOnMedia) / g_cvars.sys_streaming_max_bandwidth * 0.00000095367431640625f; // / (1024*1024)
if (fTheoreticalReadTime - deltaT.GetSeconds() > FLT_EPSILON)
{
uint32 nSleepTime = uint32(1000.f * (fTheoreticalReadTime - deltaT.GetSeconds()));
CrySleep(nSleepTime);
}
}
CTimeValue t1 = gEnv->pTimer->GetAsyncTime();
CTimeValue deltaT = t1 - t0;
// update the stats every second
if (deltaT.GetMilliSecondsAsInt64() > 1000)
{
m_InMemoryStats.Update(deltaT);
m_NotInMemoryStats.Update(deltaT);
t0 = t1;
}
#endif
}
}
}
#ifdef STREAMENGINE_ENABLE_STATS
void CStreamingIOThread::SStats::Update(const CTimeValue& deltaT)
{
m_nReadBytesInLastSecond = (uint32)m_nTempBytesRead;
m_nRequestCountInLastSecond = m_nTempRequestCount;
m_nTotalReadBytes += (uint32)m_nTempBytesRead;
m_nTotalRequestCount += m_nTempRequestCount;
m_TotalReadTime += m_TempReadTime;
if (m_TempReadTime.GetValue() != 0)
{
m_nActualReadBandwith = (uint32)(m_nTempBytesRead / m_TempReadTime.GetSeconds());
}
else
{
m_nActualReadBandwith = 0;
}
m_nCurrentReadBandwith = (uint32)(m_nTempBytesRead / deltaT.GetSeconds());
m_fReadingDuringLastSecond = m_TempReadTime.GetSeconds() / deltaT.GetSeconds() * 100;
if (m_nTempRequestCount > 0)
{
m_nReadOffsetInLastSecond = m_nTempReadOffset / m_nTempRequestCount;
}
else
{
m_nReadOffsetInLastSecond = 0;
}
m_TempReadTime.SetValue(0);
m_nTempBytesRead = 0;
m_nTempReadOffset = 0;
m_nTempRequestCount = 0;
}
#endif
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::Cancel()
{
m_bCancelThreadRequest = true;
m_awakeEvent.Set();
}
//////////////////////////////////////////////////////////////////////////
struct SCompareAsyncFileRequest
{
bool operator()(CAsyncIOFileRequest* pFile1, CAsyncIOFileRequest* pFile2) const
{
return pFile1->m_nSortKey > pFile2->m_nSortKey;
}
};
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::SortRequests()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
std::sort(m_fileRequestQueue.begin(), m_fileRequestQueue.end(), SCompareAsyncFileRequest());
/*
int nStartOfQueue = 0;
int64 nDiskOffsetLimit = m_nLastReadDiskOffset - 32*1024; // 32KB less only
int nCount = (int)m_fileRequestQueue.size();
for (int i = nCount-1; i >= 0; i--)
{
if (m_fileRequestQueue[i]->m_nDiskOffset > nDiskOffsetLimit)
{
nStartOfQueue = i+1;
break;
}
}
if (nStartOfQueue < nCount && nStartOfQueue > 0)
{
int nElements = nCount - nStartOfQueue;
// Move all elements up to nStartOfQueue, from begining of the request array to the end.
m_temporaryArray.resize(0);
// Copy to temp array elements up to nStartOfQueue
m_temporaryArray.insert( m_temporaryArray.end(),m_fileRequestQueue.begin()+nStartOfQueue,m_fileRequestQueue.end() );
// Remove elements up to nStartOfQueue from request list
m_fileRequestQueue.erase( m_fileRequestQueue.begin()+nStartOfQueue,m_fileRequestQueue.end() );
// Add elemenets at the end from temp array.
m_fileRequestQueue.insert( m_fileRequestQueue.begin(),m_temporaryArray.begin(),m_temporaryArray.end() );
}
*/
m_bNeedSorting = false;
}
void CStreamingIOThread::NeedSorting()
{
m_bNeedSorting = true;
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::ProcessNewRequests()
{
m_bNewRequests = false;
std::vector<CAsyncIOFileRequest*> temporaryArray;
temporaryArray.reserve(m_newFileRequests.size());
m_newFileRequests.swap(temporaryArray);
std::vector<CAsyncIOFileRequest*>& newFiles = temporaryArray;
if (!newFiles.empty())
{
uint64 nCurrentKeyInProgress = m_fileRequestQueue.size() ? m_fileRequestQueue.back()->m_nSortKey : 0;
// Compute sorting key for new file entries.
int iWakeFallback(0);
const TFallbackIOVecConstIt itEnd = m_FallbackIOThreads.end();
const size_t fallbackNum = m_FallbackIOThreads.size();
PREFAST_SUPPRESS_WARNING(6255)
uint8 * pFallbackSignals = fallbackNum ? (uint8*)alloca(fallbackNum) : NULL;
for (uint32 fb = 0; fb < fallbackNum; ++fb)
{
pFallbackSignals[fb] = 0;
}
for (size_t i = 0, num = newFiles.size(); i < num; i++)
{
CAsyncIOFileRequest* pFilepRequest = newFiles[i];
pFilepRequest->ComputeSortKey(nCurrentKeyInProgress);
static_cast<CReadStream*>(&*pFilepRequest->m_pReadStream)->ComputedMediaType(pFilepRequest->m_eMediaType);
#ifdef STREAMENGINE_ENABLE_LISTENER
IStreamEngineListener* pListener = m_pStreamEngine->GetListener();
if (pListener)
{
pListener->OnStreamComputedSortKey(pFilepRequest, pFilepRequest->m_nSortKey);
}
#endif
bool bFallback = false;
int idx = -1;
for (TFallbackIOVecConstIt it = m_FallbackIOThreads.begin(); it != itEnd && !bFallback; ++it)
{
++idx;
if (it->second == pFilepRequest->GetMediaType())
{
if (pFilepRequest->IgnoreOutofTmpMem())
{
CryInterlockedDecrement(&m_iUrgentRequests);
}
(it->first)->AddRequest(pFilepRequest, true);
pFilepRequest->Release(); // Release local ownership of request (moved to fallback IO thread)
iWakeFallback++;
bFallback = true;
pFallbackSignals[idx] = 1;
}
}
if (!bFallback)
{
m_fileRequestQueue.push_back(pFilepRequest);
}
}
for (uint32 fb = 0; fb < fallbackNum; ++fb)
{
if (pFallbackSignals[fb] != 0)
{
(m_FallbackIOThreads[fb].first)->SignalStartWork(false);
}
}
SortRequests();
/*
if (m_fileRequestQueue.back() != pRequest && pRequest != 0)
{
// Highest priority changed.
if (m_fileRequestQueue.back()->m_nDiskOffset < (m_nLastReadDiskOffset-32*1024))
{
//CryLog( "Bad Offset in Queue" );
}
}
*/
}
}
void CStreamingIOThread::ProcessReset()
{
if (!m_fileRequestQueue.empty())
{
for (std::vector<CAsyncIOFileRequest*>::iterator it = m_fileRequestQueue.begin(), itEnd = m_fileRequestQueue.end(); it != itEnd; ++it)
{
(*it)->Release();
}
}
stl::free_container(m_fileRequestQueue);
if (!m_temporaryArray.empty())
{
for (std::vector<CAsyncIOFileRequest*>::iterator it = m_temporaryArray.begin(), itEnd = m_temporaryArray.end(); it != itEnd; ++it)
{
(*it)->Release();
}
}
stl::free_container(m_temporaryArray);
m_bNeedReset = false;
m_resetDoneEvent.Set();
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::CancelAll()
{
{
CryMT::vector<CAsyncIOFileRequest*>::AutoLock lock(m_newFileRequests.get_lock());
if (!m_newFileRequests.empty())
{
CAsyncIOFileRequest* const* it = &m_newFileRequests.front();
CAsyncIOFileRequest* const* itEnd = it + m_newFileRequests.size();
for (; it != itEnd; ++it)
{
(*it)->Release();
}
}
}
m_newFileRequests.free_memory();
m_iUrgentRequests = 0;
}
void CStreamingIOThread::AbortAll(bool bAbort)
{
m_bAbortReads = bAbort;
}
void CStreamingIOThread::BeginReset()
{
CancelAll();
m_resetDoneEvent.Reset();
m_bNeedReset = true;
m_awakeEvent.Set();
}
void CStreamingIOThread::EndReset()
{
m_resetDoneEvent.Wait();
}
//////////////////////////////////////////////////////////////////////////
void CStreamingIOThread::RegisterFallbackIOThread(EStreamSourceMediaType mediaType, CStreamingIOThread* pIOThread)
{
//check if media has not yet been registered
if (!pIOThread)
{
return;//no need for NULL register anymore
}
const TFallbackIOVecConstIt itEnd = m_FallbackIOThreads.end();
for (TFallbackIOVecConstIt it = m_FallbackIOThreads.begin(); it != itEnd; ++it)
{
if (it->second == mediaType)
{
return;
}
}
m_FallbackIOThreads.push_back(std::make_pair(pIOThread, mediaType));
m_nFallbackMTs |= 1 << mediaType;
}
bool CStreamingIOThread::HasUrgentRequests()
{
bool ret = false;
if (m_iUrgentRequests > 0)
{
//lock to prevent list modification whilst traversing
m_newFileRequests.get_lock().Lock();
int nRequests = m_newFileRequests.size();
if (nRequests)
{
for (int i = 0; i < nRequests; i++)
{
if (m_newFileRequests[i]->m_ePriority == estpUrgent)
{
//printf("Urgent task pending: %s\n", m_newFileRequests[i]->m_strFileName.c_str());
ret = true;
break;
}
}
}
m_newFileRequests.get_lock().Unlock();
}
return ret;
}
bool CStreamingIOThread::IsMisscheduled(EStreamSourceMediaType mt) const
{
if (mt == m_eMediaType)
{
return false;
}
if (m_nFallbackMTs & (1 << mt))
{
return true;
}
return false;
}
//////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////
CStreamingWorkerThread::CStreamingWorkerThread(CStreamEngine* pStreamEngine, const char* name, EWorkerType type, SStreamRequestQueue* pQueue)
{
m_type = type;
m_name = name;
m_pStreamEngine = pStreamEngine;
m_pQueue = pQueue;
m_bCancelThreadRequest = false;
m_bNeedsReset = false;
Start((unsigned)1 << g_cvars.sys_streaming_cpu_worker, name);
}
CStreamingWorkerThread::~CStreamingWorkerThread()
{
Cancel();
Stop();
WaitForThread();
}
//////////////////////////////////////////////////////////////////////////
void CStreamingWorkerThread::Run()
{
SetName(m_name);
#if defined(AZ_RESTRICTED_PLATFORM)
#define AZ_RESTRICTED_SECTION STREAMIOTHREAD_CPP_SECTION_2
#include AZ_RESTRICTED_FILE(StreamEngine/StreamIOThread_cpp)
#endif
// Main thread loop
while (!m_bCancelThreadRequest)
{
m_pQueue->m_awakeEvent.Wait();
m_pQueue->m_awakeEvent.Reset();
CAsyncIOFileRequest_AutoPtr pFileRequest;
while (!m_bCancelThreadRequest && !m_bNeedsReset && m_pQueue->TryPopRequest(pFileRequest))
{
switch (m_type)
{
case eWorkerAsyncCallback:
{
float fTime = gEnv->pTimer->GetAsyncCurTime();
m_pStreamEngine->ReportAsyncFileRequestComplete(pFileRequest);
float fTime1 = gEnv->pTimer->GetAsyncCurTime();
#ifdef STREAMENGINE_ENABLE_STATS
CryInterlockedDecrement(&m_pStreamEngine->GetStreamingStatistics().nCurrentAsyncCount);
#endif
#ifndef _RELEASE
if ((fTime1 - fTime) > 1.f && !pFileRequest->m_strFileName.empty())
{
string str;
str.Format("[ACALL] %s time=%.5f\n", pFileRequest->m_strFileName.c_str(), (fTime1 - fTime));
if (gEnv && gEnv->pSystem && gEnv->pLog)
{
gEnv->pLog->Log(str.c_str());
}
}
#endif
}
break;
}
}
if (m_bNeedsReset)
{
m_pQueue->Reset();
m_bNeedsReset = false;
m_resetDoneEvent.Set();
}
}
}
//////////////////////////////////////////////////////////////////////////
void CStreamingWorkerThread::Cancel()
{
m_bCancelThreadRequest = true;
m_pQueue->m_awakeEvent.Set();
}
//////////////////////////////////////////////////////////////////////////
void CStreamingWorkerThread::CancelAll()
{
m_pQueue->Reset();
}
void CStreamingWorkerThread::BeginReset()
{
CancelAll();
m_resetDoneEvent.Reset();
m_bNeedsReset = true;
m_pQueue->m_awakeEvent.Set();
}
void CStreamingWorkerThread::EndReset()
{
m_resetDoneEvent.Wait();
}
Code/CryEngine/CrySystem/StreamEngine/StreamIOThread.h
+193
View File
@@ -0,0 +1,193 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Thread for IO
#ifndef CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMIOTHREAD_H
#define CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMIOTHREAD_H
#pragma once
#include <IStreamEngine.h>
#include "StreamAsyncFileRequest.h"
class CStreamEngine;
//////////////////////////////////////////////////////////////////////////
// Thread that performs IO operations.
//////////////////////////////////////////////////////////////////////////
class CStreamingIOThread
: public CrySimpleThread<CStreamingIOThread>
, public CMultiThreadRefCount
{
public:
CStreamingIOThread(CStreamEngine* pStreamEngine, EStreamSourceMediaType mediaType, const char* name);
~CStreamingIOThread();
void CancelAll();
void AbortAll(bool bAbort);
void BeginReset();
void EndReset();
void AddRequest(CAsyncIOFileRequest* pRequest, bool bStartImmidietly);
int GetRequestCount() const { return m_fileRequestQueue.size(); };
void SortRequests();
void NeedSorting();
void SignalStartWork(bool bForce);
bool HasUrgentRequests();
EStreamSourceMediaType GetMediaType() const { return m_eMediaType; }
bool IsMisscheduled(EStreamSourceMediaType mt) const;
void Pause(bool bPause);
void RegisterFallbackIOThread(EStreamSourceMediaType mediaType, CStreamingIOThread* pIOThread);
CStreamEngineWakeEvent& GetWakeEvent() { return m_awakeEvent; }
//////////////////////////////////////////////////////////////////////////
// CrySimpleThread
//////////////////////////////////////////////////////////////////////////
virtual void Run();
virtual void Cancel();
//////////////////////////////////////////////////////////////////////////
protected:
void ProcessNewRequests();
void ProcessReset();
public:
#ifdef STREAMENGINE_ENABLE_STATS
struct SStats
{
SStats()
: m_nTotalReadBytes(0)
, m_nCurrentReadBandwith(0)
, m_nReadBytesInLastSecond(0)
, m_fReadingDuringLastSecond(.0f)
, m_nTempBytesRead(0)
, m_nActualReadBandwith(0)
, m_nTempReadOffset(0)
, m_nTotalReadOffset(0)
, m_nReadOffsetInLastSecond(0)
, m_nTempRequestCount(0)
, m_nTotalRequestCount(0)
, m_nRequestCountInLastSecond(0)
{}
void Update(const CTimeValue& deltaT);
void Reset()
{
m_nTotalReadBytes = 0;
m_nTotalReadOffset = 0;
m_nTotalRequestCount = 0;
m_TotalReadTime.SetValue(0);
}
float m_fReadingDuringLastSecond;
CTimeValue m_TotalReadTime;
uint64 m_nTotalReadBytes;
uint64 m_nTotalReadOffset;
uint32 m_nTotalRequestCount;
uint32 m_nCurrentReadBandwith; // Read bandwidth over one second
uint32 m_nActualReadBandwith; // Actual read bandwidth extrapolated over one second
uint32 m_nReadBytesInLastSecond;
uint32 m_nRequestCountInLastSecond;
uint64 m_nReadOffsetInLastSecond;
uint32 m_nTempRequestCount;
uint64 m_nTempBytesRead;
uint64 m_nTempReadOffset;
CTimeValue m_TempReadTime;
};
SStats m_InMemoryStats;
SStats m_NotInMemoryStats;
#endif
int64 m_nLastReadDiskOffset;
int m_nStreamingCPU;
private:
CStreamEngine* m_pStreamEngine;
std::vector<CAsyncIOFileRequest*> m_fileRequestQueue;
std::vector<CAsyncIOFileRequest*> m_temporaryArray;
CryMT::vector<CAsyncIOFileRequest*> m_newFileRequests;
EStreamSourceMediaType m_eMediaType;
uint32 m_nFallbackMTs;
typedef std::pair<CStreamingIOThread*, EStreamSourceMediaType> TFallbackIOPair;
typedef std::vector<TFallbackIOPair> TFallbackIOVec;
typedef TFallbackIOVec::iterator TFallbackIOVecConstIt;
TFallbackIOVec m_FallbackIOThreads;
volatile bool m_bCancelThreadRequest;
volatile bool m_bNeedSorting;
volatile bool m_bNewRequests;
volatile bool m_bPaused;
volatile bool m_bNeedReset;
volatile bool m_bAbortReads;
volatile int m_iUrgentRequests;
CStreamEngineWakeEvent m_awakeEvent;
CryEvent m_resetDoneEvent;
string m_name;
uint32 m_nReadCounter;
};
//////////////////////////////////////////////////////////////////////////
// Thread that performs IO operations.
//////////////////////////////////////////////////////////////////////////
class CStreamingWorkerThread
: public CrySimpleThread<CStreamingIOThread>
, public CMultiThreadRefCount
{
public:
enum EWorkerType
{
eWorkerAsyncCallback,
};
CStreamingWorkerThread(CStreamEngine* pStreamEngine, const char* name, EWorkerType type, SStreamRequestQueue* pQueue);
~CStreamingWorkerThread();
void BeginReset();
void EndReset();
void CancelAll();
//////////////////////////////////////////////////////////////////////////
// CrySimpleThread
//////////////////////////////////////////////////////////////////////////
virtual void Run();
virtual void Cancel();
//////////////////////////////////////////////////////////////////////////
private:
EWorkerType m_type;
CStreamEngine* m_pStreamEngine;
SStreamRequestQueue* m_pQueue;
volatile bool m_bCancelThreadRequest;
volatile bool m_bNeedsReset;
CryEvent m_resetDoneEvent;
string m_name;
};
#endif // CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMIOTHREAD_H
Code/CryEngine/CrySystem/StreamEngine/StreamReadStream.cpp
+517
View File
@@ -0,0 +1,517 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Engine
#include "CrySystem_precompiled.h"
#include <ISystem.h>
#include <ILog.h>
#include "StreamReadStream.h"
#include "StreamEngine.h"
#include "MTSafeAllocator.h"
extern CMTSafeHeap* g_pPakHeap;
SLockFreeSingleLinkedListHeader CReadStream::s_freeRequests;
CReadStream* CReadStream::Allocate(CStreamEngine* pEngine, const EStreamTaskType tSource, const char* szFilename, IStreamCallback* pCallback, const StreamReadParams* pParams)
{
char* pFree = reinterpret_cast<char*>(CryInterlockedPopEntrySList(s_freeRequests));
CReadStream* pReq;
IF_LIKELY (pFree)
{
AZ_PUSH_DISABLE_WARNING(,"-Winvalid-offsetof")
ptrdiff_t offs = offsetof(CReadStream, m_nextFree);
AZ_POP_DISABLE_WARNING
pReq = reinterpret_cast<CReadStream*>(pFree - offs);
}
else
{
pReq = new CReadStream;
}
pReq->m_pEngine = pEngine;
pReq->m_Type = tSource;
pReq->m_strFileName = szFilename;
pReq->m_pCallback = pCallback;
if (pParams)
{
pReq->m_Params = *pParams;
}
pReq->m_pBuffer = pReq->m_Params.pBuffer;
#ifdef STREAMENGINE_ENABLE_STATS
pReq->m_requestTime = gEnv->pTimer->GetAsyncTime();
#endif
return pReq;
}
void CReadStream::Flush()
{
AZ_PUSH_DISABLE_WARNING(, "-Winvalid-offsetof")
ptrdiff_t offs = offsetof(CReadStream, m_nextFree);
AZ_POP_DISABLE_WARNING
for (char* pFree = reinterpret_cast<char*>(CryInterlockedPopEntrySList(s_freeRequests));
pFree;
pFree = reinterpret_cast<char*>(CryInterlockedPopEntrySList(s_freeRequests)))
{
CReadStream* pReq = reinterpret_cast<CReadStream*>(pFree - offs);
delete pReq;
}
}
//////////////////////////////////////////////////////////////////////////
CReadStream::CReadStream()
{
Reset();
}
//////////////////////////////////////////////////////////////////////////
CReadStream::~CReadStream()
{
}
// returns true if the file read was completed (successfully or unsuccessfully)
// check IsError to check if the whole requested file (piece) was read
bool CReadStream::IsFinished()
{
return m_bFinished;
}
// returns the number of bytes read so far (the whole buffer size if IsFinished())
unsigned int CReadStream::GetBytesRead ([[maybe_unused]] bool bWait)
{
if (!m_bError)
{
return m_Params.nSize;
}
return 0;
}
// returns the buffer into which the data has been or will be read
// at least GetBytesRead() bytes in this buffer are guaranteed to be already read
const void* CReadStream::GetBuffer ()
{
return m_pBuffer;
}
void CReadStream::AbortShutdown()
{
{
CryAutoCriticalSection lock(m_callbackLock);
m_bError = true;
m_nIOError = ERROR_ABORTED_ON_SHUTDOWN;
m_bFileRequestComplete = true;
if (m_pFileRequest)
{
__debugbreak();
}
}
// lock this object to avoid preliminary destruction
CReadStream_AutoPtr pLock(this);
{
CryAutoCriticalSection lock(m_callbackLock);
// all the callbacks have to handle error cases and needs to be called anyway, even if the stream I/O is aborted
ExecuteAsyncCallback_CBLocked();
ExecuteSyncCallback_CBLocked();
m_pCallback = NULL;
}
}
// tries to stop reading the stream; this is advisory and may have no effect
// all the callbacks will be called after this. If you just destructing object,
// dereference this object and it will automatically abort and release all associated resources.
void CReadStream::Abort()
{
{
CryAutoCriticalSection lock(m_callbackLock);
m_bError = true;
m_nIOError = ERROR_USER_ABORT;
m_bFileRequestComplete = true;
if (m_pFileRequest)
{
m_pFileRequest->Cancel();
m_pFileRequest = 0;
}
}
// lock this object to avoid preliminary destruction
CReadStream_AutoPtr pLock(this);
{
CryAutoCriticalSection lock(m_callbackLock);
// all the callbacks have to handle error cases and needs to be called anyway, even if the stream I/O is aborted
ExecuteAsyncCallback_CBLocked();
ExecuteSyncCallback_CBLocked();
m_pCallback = NULL;
}
m_pEngine->AbortJob(this);
}
bool CReadStream::TryAbort()
{
if (!m_callbackLock.TryLock())
{
return false;
}
if (m_pFileRequest && !m_pFileRequest->TryCancel())
{
m_callbackLock.Unlock();
return false;
}
m_bError = true;
m_nIOError = ERROR_USER_ABORT;
m_bFileRequestComplete = true;
m_pFileRequest = 0;
// lock this object to avoid preliminary destruction
CReadStream_AutoPtr pLock(this);
// all the callbacks have to handle error cases and needs to be called anyway, even if the stream I/O is aborted
ExecuteAsyncCallback_CBLocked();
ExecuteSyncCallback_CBLocked();
m_pCallback = NULL;
m_callbackLock.Unlock();
m_pEngine->AbortJob(this);
return true;
}
// tries to raise the priority of the read; this is advisory and may have no effect
void CReadStream::SetPriority (EStreamTaskPriority ePriority)
{
if (m_Params.ePriority != ePriority)
{
m_Params.ePriority = ePriority;
if (m_pFileRequest && m_pFileRequest->m_status == CAsyncIOFileRequest::eStatusInFileQueue)
{
m_pEngine->UpdateJobPriority(this);
}
}
}
// unconditionally waits until the callback is called
// i.e. if the stream hasn't yet finish, it's guaranteed that the user-supplied callback
// is called before return from this function (unless no callback was specified)
void CReadStream::Wait(int nMaxWaitMillis)
{
// lock this object to avoid preliminary destruction
CReadStream_AutoPtr pLock(this);
bool bNeedFinalize = (m_Params.nFlags & IStreamEngine::FLAGS_NO_SYNC_CALLBACK) == 0;
if (!m_bFinished && !m_bError && !m_pFileRequest)
{
assert(m_pFileRequest != NULL); // If we want to Wait for stream its file request must not be NULL.
// This will almost certainly cause Dead-Lock
CryFatalError("Waiting for stream when StreamingEngine is paused");
}
CTimeValue t0;
if (nMaxWaitMillis > 0)
{
t0 = gEnv->pTimer->GetAsyncTime();
}
while (!m_bFinished && !m_bError)
{
if (bNeedFinalize)
{
m_pEngine->MainThread_FinalizeIOJobs();
}
if (!m_bFileRequestComplete)
{
CrySleep(5);
}
if (nMaxWaitMillis > 0)
{
CTimeValue t1 = gEnv->pTimer->GetAsyncTime();
if (CTimeValue(t1 - t0).GetMilliSeconds() > nMaxWaitMillis)
{
// Break if we are waiting for too long.
break;
}
}
}
}
//////////////////////////////////////////////////////////////////////////
uint64 CReadStream::GetPriority() const
{
return 0;
}
// this gets called upon the IO has been executed to call the callbacks
void CReadStream::MainThread_Finalize()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
// call asynchronous callback function if needed synchronously
{
CryAutoCriticalSection lock(m_callbackLock);
ExecuteSyncCallback_CBLocked();
}
m_pFileRequest = 0;
}
IStreamCallback* CReadStream::GetCallback() const
{
return m_pCallback;
}
unsigned CReadStream::GetError() const
{
return m_nIOError;
}
const char* CReadStream::GetErrorName() const
{
switch (m_nIOError)
{
case ERROR_UNKNOWN_ERROR:
return "Unknown error";
case ERROR_UNEXPECTED_DESTRUCTION:
return "Unexpected destruction";
case ERROR_INVALID_CALL:
return "Invalid call";
case ERROR_CANT_OPEN_FILE:
return "Cannot open the file";
case ERROR_REFSTREAM_ERROR:
return "Refstream error";
case ERROR_OFFSET_OUT_OF_RANGE:
return "Offset out of range";
case ERROR_REGION_OUT_OF_RANGE:
return "Region out of range";
case ERROR_SIZE_OUT_OF_RANGE:
return "Size out of range";
case ERROR_CANT_START_READING:
return "Cannot start reading";
case ERROR_OUT_OF_MEMORY:
return "Out of memory";
case ERROR_ABORTED_ON_SHUTDOWN:
return "Aborted on shutdown";
case ERROR_OUT_OF_MEMORY_QUOTA:
return "Out of memory quota";
case ERROR_ZIP_CACHE_FAILURE:
return "ZIP cache failure";
case ERROR_USER_ABORT:
return "User aborted";
}
return "Unrecognized error";
}
int CReadStream::AddRef()
{
return CryInterlockedIncrement(&m_nRefCount);
}
int CReadStream::Release()
{
int nRef = CryInterlockedDecrement(&m_nRefCount);
#ifndef _RELEASE
if (nRef < 0)
{
__debugbreak();
}
#endif
if (nRef == 0)
{
Reset();
CryInterlockedPushEntrySList(s_freeRequests, m_nextFree);
}
return nRef;
}
void CReadStream::Reset()
{
m_strFileName.clear();
m_pFileRequest = NULL;
m_Params = StreamReadParams();
memset((void*)&m_nRefCount, 0, (char*)(this + 1) - (char*)(&m_nRefCount));
}
void CReadStream::SetUserData(DWORD_PTR dwUserData)
{
m_Params.dwUserData = dwUserData;
}
//////////////////////////////////////////////////////////////////////////
void CReadStream::ExecuteAsyncCallback_CBLocked()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
if (!m_bIsAsyncCallbackExecuted && m_pCallback)
{
m_bIsAsyncCallbackExecuted = true;
m_pCallback->StreamAsyncOnComplete(this, m_nIOError);
}
}
void CReadStream::ExecuteSyncCallback_CBLocked()
{
FUNCTION_PROFILER(gEnv->pSystem, PROFILE_SYSTEM);
if (!m_bIsSyncCallbackExecuted && m_pCallback && (0 == (m_Params.nFlags & IStreamEngine::FLAGS_NO_SYNC_CALLBACK)))
{
m_bIsSyncCallbackExecuted = true;
CReadStream_AutoPtr protectMe(this); // Stream can be freed inside the callback!
m_pCallback->StreamOnComplete(this, m_nIOError);
// We do not need FileRequest here anymore, and not its temporary memory.
m_pFileRequest = 0;
m_pBuffer = NULL;
m_bFinished = true;
}
else
{
m_pFileRequest = 0;
m_pBuffer = NULL;
m_bFinished = true;
}
#ifdef STREAMENGINE_ENABLE_LISTENER
IStreamEngineListener* pListener = m_pEngine->GetListener();
if (pListener)
{
pListener->OnStreamDone(this);
}
#endif
}
void* CReadStream::operator new (size_t sz)
{
return CryModuleMemalign(sz, alignof(CReadStream));
}
void CReadStream::operator delete(void* p)
{
CryModuleMemalignFree(p);
}
//////////////////////////////////////////////////////////////////////////
void CReadStream::FreeTemporaryMemory()
{
// Free temporary block.
if (m_pFileRequest)
{
m_pFileRequest->SyncWithDecompress();
m_pFileRequest->SyncWithDecrypt();
m_pFileRequest->FreeBuffer();
}
m_pBuffer = 0;
}
//////////////////////////////////////////////////////////////////////////
bool CReadStream::IsReqReading()
{
if (m_strFileName.empty())
{
return false;
}
return true;
}
//////////////////////////////////////////////////////////////////////////
CAsyncIOFileRequest* CReadStream::CreateFileRequest()
{
m_pFileRequest = CAsyncIOFileRequest::Allocate(m_Type);
m_pFileRequest->m_nRequestedSize = m_Params.nSize;
m_pFileRequest->m_nRequestedOffset = m_Params.nOffset;
m_pFileRequest->m_pExternalMemoryBuffer = m_pBuffer;
m_pFileRequest->m_bWriteOnlyExternal = (m_Params.nFlags & IStreamEngine::FLAGS_WRITE_ONLY_EXTERNAL_BUFFER) != 0;
m_pFileRequest->m_pReadStream = this;
m_pFileRequest->m_strFileName = m_strFileName;
m_pFileRequest->m_ePriority = m_Params.ePriority;
m_pFileRequest->m_eMediaType = m_Params.eMediaType;
m_bFileRequestComplete = false;
return m_pFileRequest;
}
void* CReadStream::OnNeedStorage(size_t size, bool& bAbortOnFailToAlloc)
{
CryAutoCriticalSection lock(m_callbackLock);
if (m_pCallback)
{
return m_pCallback->StreamOnNeedStorage(this, size, bAbortOnFailToAlloc);
}
return NULL;
}
//////////////////////////////////////////////////////////////////////////
void CReadStream::OnAsyncFileRequestComplete()
{
CryAutoCriticalSection lock(m_callbackLock);
if (!m_bFileRequestComplete)
{
if (m_pFileRequest)
{
m_Params.nSize = m_pFileRequest->m_nRequestedSize;
m_pBuffer = m_pFileRequest->m_pOutputMemoryBuffer;
m_nBytesRead = m_pFileRequest->m_nSizeOnMedia;
m_nIOError = m_pFileRequest->m_nError;
m_bError = m_nIOError != 0;
if (m_bError)
{
m_nBytesRead = 0;
}
#ifdef STREAMENGINE_ENABLE_STATS
m_ReadTime = m_pFileRequest->m_readTime;
#endif
}
ExecuteAsyncCallback_CBLocked();
if (m_Params.nFlags & IStreamEngine::FLAGS_NO_SYNC_CALLBACK)
{
// We do not need FileRequest here anymore, and not its temporary memory.
m_pFileRequest = 0;
m_bFinished = true;
}
m_bFileRequestComplete = true;
}
}
Code/CryEngine/CrySystem/StreamEngine/StreamReadStream.h
+178
View File
@@ -0,0 +1,178 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// Description : Streaming Engine
#ifndef CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMREADSTREAM_H
#define CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMREADSTREAM_H
#pragma once
#include "IStreamEngine.h"
#include "StreamAsyncFileRequest.h"
class CStreamEngine;
class CReadStream
: public IReadStream
{
friend class CStreamEngine;
public:
static CReadStream* Allocate(CStreamEngine* pEngine, const EStreamTaskType tSource, const char* szFilename, IStreamCallback* pCallback, const StreamReadParams* pParams);
static void Flush();
public:
CReadStream();
virtual ~CReadStream ();
virtual int AddRef();
virtual int Release();
virtual DWORD_PTR GetUserData() {return m_Params.dwUserData; }
// set user defined data into stream's params
virtual void SetUserData(DWORD_PTR dwUserData);
// returns true if the file read was not successful.
virtual bool IsError() { return m_bError; };
// returns true if the file read was completed (successfully or unsuccessfully)
// check IsError to check if the whole requested file (piece) was read
virtual bool IsFinished();
// returns the number of bytes read so far (the whole buffer size if IsFinished())
virtual unsigned int GetBytesRead (bool bWait);
// returns the buffer into which the data has been or will be read
// at least GetBytesRead() bytes in this buffer are guaranteed to be already read
virtual const void* GetBuffer ();
void AbortShutdown();
// tries to stop reading the stream; this is advisory and may have no effect
// but the callback will not be called after this. If you just destructing object,
// dereference this object and it will automatically abort and release all associated resources.
virtual void Abort();
virtual bool TryAbort();
// tries to raise the priority of the read; this is advisory and may have no effect
virtual void SetPriority (EStreamTaskPriority EPriority);
// unconditionally waits until the callback is called
// i.e. if the stream hasn't yet finish, it's guaranteed that the user-supplied callback
// is called before return from this function (unless no callback was specified)
virtual void Wait(int nMaxWaitMillis = -1);
virtual uint64 GetPriority() const;
virtual const StreamReadParams& GetParams() const {return m_Params; }
virtual const EStreamTaskType GetCallerType() const { return m_Type; }
virtual EStreamSourceMediaType GetMediaType() const { return m_MediaType; }
// return pointer to callback routine(can be NULL)
virtual IStreamCallback* GetCallback() const;
// return IO error #
virtual unsigned GetError() const;
// Returns IO error name
virtual const char* GetErrorName() const;
// return stream name
virtual const char* GetName() const { return m_strFileName.c_str(); };
virtual void FreeTemporaryMemory();
// this gets called upon the IO has been executed to call the callbacks
void MainThread_Finalize();
bool IsReqReading();
#ifdef STREAMENGINE_ENABLE_STATS
void SetRequestTime(CTimeValue& time) { m_requestTime = time; }
const CTimeValue& GetRequestTime() { return m_requestTime; }
#endif
// decompression of zip-compressed files with default behavior
CAsyncIOFileRequest* CreateFileRequest();
void ComputedMediaType(EStreamSourceMediaType eMT) { m_MediaType = eMT; }
void* OnNeedStorage(size_t size, bool& bAbortOnFailToAlloc);
void OnAsyncFileRequestComplete();
CAsyncIOFileRequest* GetFileRequest() { return m_pFileRequest; }
private:
void Reset();
// call the async callback
void ExecuteAsyncCallback_CBLocked();
// call the sync callback
void ExecuteSyncCallback_CBLocked();
private:
void* operator new (size_t sz);
void operator delete(void* p);
private:
static SLockFreeSingleLinkedListHeader s_freeRequests;
private:
STREAMENGINE_LL_ALIGN SLockFreeSingleLinkedListEntry m_nextFree;
CryStringLocal m_strFileName;
CryCriticalSection m_callbackLock;
CAsyncIOFileRequest_AutoPtr m_pFileRequest;
StreamReadParams m_Params;
// Only POD types must exist below here. They will be memset!
volatile int m_nRefCount;
CStreamEngine* m_pEngine;
// the type of the task
EStreamTaskType m_Type;
EStreamSourceMediaType m_MediaType;
// the initial data from the user
// the callback; may be NULL
IStreamCallback* m_pCallback;
// Bytes actually read from media.
uint32 m_nBytesRead;
volatile bool m_bIsAsyncCallbackExecuted;
volatile bool m_bIsSyncCallbackExecuted;
volatile bool m_bFileRequestComplete;
// the actual buffer to read to
void* m_pBuffer;
volatile bool m_bError;
volatile bool m_bFinished;
unsigned int m_nIOError;
#ifdef STREAMENGINE_ENABLE_STATS
// time when request was made
CTimeValue m_requestTime;
// Time for actual reading
CTimeValue m_ReadTime;
#endif
};
TYPEDEF_AUTOPTR(CReadStream);
#endif // CRYINCLUDE_CRYSYSTEM_STREAMENGINE_STREAMREADSTREAM_H