diff --git a/Code/Framework/AzCore/AzCore/Memory/AllocationRecords.cpp b/Code/Framework/AzCore/AzCore/Memory/AllocationRecords.cpp index 680d93501a..2b2f752b06 100644 --- a/Code/Framework/AzCore/AzCore/Memory/AllocationRecords.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/AllocationRecords.cpp @@ -16,438 +16,421 @@ #include -using namespace AZ; -using namespace AZ::Debug; +namespace AZ::Debug +{ + // Many PC tools break with alloc/free size mismatches when the memory guard is enabled. Disable for now + //#define ENABLE_MEMORY_GUARD -// Many PC tools break with alloc/free size mismatches when the memory guard is enabled. Disable for now -//#define ENABLE_MEMORY_GUARD - -//========================================================================= -// AllocationRecords -// [9/16/2009] -//========================================================================= -AllocationRecords::AllocationRecords(unsigned char stackRecordLevels, [[maybe_unused]] bool isMemoryGuard, bool isMarkUnallocatedMemory, const char* allocatorName) - : m_mode(AllocatorManager::Instance().m_defaultTrackingRecordMode) - , m_isAutoIntegrityCheck(false) - , m_isMarkUnallocatedMemory(isMarkUnallocatedMemory) - , m_saveNames(false) - , m_decodeImmediately(false) - , m_numStackLevels(stackRecordLevels) + //========================================================================= + // AllocationRecords + // [9/16/2009] + //========================================================================= + AllocationRecords::AllocationRecords( + unsigned char stackRecordLevels, [[maybe_unused]] bool isMemoryGuard, bool isMarkUnallocatedMemory, const char* allocatorName) + : m_mode(AllocatorManager::Instance().m_defaultTrackingRecordMode) + , m_isAutoIntegrityCheck(false) + , m_isMarkUnallocatedMemory(isMarkUnallocatedMemory) + , m_saveNames(false) + , m_decodeImmediately(false) + , m_numStackLevels(stackRecordLevels) #if defined(ENABLE_MEMORY_GUARD) - , m_memoryGuardSize(isMemoryGuard ? sizeof(Debug::GuardValue) : 0) + , m_memoryGuardSize(isMemoryGuard ? sizeof(Debug::GuardValue) : 0) #else - , m_memoryGuardSize(0) + , m_memoryGuardSize(0) #endif - , m_requestedAllocs(0) - , m_requestedBytes(0) - , m_requestedBytesPeak(0) - , m_allocatorName(allocatorName) -{ - -} - -//========================================================================= -// ~AllocationRecords -// [9/16/2009] -//========================================================================= -AllocationRecords::~AllocationRecords() -{ - if (!AllocatorManager::Instance().m_isAllocatorLeaking) + , m_requestedAllocs(0) + , m_requestedBytes(0) + , m_requestedBytesPeak(0) + , m_allocatorName(allocatorName) { - // dump all allocation (we should not have any at this point). - bool includeNameAndFilename = (m_saveNames || m_mode == RECORD_FULL); - EnumerateAllocations(PrintAllocationsCB(true, includeNameAndFilename)); - AZ_Error("Memory", m_records.empty(), "We still have %d allocations on record! They must be freed prior to destroy!", m_records.size()); - } -} - -//========================================================================= -// lock -// [9/16/2009] -//========================================================================= -void -AllocationRecords::lock() -{ - m_recordsMutex.lock(); -} - -//========================================================================= -// try_lock -// [9/16/2009] -//========================================================================= -bool AllocationRecords::try_lock() -{ - return m_recordsMutex.try_lock(); -} - -//========================================================================= -// unlock -// [9/16/2009] -//========================================================================= -void -AllocationRecords::unlock() -{ - m_recordsMutex.unlock(); -} - -//========================================================================= -// RegisterAllocation -// [9/11/2009] -//========================================================================= -const AllocationInfo* -AllocationRecords::RegisterAllocation(void* address, size_t byteSize, size_t alignment, const char* name, const char* fileName, int lineNum, unsigned int stackSuppressCount) -{ - (void)stackSuppressCount; - if (m_mode == RECORD_NO_RECORDS) - { - return nullptr; - } - if (address == nullptr) - { - return nullptr; } - // memory guard - if (m_memoryGuardSize == sizeof(Debug::GuardValue)) + //========================================================================= + // ~AllocationRecords + // [9/16/2009] + //========================================================================= + AllocationRecords::~AllocationRecords() { - if (m_isAutoIntegrityCheck) + if (!AllocatorManager::Instance().m_isAllocatorLeaking) { - IntegrityCheck(); + // dump all allocation (we should not have any at this point). + bool includeNameAndFilename = (m_saveNames || m_mode == RECORD_FULL); + EnumerateAllocations(PrintAllocationsCB(true, includeNameAndFilename)); + AZ_Error( + "Memory", m_records.empty(), "We still have %d allocations on record! They must be freed prior to destroy!", + m_records.size()); + } + } + + //========================================================================= + // lock + // [9/16/2009] + //========================================================================= + void AllocationRecords::lock() + { + m_recordsMutex.lock(); + } + + //========================================================================= + // try_lock + // [9/16/2009] + //========================================================================= + bool AllocationRecords::try_lock() + { + return m_recordsMutex.try_lock(); + } + + //========================================================================= + // unlock + // [9/16/2009] + //========================================================================= + void AllocationRecords::unlock() + { + m_recordsMutex.unlock(); + } + + //========================================================================= + // RegisterAllocation + // [9/11/2009] + //========================================================================= + const AllocationInfo* AllocationRecords::RegisterAllocation( + void* address, + size_t byteSize, + size_t alignment, + const char* name, + const char* fileName, + int lineNum, + unsigned int stackSuppressCount) + { + (void)stackSuppressCount; + if (m_mode == RECORD_NO_RECORDS) + { + return nullptr; + } + if (address == nullptr) + { + return nullptr; } - AZ_Assert(byteSize>sizeof(Debug::GuardValue), "Did you forget to add the extra MemoryGuardSize() bytes?"); - byteSize -= sizeof(Debug::GuardValue); - new(reinterpret_cast(address)+byteSize) Debug::GuardValue(); - } - - Debug::AllocationRecordsType::pair_iter_bool iterBool; - { - AZStd::scoped_lock lock(m_recordsMutex); - iterBool = m_records.insert_key(address); - } - - if (!iterBool.second) - { - // If that memory address was already registered, print the stack trace of the previous registration - PrintAllocationsCB(true, (m_saveNames || m_mode == RECORD_FULL))(address, iterBool.first->second, m_numStackLevels); - AZ_Assert(iterBool.second, "Memory address 0x%p is already allocated and in the records!", address); - } - - Debug::AllocationInfo& ai = iterBool.first->second; - ai.m_byteSize = byteSize; - ai.m_alignment = static_cast(alignment); - if ((m_saveNames || m_mode == RECORD_FULL) && name && fileName) - { - // In RECORD_FULL mode or when specifically enabled in app descriptor with - // m_allocationRecordsSaveNames, we allocate our own memory to save off name and fileName. - // When testing for memory leaks, on process shutdown AllocationRecords::~AllocationRecords - // gets called to enumerate the remaining (leaked) allocations. Unfortunately, any names - // referenced in dynamic module memory whose modules are unloaded won't be valid - // references anymore and we won't get useful information from the enumeration print. - // This code block ensures we keep our name/fileName valid for when we need it. - const size_t nameLength = strlen(name); - const size_t fileNameLength = strlen(fileName); - const size_t totalLength = nameLength + fileNameLength + 2; // + 2 for terminating null characters - ai.m_namesBlock = m_records.get_allocator().allocate(totalLength, 1); - ai.m_namesBlockSize = totalLength; - char* savedName = reinterpret_cast(ai.m_namesBlock); - char* savedFileName = savedName + nameLength + 1; - memcpy(reinterpret_cast(savedName), reinterpret_cast(name), nameLength + 1); - memcpy(reinterpret_cast(savedFileName), reinterpret_cast(fileName), fileNameLength + 1); - ai.m_name = savedName; - ai.m_fileName = savedFileName; - } - else - { - ai.m_name = name; - ai.m_fileName = fileName; - ai.m_namesBlock = nullptr; - ai.m_namesBlockSize = 0; - } - ai.m_lineNum = lineNum; - ai.m_timeStamp = AZStd::GetTimeNowMicroSecond(); - - // if we don't have a fileName,lineNum record the stack or if the user requested it. - if ((fileName == nullptr && m_mode == RECORD_STACK_IF_NO_FILE_LINE) || m_mode == RECORD_FULL) - { - ai.m_stackFrames = m_numStackLevels ? reinterpret_cast(m_records.get_allocator().allocate(sizeof(AZ::Debug::StackFrame)*m_numStackLevels, 1)) : nullptr; - if (ai.m_stackFrames) + // memory guard + if (m_memoryGuardSize == sizeof(Debug::GuardValue)) { - Debug::StackRecorder::Record(ai.m_stackFrames, m_numStackLevels, stackSuppressCount + 1); - - if (m_decodeImmediately) + if (m_isAutoIntegrityCheck) { - // OPTIONAL DEBUGGING CODE - enable in app descriptor m_allocationRecordsAttemptDecodeImmediately - // This is optionally-enabled code for tracking down memory allocations - // that fail to be decoded. DecodeFrames() typically runs at the end of - // your application when leaks were found. Sometimes you have stack prints - // full of "(module-name not available)" and "(function-name not available)" - // that are not actionable. If you have those, enable this code. It'll slow - // down your process significantly because for every allocation recorded - // we get the stack trace on the spot. Put a breakpoint in DecodeFrames() - // at the "(module-name not available)" and "(function-name not available)" - // locations and now at the moment those allocations happen you'll have the - // full stack trace available and the ability to debug what could be causing it + IntegrityCheck(); + } + + AZ_Assert(byteSize > sizeof(Debug::GuardValue), "Did you forget to add the extra MemoryGuardSize() bytes?"); + byteSize -= sizeof(Debug::GuardValue); + new (reinterpret_cast(address) + byteSize) Debug::GuardValue(); + } + + Debug::AllocationRecordsType::pair_iter_bool iterBool; + { + AZStd::scoped_lock lock(m_recordsMutex); + iterBool = m_records.insert_key(address); + } + + if (!iterBool.second) + { + // If that memory address was already registered, print the stack trace of the previous registration + PrintAllocationsCB(true, (m_saveNames || m_mode == RECORD_FULL))(address, iterBool.first->second, m_numStackLevels); + AZ_Assert(iterBool.second, "Memory address 0x%p is already allocated and in the records!", address); + } + + Debug::AllocationInfo& ai = iterBool.first->second; + ai.m_byteSize = byteSize; + ai.m_alignment = static_cast(alignment); + if ((m_saveNames || m_mode == RECORD_FULL) && name && fileName) + { + // In RECORD_FULL mode or when specifically enabled in app descriptor with + // m_allocationRecordsSaveNames, we allocate our own memory to save off name and fileName. + // When testing for memory leaks, on process shutdown AllocationRecords::~AllocationRecords + // gets called to enumerate the remaining (leaked) allocations. Unfortunately, any names + // referenced in dynamic module memory whose modules are unloaded won't be valid + // references anymore and we won't get useful information from the enumeration print. + // This code block ensures we keep our name/fileName valid for when we need it. + const size_t nameLength = strlen(name); + const size_t fileNameLength = strlen(fileName); + const size_t totalLength = nameLength + fileNameLength + 2; // + 2 for terminating null characters + ai.m_namesBlock = m_records.get_allocator().allocate(totalLength, 1); + ai.m_namesBlockSize = totalLength; + char* savedName = reinterpret_cast(ai.m_namesBlock); + char* savedFileName = savedName + nameLength + 1; + memcpy(reinterpret_cast(savedName), reinterpret_cast(name), nameLength + 1); + memcpy(reinterpret_cast(savedFileName), reinterpret_cast(fileName), fileNameLength + 1); + ai.m_name = savedName; + ai.m_fileName = savedFileName; + } + else + { + ai.m_name = name; + ai.m_fileName = fileName; + ai.m_namesBlock = nullptr; + ai.m_namesBlockSize = 0; + } + ai.m_lineNum = lineNum; + ai.m_timeStamp = AZStd::GetTimeNowMicroSecond(); + + // if we don't have a fileName,lineNum record the stack or if the user requested it. + if ((fileName == nullptr && m_mode == RECORD_STACK_IF_NO_FILE_LINE) || m_mode == RECORD_FULL) + { + ai.m_stackFrames = m_numStackLevels ? reinterpret_cast(m_records.get_allocator().allocate( + sizeof(AZ::Debug::StackFrame) * m_numStackLevels, 1)) + : nullptr; + if (ai.m_stackFrames) + { + Debug::StackRecorder::Record(ai.m_stackFrames, m_numStackLevels, stackSuppressCount + 1); + + if (m_decodeImmediately) { - const unsigned char decodeStep = 40; - Debug::SymbolStorage::StackLine lines[decodeStep]; - unsigned char iFrame = 0; - unsigned char numStackLevels = m_numStackLevels; - while (numStackLevels > 0) + // OPTIONAL DEBUGGING CODE - enable in app descriptor m_allocationRecordsAttemptDecodeImmediately + // This is optionally-enabled code for tracking down memory allocations + // that fail to be decoded. DecodeFrames() typically runs at the end of + // your application when leaks were found. Sometimes you have stack prints + // full of "(module-name not available)" and "(function-name not available)" + // that are not actionable. If you have those, enable this code. It'll slow + // down your process significantly because for every allocation recorded + // we get the stack trace on the spot. Put a breakpoint in DecodeFrames() + // at the "(module-name not available)" and "(function-name not available)" + // locations and now at the moment those allocations happen you'll have the + // full stack trace available and the ability to debug what could be causing it { - unsigned char numToDecode = AZStd::GetMin(decodeStep, numStackLevels); - Debug::SymbolStorage::DecodeFrames(&ai.m_stackFrames[iFrame], numToDecode, lines); - numStackLevels -= numToDecode; - iFrame += numToDecode; + const unsigned char decodeStep = 40; + Debug::SymbolStorage::StackLine lines[decodeStep]; + unsigned char iFrame = 0; + unsigned char numStackLevels = m_numStackLevels; + while (numStackLevels > 0) + { + unsigned char numToDecode = AZStd::GetMin(decodeStep, numStackLevels); + Debug::SymbolStorage::DecodeFrames(&ai.m_stackFrames[iFrame], numToDecode, lines); + numStackLevels -= numToDecode; + iFrame += numToDecode; + } } } } } + + AllocatorManager::Instance().DebugBreak(address, ai); + + // statistics + m_requestedBytes += byteSize; + + size_t currentRequestedBytePeak; + size_t newRequestedBytePeak; + do + { + currentRequestedBytePeak = m_requestedBytesPeak.load(std::memory_order::memory_order_relaxed); + newRequestedBytePeak = AZStd::GetMax(currentRequestedBytePeak, m_requestedBytes.load(std::memory_order::memory_order_relaxed)); + } while (!m_requestedBytesPeak.compare_exchange_weak(currentRequestedBytePeak, newRequestedBytePeak)); + + ++m_requestedAllocs; + + return &ai; } - AllocatorManager::Instance().DebugBreak(address, ai); - - // statistics - m_requestedBytes += byteSize; - - size_t currentRequestedBytePeak; - size_t newRequestedBytePeak; - do + //========================================================================= + // UnregisterAllocation + // [9/11/2009] + //========================================================================= + void AllocationRecords::UnregisterAllocation(void* address, size_t byteSize, size_t alignment, AllocationInfo* info) { - currentRequestedBytePeak = m_requestedBytesPeak.load(std::memory_order::memory_order_relaxed); - newRequestedBytePeak = AZStd::GetMax(currentRequestedBytePeak, m_requestedBytes.load(std::memory_order::memory_order_relaxed)); - } while (!m_requestedBytesPeak.compare_exchange_weak(currentRequestedBytePeak, newRequestedBytePeak)); - - ++m_requestedAllocs; - - return &ai; -} - -//========================================================================= -// UnregisterAllocation -// [9/11/2009] -//========================================================================= -void AllocationRecords::UnregisterAllocation(void* address, size_t byteSize, size_t alignment, AllocationInfo* info) -{ - if (m_mode == RECORD_NO_RECORDS) - { - return; - } - if (address == nullptr) - { - return; - } - - AllocationInfo allocationInfo; - { - AZStd::scoped_lock lock(m_recordsMutex); - Debug::AllocationRecordsType::iterator iter = m_records.find(address); - // We cannot assert if an allocation does not exist because allocations may have been made before tracking was enabled. - // It is currently impossible to actually track all allocations that happen before a certain point - // AZ_Assert(iter!=m_records.end(), "Could not find address 0x%p in the allocator!", address); - if (iter == m_records.end()) + if (m_mode == RECORD_NO_RECORDS) { return; } - allocationInfo = iter->second; - m_records.erase(iter); - - // try to be more aggressive and keep the memory footprint low. - // \todo store the load factor at the last rehash to avoid unnecessary rehash - if (m_records.load_factor() < 0.9f) + if (address == nullptr) { - m_records.rehash(0); + return; } - } - - AllocatorManager::Instance().DebugBreak(address, allocationInfo); - - (void)byteSize; - (void)alignment; - AZ_Assert(byteSize==0||byteSize==allocationInfo.m_byteSize, "Mismatched byteSize at deallocation! You supplied an invalid value!"); - AZ_Assert(alignment==0||alignment==allocationInfo.m_alignment, "Mismatched alignment at deallocation! You supplied an invalid value!"); - - // statistics - m_requestedBytes -= allocationInfo.m_byteSize; - -#if defined(ENABLE_MEMORY_GUARD) - // memory guard - if (m_memoryGuardSize == sizeof(Debug::GuardValue)) - { - if (m_isAutoIntegrityCheck) - { - // full integrity check - IntegrityCheck(); - } - else - { - // check current allocation - char* guardAddress = reinterpret_cast(address)+allocationInfo.m_byteSize; - Debug::GuardValue* guard = reinterpret_cast(guardAddress); - if (!guard->Validate()) - { - AZ_Printf("Memory", "Memory stomp located at address %p, part of allocation:", guardAddress); - PrintAllocationsCB printAlloc(true); - printAlloc(address, allocationInfo, m_numStackLevels); - AZ_Assert(false, "MEMORY STOMP DETECTED!!!"); - } - guard->~GuardValue(); - } - } -#endif - - // delete allocation record - if (allocationInfo.m_namesBlock) - { - m_records.get_allocator().deallocate(allocationInfo.m_namesBlock, allocationInfo.m_namesBlockSize, 1); - allocationInfo.m_namesBlock = nullptr; - allocationInfo.m_namesBlockSize = 0; - allocationInfo.m_name = nullptr; - allocationInfo.m_fileName = nullptr; - } - if (allocationInfo.m_stackFrames) - { - m_records.get_allocator().deallocate(allocationInfo.m_stackFrames, sizeof(AZ::Debug::StackFrame)*m_numStackLevels, 1); - allocationInfo.m_stackFrames = nullptr; - } - - if (info) - { - *info = allocationInfo; - } - - - - // if requested set memory to a specific value. - if (m_isMarkUnallocatedMemory) - { - memset(address, GetUnallocatedMarkValue(), byteSize); - } -} - -//========================================================================= -// ResizeAllocation -// [9/20/2009] -//========================================================================= -void -AllocationRecords::ResizeAllocation(void* address, size_t newSize) -{ - if (m_mode == RECORD_NO_RECORDS) - { - return; - } - - AllocationInfo* allocationInfo; - { - AZStd::scoped_lock lock(m_recordsMutex); - Debug::AllocationRecordsType::iterator iter = m_records.find(address); - AZ_Assert(iter != m_records.end(), "Could not find address 0x%p in the allocator!", address); - allocationInfo = &iter->second; - } - AllocatorManager::Instance().DebugBreak(address, *allocationInfo); - -#if defined(ENABLE_MEMORY_GUARD) - if (m_memoryGuardSize == sizeof(Debug::GuardValue)) - { - if (m_isAutoIntegrityCheck) - { - // full integrity check - IntegrityCheck(); - } - else - { - // check memory guard - char* guardAddress = reinterpret_cast(address) + allocationInfo->m_byteSize; - Debug::GuardValue* guard = reinterpret_cast(guardAddress); - if (!guard->Validate()) - { - AZ_Printf("Memory", "Memory stomp located at address %p, part of allocation:", guardAddress); - PrintAllocationsCB printAlloc(true); - printAlloc(address, iter->second, m_numStackLevels); - AZ_Assert(false, "MEMORY STOMP DETECTED!!!"); - } - guard->~GuardValue(); - } - // init the new memory guard - newSize -= sizeof(Debug::GuardValue); - new(reinterpret_cast(address)+newSize) Debug::GuardValue(); - } -#endif - - // statistics - m_requestedBytes -= allocationInfo->m_byteSize; - m_requestedBytes += newSize; - size_t currentRequestedBytePeak; - size_t newRequestedBytePeak; - do - { - currentRequestedBytePeak = m_requestedBytesPeak.load(std::memory_order::memory_order_relaxed); - newRequestedBytePeak = AZStd::GetMax(currentRequestedBytePeak, m_requestedBytes.load(std::memory_order::memory_order_relaxed)); - } while (!m_requestedBytesPeak.compare_exchange_weak(currentRequestedBytePeak, newRequestedBytePeak)); - ++m_requestedAllocs; - - // update allocation size - allocationInfo->m_byteSize = newSize; -} - -//========================================================================= -// EnumerateAllocations -// [9/29/2009] -//========================================================================= -void -AllocationRecords::SetMode(Mode mode) -{ - if (mode == RECORD_NO_RECORDS) - { + AllocationInfo allocationInfo; { AZStd::scoped_lock lock(m_recordsMutex); - m_records.clear(); + Debug::AllocationRecordsType::iterator iter = m_records.find(address); + // We cannot assert if an allocation does not exist because allocations may have been made before tracking was enabled. + // It is currently impossible to actually track all allocations that happen before a certain point + // AZ_Assert(iter!=m_records.end(), "Could not find address 0x%p in the allocator!", address); + if (iter == m_records.end()) + { + return; + } + allocationInfo = iter->second; + m_records.erase(iter); + + // try to be more aggressive and keep the memory footprint low. + // \todo store the load factor at the last rehash to avoid unnecessary rehash + if (m_records.load_factor() < 0.9f) + { + m_records.rehash(0); + } } - m_requestedBytes = 0; - m_requestedBytesPeak = 0; - m_requestedAllocs = 0; - } - AZ_Warning("Memory", m_mode != RECORD_NO_RECORDS || mode == RECORD_NO_RECORDS, "Records recording was disabled and now it's enabled! You might get assert when you free memory, if a you have allocations which were not recorded!"); + AllocatorManager::Instance().DebugBreak(address, allocationInfo); - m_mode = mode; -} + (void)byteSize; + (void)alignment; + AZ_Assert( + byteSize == 0 || byteSize == allocationInfo.m_byteSize, "Mismatched byteSize at deallocation! You supplied an invalid value!"); + AZ_Assert( + alignment == 0 || alignment == allocationInfo.m_alignment, + "Mismatched alignment at deallocation! You supplied an invalid value!"); -//========================================================================= -// EnumerateAllocations -// [9/29/2009] -//========================================================================= -void -AllocationRecords::EnumerateAllocations(AllocationInfoCBType cb) -{ - // enumerate all allocations and stop if requested. - // Since allocations can change during the iteration (code that prints out the records could allocate, which will - // mutate m_records), we are going to make a copy and iterate the copy. - Debug::AllocationRecordsType recordsCopy; - { - AZStd::scoped_lock lock(m_recordsMutex); - recordsCopy = m_records; - } - for (Debug::AllocationRecordsType::const_iterator iter = recordsCopy.begin(); iter != recordsCopy.end(); ++iter) - { - if (!cb(iter->first, iter->second, m_numStackLevels)) - { - break; - } - } -} + // statistics + m_requestedBytes -= allocationInfo.m_byteSize; -//========================================================================= -// IntegrityCheck -// [9/9/2011] -//========================================================================= -void -AllocationRecords::IntegrityCheck() const -{ #if defined(ENABLE_MEMORY_GUARD) - if (m_memoryGuardSize == sizeof(Debug::GuardValue)) + // memory guard + if (m_memoryGuardSize == sizeof(Debug::GuardValue)) + { + if (m_isAutoIntegrityCheck) + { + // full integrity check + IntegrityCheck(); + } + else + { + // check current allocation + char* guardAddress = reinterpret_cast(address) + allocationInfo.m_byteSize; + Debug::GuardValue* guard = reinterpret_cast(guardAddress); + if (!guard->Validate()) + { + AZ_Printf("Memory", "Memory stomp located at address %p, part of allocation:", guardAddress); + PrintAllocationsCB printAlloc(true); + printAlloc(address, allocationInfo, m_numStackLevels); + AZ_Assert(false, "MEMORY STOMP DETECTED!!!"); + } + guard->~GuardValue(); + } + } +#endif + + // delete allocation record + if (allocationInfo.m_namesBlock) + { + m_records.get_allocator().deallocate(allocationInfo.m_namesBlock, allocationInfo.m_namesBlockSize, 1); + allocationInfo.m_namesBlock = nullptr; + allocationInfo.m_namesBlockSize = 0; + allocationInfo.m_name = nullptr; + allocationInfo.m_fileName = nullptr; + } + if (allocationInfo.m_stackFrames) + { + m_records.get_allocator().deallocate(allocationInfo.m_stackFrames, sizeof(AZ::Debug::StackFrame) * m_numStackLevels, 1); + allocationInfo.m_stackFrames = nullptr; + } + + if (info) + { + *info = allocationInfo; + } + + // if requested set memory to a specific value. + if (m_isMarkUnallocatedMemory) + { + memset(address, GetUnallocatedMarkValue(), byteSize); + } + } + + //========================================================================= + // ResizeAllocation + // [9/20/2009] + //========================================================================= + void AllocationRecords::ResizeAllocation(void* address, size_t newSize) { + if (m_mode == RECORD_NO_RECORDS) + { + return; + } + + AllocationInfo* allocationInfo; + { + AZStd::scoped_lock lock(m_recordsMutex); + Debug::AllocationRecordsType::iterator iter = m_records.find(address); + AZ_Assert(iter != m_records.end(), "Could not find address 0x%p in the allocator!", address); + allocationInfo = &iter->second; + } + AllocatorManager::Instance().DebugBreak(address, *allocationInfo); + +#if defined(ENABLE_MEMORY_GUARD) + if (m_memoryGuardSize == sizeof(Debug::GuardValue)) + { + if (m_isAutoIntegrityCheck) + { + // full integrity check + IntegrityCheck(); + } + else + { + // check memory guard + char* guardAddress = reinterpret_cast(address) + allocationInfo->m_byteSize; + Debug::GuardValue* guard = reinterpret_cast(guardAddress); + if (!guard->Validate()) + { + AZ_Printf("Memory", "Memory stomp located at address %p, part of allocation:", guardAddress); + PrintAllocationsCB printAlloc(true); + printAlloc(address, iter->second, m_numStackLevels); + AZ_Assert(false, "MEMORY STOMP DETECTED!!!"); + } + guard->~GuardValue(); + } + // init the new memory guard + newSize -= sizeof(Debug::GuardValue); + new (reinterpret_cast(address) + newSize) Debug::GuardValue(); + } +#endif + + // statistics + m_requestedBytes -= allocationInfo->m_byteSize; + m_requestedBytes += newSize; + size_t currentRequestedBytePeak; + size_t newRequestedBytePeak; + do + { + currentRequestedBytePeak = m_requestedBytesPeak.load(std::memory_order::memory_order_relaxed); + newRequestedBytePeak = AZStd::GetMax(currentRequestedBytePeak, m_requestedBytes.load(std::memory_order::memory_order_relaxed)); + } while (!m_requestedBytesPeak.compare_exchange_weak(currentRequestedBytePeak, newRequestedBytePeak)); + ++m_requestedAllocs; + + // update allocation size + allocationInfo->m_byteSize = newSize; + } + + //========================================================================= + // EnumerateAllocations + // [9/29/2009] + //========================================================================= + void AllocationRecords::SetMode(Mode mode) + { + if (mode == RECORD_NO_RECORDS) + { + { + AZStd::scoped_lock lock(m_recordsMutex); + m_records.clear(); + } + m_requestedBytes = 0; + m_requestedBytesPeak = 0; + m_requestedAllocs = 0; + } + + AZ_Warning( + "Memory", m_mode != RECORD_NO_RECORDS || mode == RECORD_NO_RECORDS, + "Records recording was disabled and now it's enabled! You might get assert when you free memory, if a you have allocations " + "which were not recorded!"); + + m_mode = mode; + } + + //========================================================================= + // EnumerateAllocations + // [9/29/2009] + //========================================================================= + void AllocationRecords::EnumerateAllocations(AllocationInfoCBType cb) + { + // enumerate all allocations and stop if requested. + // Since allocations can change during the iteration (code that prints out the records could allocate, which will + // mutate m_records), we are going to make a copy and iterate the copy. Debug::AllocationRecordsType recordsCopy; { AZStd::scoped_lock lock(m_recordsMutex); @@ -455,67 +438,93 @@ AllocationRecords::IntegrityCheck() const } for (Debug::AllocationRecordsType::const_iterator iter = recordsCopy.begin(); iter != recordsCopy.end(); ++iter) { - // check memory guard - const char* guardAddress = reinterpret_cast(iter->first)+ iter->second.m_byteSize; - if (!reinterpret_cast(guardAddress)->Validate()) + if (!cb(iter->first, iter->second, m_numStackLevels)) { - // We have to turn off the integrity check at this point if we want to succesfully report the memory - // stomp we just found. If we don't turn this off, the printf just winds off the stack as each memory - // allocation done therein recurses this same code. - *const_cast(&m_isAutoIntegrityCheck) = false; - AZ_Printf("Memory", "Memory stomp located at address %p, part of allocation:", guardAddress); - PrintAllocationsCB printAlloc(true); - printAlloc(iter->first, iter->second, m_numStackLevels); - AZ_Error("Memory", false, "MEMORY STOMP DETECTED!!!"); + break; } } } -#endif -} -//========================================================================= -// operator() -// [9/29/2009] -//========================================================================= -bool -PrintAllocationsCB::operator()(void* address, const AllocationInfo& info, unsigned char numStackLevels) -{ - if (m_includeNameAndFilename && info.m_name) + //========================================================================= + // IntegrityCheck + // [9/9/2011] + //========================================================================= + void AllocationRecords::IntegrityCheck() const { - AZ_Printf("Memory", "Allocation Name: \"%s\" Addr: 0%p Size: %d Alignment: %d\n", info.m_name, address, info.m_byteSize, info.m_alignment); - } - else - { - AZ_Printf("Memory", "Allocation Addr: 0%p Size: %d Alignment: %d\n", address, info.m_byteSize, info.m_alignment); - } - - if (m_isDetailed) - { - if (!info.m_stackFrames) +#if defined(ENABLE_MEMORY_GUARD) + if (m_memoryGuardSize == sizeof(Debug::GuardValue)) { - AZ_Printf("Memory", " %s (%d)\n", info.m_fileName, info.m_lineNum); + Debug::AllocationRecordsType recordsCopy; + { + AZStd::scoped_lock lock(m_recordsMutex); + recordsCopy = m_records; + } + for (Debug::AllocationRecordsType::const_iterator iter = recordsCopy.begin(); iter != recordsCopy.end(); ++iter) + { + // check memory guard + const char* guardAddress = reinterpret_cast(iter->first) + iter->second.m_byteSize; + if (!reinterpret_cast(guardAddress)->Validate()) + { + // We have to turn off the integrity check at this point if we want to succesfully report the memory + // stomp we just found. If we don't turn this off, the printf just winds off the stack as each memory + // allocation done therein recurses this same code. + *const_cast(&m_isAutoIntegrityCheck) = false; + AZ_Printf("Memory", "Memory stomp located at address %p, part of allocation:", guardAddress); + PrintAllocationsCB printAlloc(true); + printAlloc(iter->first, iter->second, m_numStackLevels); + AZ_Error("Memory", false, "MEMORY STOMP DETECTED!!!"); + } + } + } +#endif + } + + //========================================================================= + // operator() + // [9/29/2009] + //========================================================================= + bool PrintAllocationsCB::operator()(void* address, const AllocationInfo& info, unsigned char numStackLevels) + { + if (m_includeNameAndFilename && info.m_name) + { + AZ_Printf( + "Memory", "Allocation Name: \"%s\" Addr: 0%p Size: %d Alignment: %d\n", info.m_name, address, info.m_byteSize, + info.m_alignment); } else { - // Allocation callstack - const unsigned char decodeStep = 40; - Debug::SymbolStorage::StackLine lines[decodeStep]; - unsigned char iFrame = 0; - while (numStackLevels>0) + AZ_Printf("Memory", "Allocation Addr: 0%p Size: %d Alignment: %d\n", address, info.m_byteSize, info.m_alignment); + } + + if (m_isDetailed) + { + if (!info.m_stackFrames) { - unsigned char numToDecode = AZStd::GetMin(decodeStep, numStackLevels); - Debug::SymbolStorage::DecodeFrames(&info.m_stackFrames[iFrame], numToDecode, lines); - for (unsigned char i = 0; i < numToDecode; ++i) + AZ_Printf("Memory", " %s (%d)\n", info.m_fileName, info.m_lineNum); + } + else + { + // Allocation callstack + const unsigned char decodeStep = 40; + Debug::SymbolStorage::StackLine lines[decodeStep]; + unsigned char iFrame = 0; + while (numStackLevels > 0) { - if (info.m_stackFrames[iFrame+i].IsValid()) + unsigned char numToDecode = AZStd::GetMin(decodeStep, numStackLevels); + Debug::SymbolStorage::DecodeFrames(&info.m_stackFrames[iFrame], numToDecode, lines); + for (unsigned char i = 0; i < numToDecode; ++i) { - AZ_Printf("Memory", " %s\n", lines[i]); + if (info.m_stackFrames[iFrame + i].IsValid()) + { + AZ_Printf("Memory", " %s\n", lines[i]); + } } + numStackLevels -= numToDecode; + iFrame += numToDecode; } - numStackLevels -= numToDecode; - iFrame += numToDecode; } } + return true; // continue enumerating } - return true; // continue enumerating -} + +} // namespace AZ::Debug diff --git a/Code/Framework/AzCore/AzCore/Memory/AllocatorBase.cpp b/Code/Framework/AzCore/AzCore/Memory/AllocatorBase.cpp index 61e883bb4e..5c510f67c1 100644 --- a/Code/Framework/AzCore/AzCore/Memory/AllocatorBase.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/AllocatorBase.cpp @@ -6,194 +6,203 @@ * */ -#include #include +#include -using namespace AZ; - -AllocatorBase::AllocatorBase(IAllocatorAllocate* allocationSource, const char* name, const char* desc) : - IAllocator(allocationSource), - m_name(name), - m_desc(desc) +namespace AZ { -} - -AllocatorBase::~AllocatorBase() -{ - AZ_Assert(!m_isReady, "Allocator %s (%s) is being destructed without first having gone through proper calls to PreDestroy() and Destroy(). Use AllocatorInstance<> for global allocators or AllocatorWrapper<> for local allocators.", m_name, m_desc); -} - -const char* AllocatorBase::GetName() const -{ - return m_name; -} - -const char* AllocatorBase::GetDescription() const -{ - return m_desc; -} - -IAllocatorAllocate* AllocatorBase::GetSchema() -{ - return nullptr; -} - -Debug::AllocationRecords* AllocatorBase::GetRecords() -{ - return m_records; -} - -void AllocatorBase::SetRecords(Debug::AllocationRecords* records) -{ - m_records = records; - m_memoryGuardSize = records ? records->MemoryGuardSize() : 0; -} - -bool AllocatorBase::IsReady() const -{ - return m_isReady; -} - -bool AllocatorBase::CanBeOverridden() const -{ - return m_canBeOverridden; -} - -void AllocatorBase::PostCreate() -{ - if (m_registrationEnabled) + AllocatorBase::AllocatorBase(IAllocatorAllocate* allocationSource, const char* name, const char* desc) + : IAllocator(allocationSource) + , m_name(name) + , m_desc(desc) { - if (AZ::Environment::IsReady()) + } + + AllocatorBase::~AllocatorBase() + { + AZ_Assert( + !m_isReady, + "Allocator %s (%s) is being destructed without first having gone through proper calls to PreDestroy() and Destroy(). Use " + "AllocatorInstance<> for global allocators or AllocatorWrapper<> for local allocators.", + m_name, m_desc); + } + + const char* AllocatorBase::GetName() const + { + return m_name; + } + + const char* AllocatorBase::GetDescription() const + { + return m_desc; + } + + IAllocatorAllocate* AllocatorBase::GetSchema() + { + return nullptr; + } + + Debug::AllocationRecords* AllocatorBase::GetRecords() + { + return m_records; + } + + void AllocatorBase::SetRecords(Debug::AllocationRecords* records) + { + m_records = records; + m_memoryGuardSize = records ? records->MemoryGuardSize() : 0; + } + + bool AllocatorBase::IsReady() const + { + return m_isReady; + } + + bool AllocatorBase::CanBeOverridden() const + { + return m_canBeOverridden; + } + + void AllocatorBase::PostCreate() + { + if (m_registrationEnabled) { - AllocatorManager::Instance().RegisterAllocator(this); + if (AZ::Environment::IsReady()) + { + AllocatorManager::Instance().RegisterAllocator(this); + } + else + { + AllocatorManager::PreRegisterAllocator(this); + } } - else + + const auto debugConfig = GetDebugConfig(); + if (!debugConfig.m_excludeFromDebugging) { - AllocatorManager::PreRegisterAllocator(this); + SetRecords(aznew Debug::AllocationRecords( + (unsigned char)debugConfig.m_stackRecordLevels, debugConfig.m_usesMemoryGuards, debugConfig.m_marksUnallocatedMemory, + GetName())); } + + m_isReady = true; } - const auto debugConfig = GetDebugConfig(); - if (!debugConfig.m_excludeFromDebugging) + void AllocatorBase::PreDestroy() { - SetRecords(aznew Debug::AllocationRecords((unsigned char)debugConfig.m_stackRecordLevels, debugConfig.m_usesMemoryGuards, debugConfig.m_marksUnallocatedMemory, GetName())); + Debug::AllocationRecords* allocatorRecords = GetRecords(); + if (allocatorRecords) + { + delete allocatorRecords; + SetRecords(nullptr); + } + + if (m_registrationEnabled && AZ::AllocatorManager::IsReady()) + { + AllocatorManager::Instance().UnRegisterAllocator(this); + } + + m_isReady = false; } - m_isReady = true; -} - -void AllocatorBase::PreDestroy() -{ - Debug::AllocationRecords* allocatorRecords = GetRecords(); - if(allocatorRecords) + void AllocatorBase::SetLazilyCreated(bool lazy) { - delete allocatorRecords; - SetRecords(nullptr); + m_isLazilyCreated = lazy; } - if (m_registrationEnabled && AZ::AllocatorManager::IsReady()) + bool AllocatorBase::IsLazilyCreated() const { - AllocatorManager::Instance().UnRegisterAllocator(this); + return m_isLazilyCreated; } - m_isReady = false; -} + void AllocatorBase::SetProfilingActive(bool active) + { + m_isProfilingActive = active; + } -void AllocatorBase::SetLazilyCreated(bool lazy) -{ - m_isLazilyCreated = lazy; -} + bool AllocatorBase::IsProfilingActive() const + { + return m_isProfilingActive; + } -bool AllocatorBase::IsLazilyCreated() const -{ - return m_isLazilyCreated; -} + void AllocatorBase::DisableOverriding() + { + m_canBeOverridden = false; + } -void AllocatorBase::SetProfilingActive(bool active) -{ - m_isProfilingActive = active; -} + void AllocatorBase::DisableRegistration() + { + m_registrationEnabled = false; + } -bool AllocatorBase::IsProfilingActive() const -{ - return m_isProfilingActive; -} - -void AllocatorBase::DisableOverriding() -{ - m_canBeOverridden = false; -} - -void AllocatorBase::DisableRegistration() -{ - m_registrationEnabled = false; -} - -void AllocatorBase::ProfileAllocation(void* ptr, size_t byteSize, size_t alignment, const char* name, const char* fileName, int lineNum, int suppressStackRecord) -{ + void AllocatorBase::ProfileAllocation( + void* ptr, size_t byteSize, size_t alignment, const char* name, const char* fileName, int lineNum, int suppressStackRecord) + { #if defined(AZ_HAS_VARIADIC_TEMPLATES) && defined(AZ_DEBUG_BUILD) - ++suppressStackRecord; // one more for the fact the ebus is a function + ++suppressStackRecord; // one more for the fact the ebus is a function #endif // AZ_HAS_VARIADIC_TEMPLATES - if (m_isProfilingActive) - { - auto records = GetRecords(); - if (records) + if (m_isProfilingActive) { - records->RegisterAllocation(ptr, byteSize, alignment, name, fileName, lineNum, suppressStackRecord + 1); + auto records = GetRecords(); + if (records) + { + records->RegisterAllocation(ptr, byteSize, alignment, name, fileName, lineNum, suppressStackRecord + 1); + } } } -} -void AllocatorBase::ProfileDeallocation(void* ptr, size_t byteSize, size_t alignment, Debug::AllocationInfo* info) -{ - if (m_isProfilingActive) + void AllocatorBase::ProfileDeallocation(void* ptr, size_t byteSize, size_t alignment, Debug::AllocationInfo* info) { - auto records = GetRecords(); - if (records) + if (m_isProfilingActive) { - records->UnregisterAllocation(ptr, byteSize, alignment, info); + auto records = GetRecords(); + if (records) + { + records->UnregisterAllocation(ptr, byteSize, alignment, info); + } } } -} -void AllocatorBase::ProfileReallocationBegin([[maybe_unused]] void* ptr, [[maybe_unused]] size_t newSize) -{ -} - -void AllocatorBase::ProfileReallocationEnd(void* ptr, void* newPtr, size_t newSize, size_t newAlignment) -{ - if (m_isProfilingActive) + void AllocatorBase::ProfileReallocationBegin([[maybe_unused]] void* ptr, [[maybe_unused]] size_t newSize) { - Debug::AllocationInfo info; - ProfileDeallocation(ptr, 0, 0, &info); - ProfileAllocation(newPtr, newSize, newAlignment, info.m_name, info.m_fileName, info.m_lineNum, 0); } -} -void AllocatorBase::ProfileReallocation(void* ptr, void* newPtr, size_t newSize, size_t newAlignment) -{ - ProfileReallocationEnd(ptr, newPtr, newSize, newAlignment); -} - -void AllocatorBase::ProfileResize(void* ptr, size_t newSize) -{ - if (newSize && m_isProfilingActive) + void AllocatorBase::ProfileReallocationEnd(void* ptr, void* newPtr, size_t newSize, size_t newAlignment) { - auto records = GetRecords(); - if (records) + if (m_isProfilingActive) { - records->ResizeAllocation(ptr, newSize); + Debug::AllocationInfo info; + ProfileDeallocation(ptr, 0, 0, &info); + ProfileAllocation(newPtr, newSize, newAlignment, info.m_name, info.m_fileName, info.m_lineNum, 0); } } -} -bool AllocatorBase::OnOutOfMemory(size_t byteSize, size_t alignment, int flags, const char* name, const char* fileName, int lineNum) -{ - if (AllocatorManager::IsReady() && AllocatorManager::Instance().m_outOfMemoryListener) + void AllocatorBase::ProfileReallocation(void* ptr, void* newPtr, size_t newSize, size_t newAlignment) { - AllocatorManager::Instance().m_outOfMemoryListener(this, byteSize, alignment, flags, name, fileName, lineNum); - return true; + ProfileReallocationEnd(ptr, newPtr, newSize, newAlignment); } - return false; -} + + void AllocatorBase::ProfileResize(void* ptr, size_t newSize) + { + if (newSize && m_isProfilingActive) + { + auto records = GetRecords(); + if (records) + { + records->ResizeAllocation(ptr, newSize); + } + } + } + + bool AllocatorBase::OnOutOfMemory(size_t byteSize, size_t alignment, int flags, const char* name, const char* fileName, int lineNum) + { + if (AllocatorManager::IsReady() && AllocatorManager::Instance().m_outOfMemoryListener) + { + AllocatorManager::Instance().m_outOfMemoryListener(this, byteSize, alignment, flags, name, fileName, lineNum); + return true; + } + return false; + } + +} // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapAllocator.cpp b/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapAllocator.cpp index 30b0b78fe5..d55ca8b695 100644 --- a/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapAllocator.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapAllocator.cpp @@ -13,186 +13,182 @@ #include -using namespace AZ; - -//========================================================================= -// BestFitExternalMapAllocator -// [1/28/2011] -//========================================================================= -BestFitExternalMapAllocator::BestFitExternalMapAllocator() - : AllocatorBase(this, "BestFitExternalMapAllocator", "Best fit allocator with external tracking storage!") - , m_schema(nullptr) -{} - -//========================================================================= -// Create -// [1/28/2011] -//========================================================================= -bool -BestFitExternalMapAllocator::Create(const Descriptor& desc) +namespace AZ { - AZ_Assert(IsReady() == false, "BestFitExternalMapAllocator was already created!"); - if (IsReady()) + //========================================================================= + // BestFitExternalMapAllocator + // [1/28/2011] + //========================================================================= + BestFitExternalMapAllocator::BestFitExternalMapAllocator() + : AllocatorBase(this, "BestFitExternalMapAllocator", "Best fit allocator with external tracking storage!") + , m_schema(nullptr) { - return false; } - bool isReady = true; - - m_desc = desc; - BestFitExternalMapSchema::Descriptor schemaDesc; - schemaDesc.m_mapAllocator = desc.m_mapAllocator; - schemaDesc.m_memoryBlock = desc.m_memoryBlock; - schemaDesc.m_memoryBlockByteSize = desc.m_memoryBlockByteSize; - - m_schema = azcreate(BestFitExternalMapSchema, (schemaDesc), SystemAllocator); - if (m_schema == nullptr) + //========================================================================= + // Create + // [1/28/2011] + //========================================================================= + bool BestFitExternalMapAllocator::Create(const Descriptor& desc) { - isReady = false; + AZ_Assert(IsReady() == false, "BestFitExternalMapAllocator was already created!"); + if (IsReady()) + { + return false; + } + + bool isReady = true; + + m_desc = desc; + BestFitExternalMapSchema::Descriptor schemaDesc; + schemaDesc.m_mapAllocator = desc.m_mapAllocator; + schemaDesc.m_memoryBlock = desc.m_memoryBlock; + schemaDesc.m_memoryBlockByteSize = desc.m_memoryBlockByteSize; + + m_schema = azcreate(BestFitExternalMapSchema, (schemaDesc), SystemAllocator); + if (m_schema == nullptr) + { + isReady = false; + } + + return isReady; } - return isReady; -} + //========================================================================= + // Destroy + // [1/28/2011] + //========================================================================= + void BestFitExternalMapAllocator::Destroy() + { + azdestroy(m_schema, SystemAllocator); + m_schema = nullptr; + } -//========================================================================= -// Destroy -// [1/28/2011] -//========================================================================= -void -BestFitExternalMapAllocator::Destroy() -{ - azdestroy(m_schema, SystemAllocator); - m_schema = nullptr; -} + AllocatorDebugConfig BestFitExternalMapAllocator::GetDebugConfig() + { + return AllocatorDebugConfig() + .ExcludeFromDebugging(!m_desc.m_allocationRecords) + .StackRecordLevels(m_desc.m_stackRecordLevels) + .MarksUnallocatedMemory(false) + .UsesMemoryGuards(false); + } -AllocatorDebugConfig BestFitExternalMapAllocator::GetDebugConfig() -{ - return AllocatorDebugConfig() - .ExcludeFromDebugging(!m_desc.m_allocationRecords) - .StackRecordLevels(m_desc.m_stackRecordLevels) - .MarksUnallocatedMemory(false) - .UsesMemoryGuards(false); -} + //========================================================================= + // Allocate + // [1/28/2011] + //========================================================================= + BestFitExternalMapAllocator::pointer_type BestFitExternalMapAllocator::Allocate( + size_type byteSize, + size_type alignment, + int flags, + [[maybe_unused]] const char* name, + [[maybe_unused]] const char* fileName, + [[maybe_unused]] int lineNum, + unsigned int suppressStackRecord) + { + (void)suppressStackRecord; -//========================================================================= -// Allocate -// [1/28/2011] -//========================================================================= -BestFitExternalMapAllocator::pointer_type BestFitExternalMapAllocator::Allocate( - size_type byteSize, - size_type alignment, - int flags, - [[maybe_unused]] const char* name, - [[maybe_unused]] const char* fileName, - [[maybe_unused]] int lineNum, - unsigned int suppressStackRecord) -{ - (void)suppressStackRecord; + AZ_Assert(byteSize > 0, "You can not allocate 0 bytes!"); + AZ_Assert((alignment & (alignment - 1)) == 0, "Alignment must be power of 2!"); + byteSize = MemorySizeAdjustedUp(byteSize); - AZ_Assert(byteSize > 0, "You can not allocate 0 bytes!"); - AZ_Assert((alignment & (alignment - 1)) == 0, "Alignment must be power of 2!"); - byteSize = MemorySizeAdjustedUp(byteSize); + BestFitExternalMapAllocator::pointer_type address = m_schema->Allocate(byteSize, alignment, flags); + AZ_Assert( + address != nullptr, "BestFitExternalMapAllocator: Failed to allocate %d bytes aligned on %d (flags: 0x%08x) %s : %s (%d)!", + byteSize, alignment, flags, name ? name : "(no name)", fileName ? fileName : "(no file name)", lineNum); + AZ_MEMORY_PROFILE(ProfileAllocation(address, byteSize, alignment, name, fileName, lineNum, suppressStackRecord + 1)); - BestFitExternalMapAllocator::pointer_type address = m_schema->Allocate(byteSize, alignment, flags); - AZ_Assert(address != nullptr, "BestFitExternalMapAllocator: Failed to allocate %d bytes aligned on %d (flags: 0x%08x) %s : %s (%d)!", byteSize, alignment, flags, name ? name : "(no name)", fileName ? fileName : "(no file name)", lineNum); - AZ_MEMORY_PROFILE(ProfileAllocation(address, byteSize, alignment, name, fileName, lineNum, suppressStackRecord + 1)); + return address; + } - return address; -} + //========================================================================= + // DeAllocate + // [1/28/2011] + //========================================================================= + void BestFitExternalMapAllocator::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) + { + byteSize = MemorySizeAdjustedUp(byteSize); + AZ_MEMORY_PROFILE(ProfileDeallocation(ptr, byteSize, alignment, nullptr)); -//========================================================================= -// DeAllocate -// [1/28/2011] -//========================================================================= -void -BestFitExternalMapAllocator::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) -{ - byteSize = MemorySizeAdjustedUp(byteSize); - AZ_MEMORY_PROFILE(ProfileDeallocation(ptr, byteSize, alignment, nullptr)); + (void)byteSize; + (void)alignment; + m_schema->DeAllocate(ptr); + } - (void)byteSize; - (void)alignment; - m_schema->DeAllocate(ptr); -} + //========================================================================= + // Resize + // [1/28/2011] + //========================================================================= + BestFitExternalMapAllocator::size_type BestFitExternalMapAllocator::Resize(pointer_type ptr, size_type newSize) + { + (void)ptr; + (void)newSize; + /* todo */ + return 0; + } -//========================================================================= -// Resize -// [1/28/2011] -//========================================================================= -BestFitExternalMapAllocator::size_type -BestFitExternalMapAllocator::Resize(pointer_type ptr, size_type newSize) -{ - (void)ptr; - (void)newSize; - /* todo */ - return 0; -} + //========================================================================= + // ReAllocate + // [9/13/2011] + //========================================================================= + BestFitExternalMapAllocator::pointer_type BestFitExternalMapAllocator::ReAllocate( + pointer_type ptr, size_type newSize, size_type newAlignment) + { + (void)ptr; + (void)newSize; + (void)newAlignment; + AZ_Assert(false, "Not supported!"); + return nullptr; + } -//========================================================================= -// ReAllocate -// [9/13/2011] -//========================================================================= -BestFitExternalMapAllocator::pointer_type -BestFitExternalMapAllocator::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) -{ - (void)ptr; - (void)newSize; - (void)newAlignment; - AZ_Assert(false, "Not supported!"); - return nullptr; -} + //========================================================================= + // AllocationSize + // [1/28/2011] + //========================================================================= + BestFitExternalMapAllocator::size_type BestFitExternalMapAllocator::AllocationSize(pointer_type ptr) + { + return MemorySizeAdjustedDown(m_schema->AllocationSize(ptr)); + } -//========================================================================= -// AllocationSize -// [1/28/2011] -//========================================================================= -BestFitExternalMapAllocator::size_type -BestFitExternalMapAllocator::AllocationSize(pointer_type ptr) -{ - return MemorySizeAdjustedDown(m_schema->AllocationSize(ptr)); -} + //========================================================================= + // NumAllocatedBytes + // [1/28/2011] + //========================================================================= + BestFitExternalMapAllocator::size_type BestFitExternalMapAllocator::NumAllocatedBytes() const + { + return m_schema->NumAllocatedBytes(); + } -//========================================================================= -// NumAllocatedBytes -// [1/28/2011] -//========================================================================= -BestFitExternalMapAllocator::size_type -BestFitExternalMapAllocator::NumAllocatedBytes() const -{ - return m_schema->NumAllocatedBytes(); -} + //========================================================================= + // Capacity + // [1/28/2011] + //========================================================================= + BestFitExternalMapAllocator::size_type BestFitExternalMapAllocator::Capacity() const + { + return m_schema->Capacity(); + } -//========================================================================= -// Capacity -// [1/28/2011] -//========================================================================= -BestFitExternalMapAllocator::size_type -BestFitExternalMapAllocator::Capacity() const -{ - return m_schema->Capacity(); -} + //========================================================================= + // GetMaxAllocationSize + // [1/28/2011] + //========================================================================= + BestFitExternalMapAllocator::size_type BestFitExternalMapAllocator::GetMaxAllocationSize() const + { + return m_schema->GetMaxAllocationSize(); + } -//========================================================================= -// GetMaxAllocationSize -// [1/28/2011] -//========================================================================= -BestFitExternalMapAllocator::size_type -BestFitExternalMapAllocator::GetMaxAllocationSize() const -{ - return m_schema->GetMaxAllocationSize(); -} + auto BestFitExternalMapAllocator::GetMaxContiguousAllocationSize() const -> size_type + { + return m_schema->GetMaxContiguousAllocationSize(); + } -auto BestFitExternalMapAllocator::GetMaxContiguousAllocationSize() const -> size_type -{ - return m_schema->GetMaxContiguousAllocationSize(); -} + //========================================================================= + // GetSubAllocator + // [1/28/2011] + //========================================================================= + IAllocatorAllocate* BestFitExternalMapAllocator::GetSubAllocator() + { + return m_schema->GetSubAllocator(); + } -//========================================================================= -// GetSubAllocator -// [1/28/2011] -//========================================================================= -IAllocatorAllocate* -BestFitExternalMapAllocator::GetSubAllocator() -{ - return m_schema->GetSubAllocator(); -} +} // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapAllocator.h b/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapAllocator.h index 17425625b7..90e2056d65 100644 --- a/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapAllocator.h +++ b/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapAllocator.h @@ -5,8 +5,7 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT * */ -#ifndef AZ_BEST_FIT_EXT_MAP_ALLOCATOR_H -#define AZ_BEST_FIT_EXT_MAP_ALLOCATOR_H +#pragma once #include @@ -76,7 +75,3 @@ namespace AZ }; } -#endif // AZ_BEST_FIT_EXT_MAP_ALLOCATOR_H -#pragma once - - diff --git a/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapSchema.cpp b/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapSchema.cpp index 715ecd221e..841f36f58a 100644 --- a/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapSchema.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapSchema.cpp @@ -9,194 +9,199 @@ #include #include -using namespace AZ; - -//========================================================================= -// BestFitExternalMapSchema -// [1/28/2011] -//========================================================================= -BestFitExternalMapSchema::BestFitExternalMapSchema(const Descriptor& desc) - : m_desc(desc) - , m_used(0) - , m_freeChunksMap(FreeMapType::key_compare(), AZStdIAllocator(desc.m_mapAllocator != nullptr ? desc.m_mapAllocator : &AllocatorInstance::Get())) - , m_allocChunksMap(AllocMapType::hasher(), AllocMapType::key_eq(), AZStdIAllocator(desc.m_mapAllocator != nullptr ? desc.m_mapAllocator : &AllocatorInstance::Get())) +namespace AZ { - if (m_desc.m_mapAllocator == nullptr) + //========================================================================= + // BestFitExternalMapSchema + // [1/28/2011] + //========================================================================= + BestFitExternalMapSchema::BestFitExternalMapSchema(const Descriptor& desc) + : m_desc(desc) + , m_used(0) + , m_freeChunksMap( + FreeMapType::key_compare(), + AZStdIAllocator(desc.m_mapAllocator != nullptr ? desc.m_mapAllocator : &AllocatorInstance::Get())) + , m_allocChunksMap( + AllocMapType::hasher(), + AllocMapType::key_eq(), + AZStdIAllocator(desc.m_mapAllocator != nullptr ? desc.m_mapAllocator : &AllocatorInstance::Get())) { - m_desc.m_mapAllocator = &AllocatorInstance::Get(); // used as our sub allocator - } - AZ_Assert(m_desc.m_memoryBlockByteSize > 0, "You must provide memory block size!"); - AZ_Assert(m_desc.m_memoryBlock != nullptr, "You must provide memory block allocated as you with!"); - //if( m_desc.m_memoryBlock == NULL) there is no point to automate this cause we need to flag this memory special, otherwise there is no point to use this allocator at all - // m_desc.m_memoryBlock = azmalloc(SystemAllocator,m_desc.m_memoryBlockByteSize,16); - m_freeChunksMap.insert(AZStd::make_pair(m_desc.m_memoryBlockByteSize, reinterpret_cast(m_desc.m_memoryBlock))); -} - -//========================================================================= -// Allocate -// [1/28/2011] -//========================================================================= -BestFitExternalMapSchema::pointer_type -BestFitExternalMapSchema::Allocate(size_type byteSize, size_type alignment, int flags) -{ - (void)flags; - char* address = nullptr; - AZ_Assert(alignment > 0 && (alignment & (alignment - 1)) == 0, "Alignment must be >0 and power of 2!"); - for (int i = 0; i < 2; ++i) // max 2 attempts to allocate - { - FreeMapType::iterator iter = m_freeChunksMap.find(byteSize); - size_t blockSize = 0; - char* blockAddress = nullptr; - size_t preAllocBlockSize = 0; - while (iter != m_freeChunksMap.end()) + if (m_desc.m_mapAllocator == nullptr) { - blockSize = iter->first; - blockAddress = iter->second; - char* alignedAddr = PointerAlignUp(blockAddress, alignment); - preAllocBlockSize = alignedAddr - blockAddress; - if (preAllocBlockSize + byteSize <= blockSize) - { - m_freeChunksMap.erase(iter); // we have our allocation - m_used += byteSize; - address = alignedAddr; - m_allocChunksMap.insert(AZStd::make_pair(address, byteSize)); - break; - } - ++iter; + m_desc.m_mapAllocator = &AllocatorInstance::Get(); // used as our sub allocator } - if (address != nullptr) + AZ_Assert(m_desc.m_memoryBlockByteSize > 0, "You must provide memory block size!"); + AZ_Assert(m_desc.m_memoryBlock != nullptr, "You must provide memory block allocated as you with!"); + // if( m_desc.m_memoryBlock == NULL) there is no point to automate this cause we need to flag this memory special, otherwise there + // is no point to use this allocator at all + // m_desc.m_memoryBlock = azmalloc(SystemAllocator,m_desc.m_memoryBlockByteSize,16); + m_freeChunksMap.insert(AZStd::make_pair(m_desc.m_memoryBlockByteSize, reinterpret_cast(m_desc.m_memoryBlock))); + } + + //========================================================================= + // Allocate + // [1/28/2011] + //========================================================================= + BestFitExternalMapSchema::pointer_type BestFitExternalMapSchema::Allocate(size_type byteSize, size_type alignment, int flags) + { + (void)flags; + char* address = nullptr; + AZ_Assert(alignment > 0 && (alignment & (alignment - 1)) == 0, "Alignment must be >0 and power of 2!"); + for (int i = 0; i < 2; ++i) // max 2 attempts to allocate { - // split blocks - if (preAllocBlockSize) // if we have a block before the alignment + FreeMapType::iterator iter = m_freeChunksMap.find(byteSize); + size_t blockSize = 0; + char* blockAddress = nullptr; + size_t preAllocBlockSize = 0; + while (iter != m_freeChunksMap.end()) { - m_freeChunksMap.insert(AZStd::make_pair(preAllocBlockSize, blockAddress)); - } - size_t postAllocBlockSize = blockSize - preAllocBlockSize - byteSize; - if (postAllocBlockSize) - { - m_freeChunksMap.insert(AZStd::make_pair(postAllocBlockSize, address + byteSize)); - } - - break; - } - else - { - GarbageCollect(); - } - } - return address; -} - -//========================================================================= -// DeAllocate -// [1/28/2011] -//========================================================================= -void -BestFitExternalMapSchema::DeAllocate(pointer_type ptr) -{ - if (ptr == nullptr) - { - return; - } - AllocMapType::iterator iter = m_allocChunksMap.find(reinterpret_cast(ptr)); - if (iter != m_allocChunksMap.end()) - { - m_used -= iter->second; - m_freeChunksMap.insert(AZStd::make_pair(iter->second, iter->first)); - m_allocChunksMap.erase(iter); - } -} - -//========================================================================= -// AllocationSize -// [1/28/2011] -//========================================================================= -BestFitExternalMapSchema::size_type -BestFitExternalMapSchema::AllocationSize(pointer_type ptr) -{ - AllocMapType::iterator iter = m_allocChunksMap.find(reinterpret_cast(ptr)); - if (iter != m_allocChunksMap.end()) - { - return iter->second; - } - return 0; -} - -//========================================================================= -// GetMaxAllocationSize -// [1/28/2011] -//========================================================================= -BestFitExternalMapSchema::size_type -BestFitExternalMapSchema::GetMaxAllocationSize() const -{ - if (!m_freeChunksMap.empty()) - { - return m_freeChunksMap.rbegin()->first; - } - return 0; -} - -auto BestFitExternalMapSchema::GetMaxContiguousAllocationSize() const -> size_type -{ - // Return the maximum size of any single allocation - return AZ_CORE_MAX_ALLOCATOR_SIZE; -} - -//========================================================================= -// GarbageCollect -// [1/28/2011] -//========================================================================= -void -BestFitExternalMapSchema::GarbageCollect() -{ - for (FreeMapType::iterator curBlock = m_freeChunksMap.begin(); curBlock != m_freeChunksMap.end(); ) - { - char* curStart = curBlock->second; - char* curEnd = curStart + curBlock->first; - bool isMerge = false; - for (FreeMapType::iterator nextBlock = curBlock++; nextBlock != m_freeChunksMap.end(); ) - { - char* nextStart = nextBlock->second; - char* nextEnd = nextStart + nextBlock->first; - if (curStart == nextEnd) - { - // merge - size_t newBlockSize = curBlock->first + nextBlock->first; - char* newBlockAddress = nextStart; - m_freeChunksMap.erase(nextBlock); - FreeMapType::iterator toErase = curBlock; - ++curBlock; - m_freeChunksMap.erase(toErase); - FreeMapType::iterator newBlock = m_freeChunksMap.insert(AZStd::make_pair(newBlockSize, newBlockAddress)).first; - if (curBlock != m_freeChunksMap.end() && newBlockSize < curBlock->first) // if the newBlock in before the next in the list, update next in the list to current + blockSize = iter->first; + blockAddress = iter->second; + char* alignedAddr = PointerAlignUp(blockAddress, alignment); + preAllocBlockSize = alignedAddr - blockAddress; + if (preAllocBlockSize + byteSize <= blockSize) { - curBlock = newBlock; + m_freeChunksMap.erase(iter); // we have our allocation + m_used += byteSize; + address = alignedAddr; + m_allocChunksMap.insert(AZStd::make_pair(address, byteSize)); + break; } - isMerge = true; - break; + ++iter; } - else if (curEnd == nextStart) + if (address != nullptr) { - // merge - size_t newBlockSize = curBlock->first + nextBlock->first; - char* newBlockAddress = curStart; - m_freeChunksMap.erase(nextBlock); - FreeMapType::iterator toErase = curBlock; - ++curBlock; - m_freeChunksMap.erase(toErase); - FreeMapType::iterator newBlock = m_freeChunksMap.insert(AZStd::make_pair(newBlockSize, newBlockAddress)).first; - if (curBlock != m_freeChunksMap.end() && newBlockSize < curBlock->first) // if the newBlock in before the next in the list, update next in the list to current + // split blocks + if (preAllocBlockSize) // if we have a block before the alignment { - curBlock = newBlock; + m_freeChunksMap.insert(AZStd::make_pair(preAllocBlockSize, blockAddress)); } - isMerge = true; + size_t postAllocBlockSize = blockSize - preAllocBlockSize - byteSize; + if (postAllocBlockSize) + { + m_freeChunksMap.insert(AZStd::make_pair(postAllocBlockSize, address + byteSize)); + } + break; } - ++nextBlock; + else + { + GarbageCollect(); + } } - if (!isMerge) + return address; + } + + //========================================================================= + // DeAllocate + // [1/28/2011] + //========================================================================= + void BestFitExternalMapSchema::DeAllocate(pointer_type ptr) + { + if (ptr == nullptr) { - ++curBlock; + return; + } + AllocMapType::iterator iter = m_allocChunksMap.find(reinterpret_cast(ptr)); + if (iter != m_allocChunksMap.end()) + { + m_used -= iter->second; + m_freeChunksMap.insert(AZStd::make_pair(iter->second, iter->first)); + m_allocChunksMap.erase(iter); } } -} + + //========================================================================= + // AllocationSize + // [1/28/2011] + //========================================================================= + BestFitExternalMapSchema::size_type BestFitExternalMapSchema::AllocationSize(pointer_type ptr) + { + AllocMapType::iterator iter = m_allocChunksMap.find(reinterpret_cast(ptr)); + if (iter != m_allocChunksMap.end()) + { + return iter->second; + } + return 0; + } + + //========================================================================= + // GetMaxAllocationSize + // [1/28/2011] + //========================================================================= + BestFitExternalMapSchema::size_type BestFitExternalMapSchema::GetMaxAllocationSize() const + { + if (!m_freeChunksMap.empty()) + { + return m_freeChunksMap.rbegin()->first; + } + return 0; + } + + auto BestFitExternalMapSchema::GetMaxContiguousAllocationSize() const -> size_type + { + // Return the maximum size of any single allocation + return AZ_CORE_MAX_ALLOCATOR_SIZE; + } + + //========================================================================= + // GarbageCollect + // [1/28/2011] + //========================================================================= + void BestFitExternalMapSchema::GarbageCollect() + { + for (FreeMapType::iterator curBlock = m_freeChunksMap.begin(); curBlock != m_freeChunksMap.end();) + { + char* curStart = curBlock->second; + char* curEnd = curStart + curBlock->first; + bool isMerge = false; + for (FreeMapType::iterator nextBlock = curBlock++; nextBlock != m_freeChunksMap.end();) + { + char* nextStart = nextBlock->second; + char* nextEnd = nextStart + nextBlock->first; + if (curStart == nextEnd) + { + // merge + size_t newBlockSize = curBlock->first + nextBlock->first; + char* newBlockAddress = nextStart; + m_freeChunksMap.erase(nextBlock); + FreeMapType::iterator toErase = curBlock; + ++curBlock; + m_freeChunksMap.erase(toErase); + FreeMapType::iterator newBlock = m_freeChunksMap.insert(AZStd::make_pair(newBlockSize, newBlockAddress)).first; + // if the newBlock in before the next in the list, update next in the list to current + if (curBlock != m_freeChunksMap.end() && newBlockSize < curBlock->first) + { + curBlock = newBlock; + } + isMerge = true; + break; + } + else if (curEnd == nextStart) + { + // merge + size_t newBlockSize = curBlock->first + nextBlock->first; + char* newBlockAddress = curStart; + m_freeChunksMap.erase(nextBlock); + FreeMapType::iterator toErase = curBlock; + ++curBlock; + m_freeChunksMap.erase(toErase); + FreeMapType::iterator newBlock = m_freeChunksMap.insert(AZStd::make_pair(newBlockSize, newBlockAddress)).first; + // if the newBlock in before the next in the list, update next in the list to current + if (curBlock != m_freeChunksMap.end() && newBlockSize < curBlock->first) + { + curBlock = newBlock; + } + isMerge = true; + break; + } + ++nextBlock; + } + if (!isMerge) + { + ++curBlock; + } + } + } + +} // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapSchema.h b/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapSchema.h index eaab614593..0055a86ee2 100644 --- a/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapSchema.h +++ b/Code/Framework/AzCore/AzCore/Memory/BestFitExternalMapSchema.h @@ -5,8 +5,7 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT * */ -#ifndef AZ_BEST_FIT_EXT_MAP_ALLOCATION_SCHEME_H -#define AZ_BEST_FIT_EXT_MAP_ALLOCATION_SCHEME_H +#pragma once #include #include @@ -77,8 +76,3 @@ namespace AZ AllocMapType m_allocChunksMap; }; } - -#endif // AZ_BEST_FIT_EXT_MAP_ALLOCATION_SCHEME_H -#pragma once - - diff --git a/Code/Framework/AzCore/AzCore/Memory/PoolSchema.cpp b/Code/Framework/AzCore/AzCore/Memory/PoolSchema.cpp index 9167864450..c57cfea222 100644 --- a/Code/Framework/AzCore/AzCore/Memory/PoolSchema.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/PoolSchema.cpp @@ -27,8 +27,8 @@ namespace AZ ////////////////////////////////////////////////////////////////////////// // Pool Allocation algorithm /** - * Pool Allocation algorithm implementation. Used in both PoolAllocator and ThreadPoolAllocator. - */ + * Pool Allocation algorithm implementation. Used in both PoolAllocator and ThreadPoolAllocator. + */ template class PoolAllocation { @@ -41,20 +41,20 @@ namespace AZ PoolAllocation(Allocator* alloc, size_t pageSize, size_t minAllocationSize, size_t maxAllocationSize); virtual ~PoolAllocation(); - void* Allocate(size_t byteSize, size_t alignment); - void DeAllocate(void* ptr); - size_t AllocationSize(void* ptr); + void* Allocate(size_t byteSize, size_t alignment); + void DeAllocate(void* ptr); + size_t AllocationSize(void* ptr); // if isForceFreeAllPages is true we will free all pages even if they have allocations in them. - void GarbageCollect(bool isForceFreeAllPages = false); + void GarbageCollect(bool isForceFreeAllPages = false); - Allocator* m_allocator; - size_t m_pageSize; - size_t m_minAllocationShift; - size_t m_minAllocationSize; - size_t m_maxAllocationSize; - size_t m_numBuckets; - BucketType* m_buckets; - size_t m_numBytesAllocated; + Allocator* m_allocator; + size_t m_pageSize; + size_t m_minAllocationShift; + size_t m_minAllocationSize; + size_t m_maxAllocationSize; + size_t m_numBuckets; + BucketType* m_buckets; + size_t m_numBytesAllocated; }; /** @@ -68,23 +68,22 @@ namespace AZ PoolSchemaImpl(const PoolSchema::Descriptor& desc); ~PoolSchemaImpl(); - PoolSchema::pointer_type Allocate(PoolSchema::size_type byteSize, PoolSchema::size_type alignment, int flags = 0); - void DeAllocate(PoolSchema::pointer_type ptr); - PoolSchema::size_type AllocationSize(PoolSchema::pointer_type ptr); + PoolSchema::pointer_type Allocate(PoolSchema::size_type byteSize, PoolSchema::size_type alignment, int flags = 0); + void DeAllocate(PoolSchema::pointer_type ptr); + PoolSchema::size_type AllocationSize(PoolSchema::pointer_type ptr); /** - * We allocate memory for pools in pages. Page is a information struct - * located at the end of the allocated page. When it's in the at the end - * we can usually hide it's size in the free bytes left from the pagesize/poolsize. - * \note IMPORTANT pages are aligned on the page size, this way can find quickly which - * pool the pointer belongs to. - */ - struct Page - : public AZStd::intrusive_list_node + * We allocate memory for pools in pages. Page is a information struct + * located at the end of the allocated page. When it's in the at the end + * we can usually hide it's size in the free bytes left from the pagesize/poolsize. + * \note IMPORTANT pages are aligned on the page size, this way can find quickly which + * pool the pointer belongs to. + */ + struct Page : public AZStd::intrusive_list_node { - struct FakeNode - : public AZStd::intrusive_slist_node - {}; + struct FakeNode : public AZStd::intrusive_slist_node + { + }; void SetupFreeList(size_t elementSize, size_t pageDataBlockSize); @@ -99,22 +98,22 @@ namespace AZ }; /** - * A bucket has a list of pages used with the specific pool size. - */ + * A bucket has a list of pages used with the specific pool size. + */ struct Bucket { using PageListType = AZStd::intrusive_list>; - PageListType m_pages; + PageListType m_pages; }; // Functions used by PoolAllocation template AZ_INLINE Page* PopFreePage(); - AZ_INLINE void PushFreePage(Page* page); - void GarbageCollect(); - inline bool IsInStaticBlock(Page* page) + AZ_INLINE void PushFreePage(Page* page); + void GarbageCollect(); + inline bool IsInStaticBlock(Page* page) { const char* staticBlockStart = reinterpret_cast(m_staticDataBlock); - const char* staticBlockEnd = staticBlockStart + m_numStaticPages*m_pageSize; + const char* staticBlockEnd = staticBlockStart + m_numStaticPages * m_pageSize; const char* pageAddress = reinterpret_cast(page); // all pages are the same size so we either in or out, no need to check the pageAddressEnd if (pageAddress >= staticBlockStart && pageAddress < staticBlockEnd) @@ -126,21 +125,22 @@ namespace AZ return false; } } - inline Page* ConstructPage(size_t elementSize) + inline Page* ConstructPage(size_t elementSize) { AZ_Assert(m_isDynamic, "We run out of static pages (%d) and this is a static allocator!", m_numStaticPages); // We store the page struct at the end of the block char* memBlock; - memBlock = reinterpret_cast(m_pageAllocator->Allocate(m_pageSize, m_pageSize, 0, "AZSystem::PoolSchemaImpl::ConstructPage", __FILE__, __LINE__)); + memBlock = reinterpret_cast( + m_pageAllocator->Allocate(m_pageSize, m_pageSize, 0, "AZSystem::PoolSchemaImpl::ConstructPage", __FILE__, __LINE__)); size_t pageDataSize = m_pageSize - sizeof(Page); - Page* page = new(memBlock+pageDataSize)Page(); + Page* page = new (memBlock + pageDataSize) Page(); page->SetupFreeList(elementSize, pageDataSize); page->m_elementSize = static_cast(elementSize); page->m_maxNumElements = static_cast(pageDataSize / elementSize); return page; } - inline void FreePage(Page* page) + inline void FreePage(Page* page) { // TODO: It's optional if we want to check the guard value for corruption, since we are not going // to use this memory. Yet it might be useful to catch bugs. @@ -150,9 +150,9 @@ namespace AZ m_pageAllocator->DeAllocate(memBlock); } - inline Page* PageFromAddress(void* address) + inline Page* PageFromAddress(void* address) { - char* memBlock = reinterpret_cast(reinterpret_cast(address) & ~(m_pageSize-1)); + char* memBlock = reinterpret_cast(reinterpret_cast(address) & ~(m_pageSize - 1)); memBlock += m_pageSize - sizeof(Page); Page* page = reinterpret_cast(memBlock); if (!page->m_magic.Validate()) @@ -163,18 +163,18 @@ namespace AZ } using AllocatorType = PoolAllocation; - IAllocatorAllocate* m_pageAllocator; - AllocatorType m_allocator; - void* m_staticDataBlock; - unsigned int m_numStaticPages; - bool m_isDynamic; - size_t m_pageSize; - Bucket::PageListType m_freePages; + IAllocatorAllocate* m_pageAllocator; + AllocatorType m_allocator; + void* m_staticDataBlock; + unsigned int m_numStaticPages; + bool m_isDynamic; + size_t m_pageSize; + Bucket::PageListType m_freePages; }; /** - * Thread safe pool allocator. - */ + * Thread safe pool allocator. + */ class ThreadPoolSchemaImpl { public: @@ -183,18 +183,20 @@ namespace AZ /** * Specialized \ref PoolAllocator::Page page for lock free allocator. */ - struct Page - : public AZStd::intrusive_list_node + struct Page : public AZStd::intrusive_list_node { Page(ThreadPoolData* threadData) - : m_threadData(threadData) {} + : m_threadData(threadData) + { + } - struct FakeNode - : public AZStd::intrusive_slist_node - {}; + struct FakeNode : public AZStd::intrusive_slist_node + { + }; // Fake Lock Free node used when we delete an element from another thread. - struct FakeNodeLF - : public AZStd::lock_free_intrusive_stack_node{}; + struct FakeNodeLF : public AZStd::lock_free_intrusive_stack_node + { + }; void SetupFreeList(size_t elementSize, size_t pageDataBlockSize); @@ -202,8 +204,8 @@ namespace AZ using FreeListType = AZStd::intrusive_slist>; FreeListType m_freeList; - AZStd::lock_free_intrusive_stack_node m_lfStack; ///< Lock Free stack node - struct ThreadPoolData* m_threadData; ///< The thread data that own's the page. + AZStd::lock_free_intrusive_stack_node m_lfStack; ///< Lock Free stack node + struct ThreadPoolData* m_threadData; ///< The thread data that own's the page. u32 m_bin; Debug::Magic32 m_magic; u32 m_elementSize; @@ -211,31 +213,34 @@ namespace AZ }; /** - * A bucket has a list of pages used with the specific pool size. - */ + * A bucket has a list of pages used with the specific pool size. + */ struct Bucket { using PageListType = AZStd::intrusive_list>; - PageListType m_pages; + PageListType m_pages; }; - ThreadPoolSchemaImpl(const ThreadPoolSchema::Descriptor& desc, ThreadPoolSchema::GetThreadPoolData threadPoolGetter, ThreadPoolSchema::SetThreadPoolData threadPoolSetter); + ThreadPoolSchemaImpl( + const ThreadPoolSchema::Descriptor& desc, + ThreadPoolSchema::GetThreadPoolData threadPoolGetter, + ThreadPoolSchema::SetThreadPoolData threadPoolSetter); ~ThreadPoolSchemaImpl(); - ThreadPoolSchema::pointer_type Allocate(ThreadPoolSchema::size_type byteSize, ThreadPoolSchema::size_type alignment, int flags = 0); - void DeAllocate(ThreadPoolSchema::pointer_type ptr); - ThreadPoolSchema::size_type AllocationSize(ThreadPoolSchema::pointer_type ptr); + ThreadPoolSchema::pointer_type Allocate(ThreadPoolSchema::size_type byteSize, ThreadPoolSchema::size_type alignment, int flags = 0); + void DeAllocate(ThreadPoolSchema::pointer_type ptr); + ThreadPoolSchema::size_type AllocationSize(ThreadPoolSchema::pointer_type ptr); /// Return unused memory to the OS. Don't call this too often because you will force unnecessary allocations. - void GarbageCollect(); + void GarbageCollect(); ////////////////////////////////////////////////////////////////////////// // Functions used by PoolAllocation template AZ_INLINE Page* PopFreePage(); - AZ_INLINE void PushFreePage(Page* page); - inline bool IsInStaticBlock(Page* page) + AZ_INLINE void PushFreePage(Page* page); + inline bool IsInStaticBlock(Page* page) { const char* staticBlockStart = reinterpret_cast(m_staticDataBlock); - const char* staticBlockEnd = staticBlockStart + m_numStaticPages*m_pageSize; + const char* staticBlockEnd = staticBlockStart + m_numStaticPages * m_pageSize; const char* pageAddress = reinterpret_cast(page); // all pages are the same size so we either in or out, no need to check the pageAddressEnd if (pageAddress > staticBlockStart && pageAddress < staticBlockEnd) @@ -247,33 +252,34 @@ namespace AZ return false; } } - inline Page* ConstructPage(size_t elementSize) + inline Page* ConstructPage(size_t elementSize) { AZ_Assert(m_isDynamic, "We run out of static pages (%d) and this is a static allocator!", m_numStaticPages); // We store the page struct at the end of the block char* memBlock; - memBlock = reinterpret_cast(m_pageAllocator->Allocate(m_pageSize, m_pageSize, 0, "AZSystem::ThreadPoolSchema::ConstructPage", __FILE__, __LINE__)); + memBlock = reinterpret_cast( + m_pageAllocator->Allocate(m_pageSize, m_pageSize, 0, "AZSystem::ThreadPoolSchema::ConstructPage", __FILE__, __LINE__)); size_t pageDataSize = m_pageSize - sizeof(Page); - Page* page = new(memBlock+pageDataSize)Page(m_threadPoolGetter()); + Page* page = new (memBlock + pageDataSize) Page(m_threadPoolGetter()); page->SetupFreeList(elementSize, pageDataSize); page->m_elementSize = static_cast(elementSize); page->m_maxNumElements = static_cast(pageDataSize / elementSize); return page; } - inline void FreePage(Page* page) + inline void FreePage(Page* page) { // TODO: It's optional if we want to check the guard value for corruption, since we are not going // to use this memory. Yet it might be useful to catch bugs. // We store the page struct at the end of the block char* memBlock = reinterpret_cast(page) - m_pageSize + sizeof(Page); - page->~Page(); // destroy the page + page->~Page(); // destroy the page m_pageAllocator->DeAllocate(memBlock); } - inline Page* PageFromAddress(void* address) + inline Page* PageFromAddress(void* address) { - char* memBlock = reinterpret_cast(reinterpret_cast(address) & ~static_cast(m_pageSize-1)); + char* memBlock = reinterpret_cast(reinterpret_cast(address) & ~static_cast(m_pageSize - 1)); memBlock += m_pageSize - sizeof(Page); Page* page = reinterpret_cast(memBlock); if (!page->m_magic.Validate()) @@ -292,18 +298,18 @@ namespace AZ // Fox X64 we push/pop pages using the m_mutex to sync. Pages are using FreePagesType = Bucket::PageListType; - FreePagesType m_freePages; - AZStd::vector m_threads; ///< Array with all separate thread data. Used to traverse end free elements. + FreePagesType m_freePages; + AZStd::vector m_threads; ///< Array with all separate thread data. Used to traverse end free elements. - IAllocatorAllocate* m_pageAllocator; - void* m_staticDataBlock; - size_t m_numStaticPages; - size_t m_pageSize; - size_t m_minAllocationSize; - size_t m_maxAllocationSize; - bool m_isDynamic; + IAllocatorAllocate* m_pageAllocator; + void* m_staticDataBlock; + size_t m_numStaticPages; + size_t m_pageSize; + size_t m_minAllocationSize; + size_t m_maxAllocationSize; + bool m_isDynamic; // TODO rbbaklov Changed to recursive_mutex from mutex for Linux support. - AZStd::recursive_mutex m_mutex; + AZStd::recursive_mutex m_mutex; }; struct ThreadPoolData @@ -315,1090 +321,1088 @@ namespace AZ using AllocatorType = PoolAllocation; /** - * Stack with freed elements from other threads. We don't need stamped stack since the ABA problem can not - * happen here. We push from many threads and pop from only one (we don't push from it). - */ - using FreedElementsStack = AZStd::lock_free_intrusive_stack>; + * Stack with freed elements from other threads. We don't need stamped stack since the ABA problem can not + * happen here. We push from many threads and pop from only one (we don't push from it). + */ + using FreedElementsStack = AZStd::lock_free_intrusive_stack< + ThreadPoolSchemaImpl::Page::FakeNodeLF, + AZStd::lock_free_intrusive_stack_base_hook>; - AllocatorType m_allocator; - FreedElementsStack m_freedElements; + AllocatorType m_allocator; + FreedElementsStack m_freedElements; }; -} +} // namespace AZ -using namespace AZ; - -//========================================================================= -// PoolAllocation -// [9/09/2009] -//========================================================================= -template -PoolAllocation::PoolAllocation(Allocator* alloc, size_t pageSize, size_t minAllocationSize, size_t maxAllocationSize) - : m_allocator(alloc) - , m_pageSize(pageSize) - , m_numBytesAllocated(0) +namespace AZ { - AZ_Assert(alloc->m_pageAllocator, "We need the page allocator setup!"); - AZ_Assert(pageSize >= maxAllocationSize * 4, "We need to fit at least 4 objects in a pool! Increase your page size! Page %d MaxAllocationSize %d",pageSize,maxAllocationSize); - AZ_Assert(minAllocationSize == maxAllocationSize || ((minAllocationSize)&(minAllocationSize - 1)) == 0, "Min allocation should be either equal to max allocation size or power of two"); - - m_minAllocationSize = AZ::GetMax(minAllocationSize, size_t(8)); - m_maxAllocationSize = AZ::GetMax(maxAllocationSize, minAllocationSize); - - m_minAllocationShift = 0; - for (size_t i = 1; i < sizeof(unsigned int)*8; i++) + //========================================================================= + // PoolAllocation + // [9/09/2009] + //========================================================================= + template + PoolAllocation::PoolAllocation(Allocator* alloc, size_t pageSize, size_t minAllocationSize, size_t maxAllocationSize) + : m_allocator(alloc) + , m_pageSize(pageSize) + , m_numBytesAllocated(0) { - if (m_minAllocationSize >> i == 0) + AZ_Assert(alloc->m_pageAllocator, "We need the page allocator setup!"); + AZ_Assert( + pageSize >= maxAllocationSize * 4, + "We need to fit at least 4 objects in a pool! Increase your page size! Page %d MaxAllocationSize %d", pageSize, + maxAllocationSize); + AZ_Assert( + minAllocationSize == maxAllocationSize || ((minAllocationSize) & (minAllocationSize - 1)) == 0, + "Min allocation should be either equal to max allocation size or power of two"); + + m_minAllocationSize = AZ::GetMax(minAllocationSize, size_t(8)); + m_maxAllocationSize = AZ::GetMax(maxAllocationSize, minAllocationSize); + + m_minAllocationShift = 0; + for (size_t i = 1; i < sizeof(unsigned int) * 8; i++) { - m_minAllocationShift = i-1; - break; - } - } - - AZ_Assert(m_maxAllocationSize % m_minAllocationSize == 0, "You need to be able to divide m_maxAllocationSize (%d) / m_minAllocationSize (%d) without fraction!", m_maxAllocationSize, m_minAllocationSize); - m_numBuckets = m_maxAllocationSize / m_minAllocationSize; - AZ_Assert(m_numBuckets <= 0xffff, "You can't have more than 65535 number of buckets! We need to increase the index size!"); - m_buckets = reinterpret_cast(alloc->m_pageAllocator->Allocate(sizeof(BucketType)*m_numBuckets, AZStd::alignment_of::value)); - for (size_t i = 0; i < m_numBuckets; ++i) - { - new(m_buckets + i)BucketType(); - } -} - -//========================================================================= -// ~PoolAllocation -// [9/09/2009] -//========================================================================= -template -PoolAllocation::~PoolAllocation() -{ - GarbageCollect(true); - - for (size_t i = 0; i < m_numBuckets; ++i) - { - m_buckets[i].~BucketType(); - } - m_allocator->m_pageAllocator->DeAllocate(m_buckets, sizeof(BucketType) * m_numBuckets); -} - -//========================================================================= -// Allocate -// [9/09/2009] -//========================================================================= -template -AZ_INLINE void* -PoolAllocation::Allocate(size_t byteSize, size_t alignment) -{ - AZ_Assert(byteSize>0, "You can not allocate 0 bytes!"); - AZ_Assert(alignment>0&&(alignment&(alignment-1))==0, "Alignment must be >0 and power of 2!"); - - // pad the size to the min allocation size. - byteSize = AZ::SizeAlignUp(byteSize, m_minAllocationSize); - byteSize = AZ::SizeAlignUp(byteSize, alignment); - - if (byteSize > m_maxAllocationSize) - { - AZ_Assert(false, "Allocation size (%d) is too big (max: %d) for pools!", byteSize, m_maxAllocationSize); - return nullptr; - } - - u32 bucketIndex = static_cast((byteSize >> m_minAllocationShift)-1); - BucketType& bucket = m_buckets[bucketIndex]; - PageType* page = nullptr; - if (!bucket.m_pages.empty()) - { - page = &bucket.m_pages.front(); - - // check if we have free slot in the page - if (page->m_freeList.empty()) - { - page = nullptr; - } - else if (page->m_freeList.size()==1) - { - // if we have only 1 free slot this allocation will - // fill the page, so put in on the back - bucket.m_pages.pop_front(); - bucket.m_pages.push_back(*page); - } - } - if (!page) - { - page = m_allocator->PopFreePage(); - if (page) - { - // We have any pages available on free page stack. - if (page->m_bin != bucketIndex) // if this page was used the same bucket we are ready to roll. + if (m_minAllocationSize >> i == 0) { - size_t elementSize = byteSize; - size_t pageDataSize = m_pageSize - sizeof(PageType); - page->SetupFreeList(elementSize, pageDataSize); - page->m_bin = bucketIndex; - page->m_elementSize = static_cast(elementSize); - page->m_maxNumElements = static_cast(pageDataSize / elementSize); + m_minAllocationShift = i - 1; + break; } } - else + + AZ_Assert( + m_maxAllocationSize % m_minAllocationSize == 0, + "You need to be able to divide m_maxAllocationSize (%d) / m_minAllocationSize (%d) without fraction!", m_maxAllocationSize, + m_minAllocationSize); + m_numBuckets = m_maxAllocationSize / m_minAllocationSize; + AZ_Assert(m_numBuckets <= 0xffff, "You can't have more than 65535 number of buckets! We need to increase the index size!"); + m_buckets = reinterpret_cast( + alloc->m_pageAllocator->Allocate(sizeof(BucketType) * m_numBuckets, AZStd::alignment_of::value)); + for (size_t i = 0; i < m_numBuckets; ++i) { - // We need to align each page on it's size, this way we can quickly find which page the pointer belongs to. - page = m_allocator->ConstructPage(byteSize); - page->m_bin = bucketIndex; + new (m_buckets + i) BucketType(); } - bucket.m_pages.push_front(*page); } - // The data address and the fake node address are shared. - void* address = &page->m_freeList.front(); - page->m_freeList.pop_front(); - - m_numBytesAllocated += byteSize; - - return address; -} - -//========================================================================= -// DeAllocate -// [9/09/2009] -//========================================================================= -template -AZ_INLINE void -PoolAllocation::DeAllocate(void* ptr) -{ - PageType* page = m_allocator->PageFromAddress(ptr); - if (page==nullptr) + //========================================================================= + // ~PoolAllocation + // [9/09/2009] + //========================================================================= + template + PoolAllocation::~PoolAllocation() { - AZ_Error("Memory", false, "Address 0x%08x is not in the ThreadPool!", ptr); - return; - } + GarbageCollect(true); - // (pageSize - info struct at the end) / (element size) - size_t maxElementsPerBucket = page->m_maxNumElements; - - size_t numFreeNodes = page->m_freeList.size(); - typename PageType::FakeNode* node = new(ptr) typename PageType::FakeNode(); - page->m_freeList.push_front(*node); - - if (numFreeNodes==0) - { - // if the page was full before sort at the front - BucketType& bucket = m_buckets[page->m_bin]; - bucket.m_pages.erase(*page); - bucket.m_pages.push_front(*page); - } - else if (numFreeNodes == maxElementsPerBucket-1) - { - // push to the list of free pages - BucketType& bucket = m_buckets[page->m_bin]; - PageType* frontPage = &bucket.m_pages.front(); - if (frontPage != page) + for (size_t i = 0; i < m_numBuckets; ++i) { - bucket.m_pages.erase(*page); - // check if the front page is full if so push the free page to the front otherwise push - // push it on the free pages list so it can be reused by other bins. - if (frontPage->m_freeList.empty()) + m_buckets[i].~BucketType(); + } + m_allocator->m_pageAllocator->DeAllocate(m_buckets, sizeof(BucketType) * m_numBuckets); + } + + //========================================================================= + // Allocate + // [9/09/2009] + //========================================================================= + template + AZ_INLINE void* PoolAllocation::Allocate(size_t byteSize, size_t alignment) + { + AZ_Assert(byteSize > 0, "You can not allocate 0 bytes!"); + AZ_Assert(alignment > 0 && (alignment & (alignment - 1)) == 0, "Alignment must be >0 and power of 2!"); + + // pad the size to the min allocation size. + byteSize = AZ::SizeAlignUp(byteSize, m_minAllocationSize); + byteSize = AZ::SizeAlignUp(byteSize, alignment); + + if (byteSize > m_maxAllocationSize) + { + AZ_Assert(false, "Allocation size (%d) is too big (max: %d) for pools!", byteSize, m_maxAllocationSize); + return nullptr; + } + + u32 bucketIndex = static_cast((byteSize >> m_minAllocationShift) - 1); + BucketType& bucket = m_buckets[bucketIndex]; + PageType* page = nullptr; + if (!bucket.m_pages.empty()) + { + page = &bucket.m_pages.front(); + + // check if we have free slot in the page + if (page->m_freeList.empty()) { - bucket.m_pages.push_front(*page); + page = nullptr; + } + else if (page->m_freeList.size() == 1) + { + // if we have only 1 free slot this allocation will + // fill the page, so put in on the back + bucket.m_pages.pop_front(); + bucket.m_pages.push_back(*page); + } + } + if (!page) + { + page = m_allocator->PopFreePage(); + if (page) + { + // We have any pages available on free page stack. + if (page->m_bin != bucketIndex) // if this page was used the same bucket we are ready to roll. + { + size_t elementSize = byteSize; + size_t pageDataSize = m_pageSize - sizeof(PageType); + page->SetupFreeList(elementSize, pageDataSize); + page->m_bin = bucketIndex; + page->m_elementSize = static_cast(elementSize); + page->m_maxNumElements = static_cast(pageDataSize / elementSize); + } } else { - m_allocator->PushFreePage(page); + // We need to align each page on it's size, this way we can quickly find which page the pointer belongs to. + page = m_allocator->ConstructPage(byteSize); + page->m_bin = bucketIndex; } + bucket.m_pages.push_front(*page); } - else if (frontPage->m_next != nullptr) + + // The data address and the fake node address are shared. + void* address = &page->m_freeList.front(); + page->m_freeList.pop_front(); + + m_numBytesAllocated += byteSize; + + return address; + } + + //========================================================================= + // DeAllocate + // [9/09/2009] + //========================================================================= + template + AZ_INLINE void PoolAllocation::DeAllocate(void* ptr) + { + PageType* page = m_allocator->PageFromAddress(ptr); + if (page == nullptr) { - // if the next page has free slots free the current page - if (frontPage->m_next->m_freeList.size() < maxElementsPerBucket) + AZ_Error("Memory", false, "Address 0x%08x is not in the ThreadPool!", ptr); + return; + } + + // (pageSize - info struct at the end) / (element size) + size_t maxElementsPerBucket = page->m_maxNumElements; + + size_t numFreeNodes = page->m_freeList.size(); + typename PageType::FakeNode* node = new (ptr) typename PageType::FakeNode(); + page->m_freeList.push_front(*node); + + if (numFreeNodes == 0) + { + // if the page was full before sort at the front + BucketType& bucket = m_buckets[page->m_bin]; + bucket.m_pages.erase(*page); + bucket.m_pages.push_front(*page); + } + else if (numFreeNodes == maxElementsPerBucket - 1) + { + // push to the list of free pages + BucketType& bucket = m_buckets[page->m_bin]; + PageType* frontPage = &bucket.m_pages.front(); + if (frontPage != page) { bucket.m_pages.erase(*page); - m_allocator->PushFreePage(page); - } - } - } - - m_numBytesAllocated -= page->m_elementSize; -} - -//========================================================================= -// AllocationSize -// [11/22/2010] -//========================================================================= -template -AZ_INLINE size_t -PoolAllocation::AllocationSize(void* ptr) -{ - PageType* page = m_allocator->PageFromAddress(ptr); - size_t elementSize; - if (page) - { - elementSize = page->m_elementSize; - } - else - { - elementSize = 0; - } - - return elementSize; -} - -//========================================================================= -// GarbageCollect -// [3/1/2012] -//========================================================================= -template -AZ_INLINE void -PoolAllocation::GarbageCollect(bool isForceFreeAllPages) -{ - // Free empty pages in the buckets (or better be empty) - for (unsigned int i = 0; i < (unsigned int)m_numBuckets; ++i) - { - // (pageSize - info struct at the end) / (element size) - size_t maxElementsPerBucket = (m_pageSize - sizeof(PageType)) / ((i+1) << m_minAllocationShift); - - typename BucketType::PageListType& pages = m_buckets[i].m_pages; - while (!pages.empty()) - { - PageType& page = pages.front(); - pages.pop_front(); - if (page.m_freeList.size()==maxElementsPerBucket || isForceFreeAllPages) - { - if (!m_allocator->IsInStaticBlock(&page)) + // check if the front page is full if so push the free page to the front otherwise push + // push it on the free pages list so it can be reused by other bins. + if (frontPage->m_freeList.empty()) { - m_allocator->FreePage(&page); + bucket.m_pages.push_front(*page); } else { - m_allocator->PushFreePage(&page); + m_allocator->PushFreePage(page); + } + } + else if (frontPage->m_next != nullptr) + { + // if the next page has free slots free the current page + if (frontPage->m_next->m_freeList.size() < maxElementsPerBucket) + { + bucket.m_pages.erase(*page); + m_allocator->PushFreePage(page); + } + } + } + + m_numBytesAllocated -= page->m_elementSize; + } + + //========================================================================= + // AllocationSize + // [11/22/2010] + //========================================================================= + template + AZ_INLINE size_t PoolAllocation::AllocationSize(void* ptr) + { + PageType* page = m_allocator->PageFromAddress(ptr); + size_t elementSize; + if (page) + { + elementSize = page->m_elementSize; + } + else + { + elementSize = 0; + } + + return elementSize; + } + + //========================================================================= + // GarbageCollect + // [3/1/2012] + //========================================================================= + template + AZ_INLINE void PoolAllocation::GarbageCollect(bool isForceFreeAllPages) + { + // Free empty pages in the buckets (or better be empty) + for (unsigned int i = 0; i < (unsigned int)m_numBuckets; ++i) + { + // (pageSize - info struct at the end) / (element size) + size_t maxElementsPerBucket = (m_pageSize - sizeof(PageType)) / ((i + 1) << m_minAllocationShift); + + typename BucketType::PageListType& pages = m_buckets[i].m_pages; + while (!pages.empty()) + { + PageType& page = pages.front(); + pages.pop_front(); + if (page.m_freeList.size() == maxElementsPerBucket || isForceFreeAllPages) + { + if (!m_allocator->IsInStaticBlock(&page)) + { + m_allocator->FreePage(&page); + } + else + { + m_allocator->PushFreePage(&page); + } } } } } -} -////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////// -// PollAllocator -////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // PollAllocator + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// -//========================================================================= -// PoolSchema -// [9/15/2009] -//========================================================================= -PoolSchema::PoolSchema(const Descriptor& desc) - : m_impl(nullptr) -{ - (void)desc; // ignored here, applied in Create() -} - -//========================================================================= -// ~PoolSchema -// [9/15/2009] -//========================================================================= -PoolSchema::~PoolSchema() -{ - AZ_Assert(m_impl==nullptr, "You did not destroy the pool schema!"); - delete m_impl; -} - -//========================================================================= -// Create -// [9/15/2009] -//========================================================================= -bool PoolSchema::Create(const Descriptor& desc) -{ - AZ_Assert(m_impl==nullptr, "PoolSchema already created!"); - if (m_impl == nullptr) + //========================================================================= + // PoolSchema + // [9/15/2009] + //========================================================================= + PoolSchema::PoolSchema(const Descriptor& desc) + : m_impl(nullptr) { - m_impl = aznew PoolSchemaImpl(desc); + (void)desc; // ignored here, applied in Create() } - return (m_impl!=nullptr); -} -//========================================================================= -// ~Destroy -// [9/15/2009] -//========================================================================= -bool PoolSchema::Destroy() -{ - delete m_impl; - m_impl = nullptr; - return true; -} - -//========================================================================= -// Allocate -// [9/15/2009] -//========================================================================= -PoolSchema::pointer_type -PoolSchema::Allocate(size_type byteSize, size_type alignment, int flags, const char* name, const char* fileName, int lineNum, unsigned int suppressStackRecord) -{ - (void)flags; - (void)name; - (void)fileName; - (void)lineNum; - (void)suppressStackRecord; - return m_impl->Allocate(byteSize, alignment); -} - -//========================================================================= -// DeAllocate -// [9/15/2009] -//========================================================================= -void -PoolSchema::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) -{ - (void)byteSize; - (void)alignment; - m_impl->DeAllocate(ptr); -} - -//========================================================================= -// Resize -// [10/14/2018] -//========================================================================= -PoolSchema::size_type -PoolSchema::Resize(pointer_type ptr, size_type newSize) -{ - (void)ptr; - (void)newSize; - return 0; // unsupported -} - -//========================================================================= -// ReAllocate -// [10/14/2018] -//========================================================================= -PoolSchema::pointer_type -PoolSchema::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) -{ - (void)ptr; - (void)newSize; - (void)newAlignment; - AZ_Assert(false, "unsupported"); - - return ptr; -} - -//========================================================================= -// AllocationSize -// [11/22/2010] -//========================================================================= -PoolSchema::size_type -PoolSchema::AllocationSize(pointer_type ptr) -{ - return m_impl->AllocationSize(ptr); -} - -//========================================================================= -// DeAllocate -// [9/15/2009] -//========================================================================= -void -PoolSchema::GarbageCollect() -{ - // External requests for garbage collection may come from any thread, and the - // garbage collection operation isn't threadsafe, which can lead to crashes. - // - // Due to the low memory consumption of this allocator in practice on Dragonfly - // (~3kb) it makes sense to not bother with garbage collection and leave it to - // occur exclusively in the destruction of the allocator. - // - // TODO: A better solution needs to be found for integrating back into mainline - // Open 3D Engine. - //m_impl->GarbageCollect(); -} - -auto PoolSchema::GetMaxContiguousAllocationSize() const -> size_type -{ - return m_impl->m_allocator.m_maxAllocationSize; -} - -//========================================================================= -// NumAllocatedBytes -// [11/1/2010] -//========================================================================= -PoolSchema::size_type -PoolSchema::NumAllocatedBytes() const -{ - return m_impl->m_allocator.m_numBytesAllocated; -} - -//========================================================================= -// Capacity -// [11/1/2010] -//========================================================================= -PoolSchema::size_type -PoolSchema::Capacity() const -{ - return m_impl->m_numStaticPages * m_impl->m_pageSize; -} - -//========================================================================= -// GetPageAllocator -// [11/17/2010] -//========================================================================= -IAllocatorAllocate* -PoolSchema::GetSubAllocator() -{ - return m_impl->m_pageAllocator; -} - - -////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////// -// PollAllocator Implementation -////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////// - -//========================================================================= -// PoolSchemaImpl -// [9/15/2009] -//========================================================================= -PoolSchemaImpl::PoolSchemaImpl(const PoolSchema::Descriptor& desc) - : m_pageAllocator(desc.m_pageAllocator ? desc.m_pageAllocator : &AllocatorInstance::Get()) - , m_allocator(this, desc.m_pageSize, desc.m_minAllocationSize, desc.m_maxAllocationSize) - , m_staticDataBlock(nullptr) - , m_numStaticPages(desc.m_numStaticPages) - , m_isDynamic(desc.m_isDynamic) - , m_pageSize(desc.m_pageSize) -{ - if (m_numStaticPages) + //========================================================================= + // ~PoolSchema + // [9/15/2009] + //========================================================================= + PoolSchema::~PoolSchema() { - // We store the page struct at the end of the block - char* memBlock = reinterpret_cast(m_pageAllocator->Allocate(m_pageSize*m_numStaticPages, m_pageSize, 0, "AZSystem::PoolAllocation::Page static array", __FILE__, __LINE__)); - m_staticDataBlock = memBlock; - size_t pageDataSize = m_pageSize - sizeof(Page); - for (unsigned int i = 0; i < m_numStaticPages; ++i) + AZ_Assert(m_impl == nullptr, "You did not destroy the pool schema!"); + delete m_impl; + } + + //========================================================================= + // Create + // [9/15/2009] + //========================================================================= + bool PoolSchema::Create(const Descriptor& desc) + { + AZ_Assert(m_impl == nullptr, "PoolSchema already created!"); + if (m_impl == nullptr) { - Page* page = new(memBlock+pageDataSize)Page(); - page->m_bin = 0xffffffff; - page->m_elementSize = 0; - page->m_maxNumElements = 0; - PushFreePage(page); - memBlock += m_pageSize; + m_impl = aznew PoolSchemaImpl(desc); + } + return (m_impl != nullptr); + } + + //========================================================================= + // ~Destroy + // [9/15/2009] + //========================================================================= + bool PoolSchema::Destroy() + { + delete m_impl; + m_impl = nullptr; + return true; + } + + //========================================================================= + // Allocate + // [9/15/2009] + //========================================================================= + PoolSchema::pointer_type PoolSchema::Allocate( + size_type byteSize, + size_type alignment, + int flags, + const char* name, + const char* fileName, + int lineNum, + unsigned int suppressStackRecord) + { + (void)flags; + (void)name; + (void)fileName; + (void)lineNum; + (void)suppressStackRecord; + return m_impl->Allocate(byteSize, alignment); + } + + //========================================================================= + // DeAllocate + // [9/15/2009] + //========================================================================= + void PoolSchema::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) + { + (void)byteSize; + (void)alignment; + m_impl->DeAllocate(ptr); + } + + //========================================================================= + // Resize + // [10/14/2018] + //========================================================================= + PoolSchema::size_type PoolSchema::Resize(pointer_type ptr, size_type newSize) + { + (void)ptr; + (void)newSize; + return 0; // unsupported + } + + //========================================================================= + // ReAllocate + // [10/14/2018] + //========================================================================= + PoolSchema::pointer_type PoolSchema::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) + { + (void)ptr; + (void)newSize; + (void)newAlignment; + AZ_Assert(false, "unsupported"); + + return ptr; + } + + //========================================================================= + // AllocationSize + // [11/22/2010] + //========================================================================= + PoolSchema::size_type PoolSchema::AllocationSize(pointer_type ptr) + { + return m_impl->AllocationSize(ptr); + } + + //========================================================================= + // DeAllocate + // [9/15/2009] + //========================================================================= + void PoolSchema::GarbageCollect() + { + // External requests for garbage collection may come from any thread, and the + // garbage collection operation isn't threadsafe, which can lead to crashes. + // + // Due to the low memory consumption of this allocator in practice on Dragonfly + // (~3kb) it makes sense to not bother with garbage collection and leave it to + // occur exclusively in the destruction of the allocator. + // + // TODO: A better solution needs to be found for integrating back into mainline + // Open 3D Engine. + // m_impl->GarbageCollect(); + } + + auto PoolSchema::GetMaxContiguousAllocationSize() const -> size_type + { + return m_impl->m_allocator.m_maxAllocationSize; + } + + //========================================================================= + // NumAllocatedBytes + // [11/1/2010] + //========================================================================= + PoolSchema::size_type PoolSchema::NumAllocatedBytes() const + { + return m_impl->m_allocator.m_numBytesAllocated; + } + + //========================================================================= + // Capacity + // [11/1/2010] + //========================================================================= + PoolSchema::size_type PoolSchema::Capacity() const + { + return m_impl->m_numStaticPages * m_impl->m_pageSize; + } + + //========================================================================= + // GetPageAllocator + // [11/17/2010] + //========================================================================= + IAllocatorAllocate* PoolSchema::GetSubAllocator() + { + return m_impl->m_pageAllocator; + } + + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // PollAllocator Implementation + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + + //========================================================================= + // PoolSchemaImpl + // [9/15/2009] + //========================================================================= + PoolSchemaImpl::PoolSchemaImpl(const PoolSchema::Descriptor& desc) + : m_pageAllocator(desc.m_pageAllocator ? desc.m_pageAllocator : &AllocatorInstance::Get()) + , m_allocator(this, desc.m_pageSize, desc.m_minAllocationSize, desc.m_maxAllocationSize) + , m_staticDataBlock(nullptr) + , m_numStaticPages(desc.m_numStaticPages) + , m_isDynamic(desc.m_isDynamic) + , m_pageSize(desc.m_pageSize) + { + if (m_numStaticPages) + { + // We store the page struct at the end of the block + char* memBlock = reinterpret_cast(m_pageAllocator->Allocate( + m_pageSize * m_numStaticPages, m_pageSize, 0, "AZSystem::PoolAllocation::Page static array", __FILE__, __LINE__)); + m_staticDataBlock = memBlock; + size_t pageDataSize = m_pageSize - sizeof(Page); + for (unsigned int i = 0; i < m_numStaticPages; ++i) + { + Page* page = new (memBlock + pageDataSize) Page(); + page->m_bin = 0xffffffff; + page->m_elementSize = 0; + page->m_maxNumElements = 0; + PushFreePage(page); + memBlock += m_pageSize; + } } } -} -//========================================================================= -// ~PoolSchemaImpl -// [9/15/2009] -//========================================================================= -PoolSchemaImpl::~PoolSchemaImpl() -{ - // Force free all pages - m_allocator.GarbageCollect(true); - - // Free all unused memory - GarbageCollect(); - - if (m_staticDataBlock) + //========================================================================= + // ~PoolSchemaImpl + // [9/15/2009] + //========================================================================= + PoolSchemaImpl::~PoolSchemaImpl() { - while (!m_freePages.empty()) + // Force free all pages + m_allocator.GarbageCollect(true); + + // Free all unused memory + GarbageCollect(); + + if (m_staticDataBlock) { - Page* page = &m_freePages.front(); - (void)page; - m_freePages.pop_front(); - AZ_Assert(IsInStaticBlock(page), "All dynamic pages should be deleted by now!"); - } - ; - - char* memBlock = reinterpret_cast(m_staticDataBlock); - size_t pageDataSize = m_pageSize - sizeof(Page); - for (unsigned int i = 0; i < m_numStaticPages; ++i) - { - Page* page = reinterpret_cast(memBlock+pageDataSize); - page->~Page(); - memBlock += m_pageSize; - } - m_pageAllocator->DeAllocate(m_staticDataBlock); - } -} - -//========================================================================= -// Allocate -// [9/15/2009] -//========================================================================= -PoolSchema::pointer_type -PoolSchemaImpl::Allocate(PoolSchema::size_type byteSize, PoolSchema::size_type alignment, int flags) -{ - //AZ_Warning("Memory",m_ownerThread==AZStd::this_thread::get_id(),"You can't allocation from a different context/thread, use ThreadPoolAllocator!"); - (void)flags; - void* address = m_allocator.Allocate(byteSize, alignment); - return address; -} - -//========================================================================= -// DeAllocate -// [9/15/2009] -//========================================================================= -void -PoolSchemaImpl::DeAllocate(PoolSchema::pointer_type ptr) -{ - //AZ_Warning("Memory",m_ownerThread==AZStd::this_thread::get_id(),"You can't deallocate from a different context/thread, use ThreadPoolAllocator!"); - m_allocator.DeAllocate(ptr); -} - -//========================================================================= -// AllocationSize -// [11/22/2010] -//========================================================================= -PoolSchema::size_type -PoolSchemaImpl::AllocationSize(PoolSchema::pointer_type ptr) -{ - //AZ_Warning("Memory",m_ownerThread==AZStd::this_thread::get_id(),"You can't use PoolAllocator from a different context/thread, use ThreadPoolAllocator!"); - return m_allocator.AllocationSize(ptr); -} - -//========================================================================= -// Pop -// [9/15/2009] -//========================================================================= -AZ_FORCE_INLINE PoolSchemaImpl::Page* -PoolSchemaImpl::PopFreePage() -{ - Page* page = nullptr; - if (!m_freePages.empty()) - { - page = &m_freePages.front(); - m_freePages.pop_front(); - } - return page; -} - -//========================================================================= -// Push -// [9/15/2009] -//========================================================================= -AZ_INLINE void -PoolSchemaImpl::PushFreePage(Page* page) -{ - m_freePages.push_front(*page); -} - -//========================================================================= -// PurgePages -// [9/11/2009] -//========================================================================= -void -PoolSchemaImpl::GarbageCollect() -{ - //if( m_ownerThread == AZStd::this_thread::get_id() ) - { - if (m_isDynamic) - { - m_allocator.GarbageCollect(); - - Bucket::PageListType staticPages; while (!m_freePages.empty()) { Page* page = &m_freePages.front(); + (void)page; m_freePages.pop_front(); - if (IsInStaticBlock(page)) - { - staticPages.push_front(*page); - } - else - { - FreePage(page); - } - } + AZ_Assert(IsInStaticBlock(page), "All dynamic pages should be deleted by now!"); + }; - while (!staticPages.empty()) + char* memBlock = reinterpret_cast(m_staticDataBlock); + size_t pageDataSize = m_pageSize - sizeof(Page); + for (unsigned int i = 0; i < m_numStaticPages; ++i) { - Page* page = &staticPages.front(); - staticPages.pop_front(); - m_freePages.push_front(*page); + Page* page = reinterpret_cast(memBlock + pageDataSize); + page->~Page(); + memBlock += m_pageSize; } - } - } -} - -//========================================================================= -// SetupFreeList -// [9/09/2009] -//========================================================================= -AZ_FORCE_INLINE void -PoolSchemaImpl::Page::SetupFreeList(size_t elementSize, size_t pageDataBlockSize) -{ - char* pageData = reinterpret_cast(this) - pageDataBlockSize; - m_freeList.clear(); - // setup free list - size_t numElements = pageDataBlockSize / elementSize; - for (unsigned int i = 0; i < numElements; ++i) - { - char* address = pageData+i*elementSize; - Page::FakeNode* node = new(address) Page::FakeNode(); - m_freeList.push_back(*node); - } -} - -////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////// -// ThreadPoolSchema -////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////// - -//========================================================================= -// ThreadPoolSchema -// [9/15/2009] -//========================================================================= -ThreadPoolSchema::ThreadPoolSchema(GetThreadPoolData getThreadPoolData, SetThreadPoolData setThreadPoolData) - : m_impl(nullptr) - , m_threadPoolGetter(getThreadPoolData) - , m_threadPoolSetter(setThreadPoolData) -{ -} - -//========================================================================= -// ~ThreadPoolSchema -// [9/15/2009] -//========================================================================= -ThreadPoolSchema::~ThreadPoolSchema() -{ - AZ_Assert(m_impl==nullptr, "You did not destroy the thread pool schema!"); - delete m_impl; -} - -//========================================================================= -// Create -// [9/15/2009] -//========================================================================= -bool ThreadPoolSchema::Create(const Descriptor& desc) -{ - AZ_Assert(m_impl==nullptr, "PoolSchema already created!"); - if (m_impl == nullptr) - { - m_impl = aznew ThreadPoolSchemaImpl(desc, m_threadPoolGetter, m_threadPoolSetter); - } - return (m_impl!=nullptr); -} - -//========================================================================= -// Destroy -// [9/15/2009] -//========================================================================= -bool ThreadPoolSchema::Destroy() -{ - delete m_impl; - m_impl = nullptr; - return true; -} -//========================================================================= -// Allocate -// [9/15/2009] -//========================================================================= -ThreadPoolSchema::pointer_type -ThreadPoolSchema::Allocate(size_type byteSize, size_type alignment, int flags, const char* name, const char* fileName, int lineNum, unsigned int suppressStackRecord) -{ - (void)flags; - (void)name; - (void)fileName; - (void)lineNum; - (void)suppressStackRecord; - return m_impl->Allocate(byteSize, alignment); -} - -//========================================================================= -// DeAllocate -// [9/15/2009] -//========================================================================= -void -ThreadPoolSchema::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) -{ - (void)byteSize; - (void)alignment; - m_impl->DeAllocate(ptr); -} - -//========================================================================= -// Resize -// [10/14/2018] -//========================================================================= -ThreadPoolSchema::size_type -ThreadPoolSchema::Resize(pointer_type ptr, size_type newSize) -{ - (void)ptr; - (void)newSize; - return 0; // unsupported -} - -//========================================================================= -// ReAllocate -// [10/14/2018] -//========================================================================= -ThreadPoolSchema::pointer_type -ThreadPoolSchema::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) -{ - (void)ptr; - (void)newSize; - (void)newAlignment; - AZ_Assert(false, "unsupported"); - - return ptr; -} - -//========================================================================= -// AllocationSize -// [11/22/2010] -//========================================================================= -ThreadPoolSchema::size_type -ThreadPoolSchema::AllocationSize(pointer_type ptr) -{ - return m_impl->AllocationSize(ptr); -} - -//========================================================================= -// DeAllocate -// [9/15/2009] -//========================================================================= -void -ThreadPoolSchema::GarbageCollect() -{ - m_impl->GarbageCollect(); -} - -auto ThreadPoolSchema::GetMaxContiguousAllocationSize() const -> size_type -{ - return m_impl->m_maxAllocationSize; -} - -//========================================================================= -// NumAllocatedBytes -// [11/1/2010] -//========================================================================= -ThreadPoolSchema::size_type -ThreadPoolSchema::NumAllocatedBytes() const -{ - size_type bytesAllocated = 0; - { - AZStd::lock_guard lock(m_impl->m_mutex); - for (size_t i = 0; i < m_impl->m_threads.size(); ++i) - { - bytesAllocated += m_impl->m_threads[i]->m_allocator.m_numBytesAllocated; - } - } - return bytesAllocated; -} - -//========================================================================= -// Capacity -// [11/1/2010] -//========================================================================= -ThreadPoolSchema::size_type -ThreadPoolSchema::Capacity() const -{ - return m_impl->m_numStaticPages * m_impl->m_pageSize; -} - -//========================================================================= -// GetPageAllocator -// [11/17/2010] -//========================================================================= -IAllocatorAllocate* -ThreadPoolSchema::GetSubAllocator() -{ - return m_impl->m_pageAllocator; -} - - -//========================================================================= -// ThreadPoolSchemaImpl -// [9/15/2009] -//========================================================================= -ThreadPoolSchemaImpl::ThreadPoolSchemaImpl(const ThreadPoolSchema::Descriptor& desc, ThreadPoolSchema::GetThreadPoolData threadPoolGetter, ThreadPoolSchema::SetThreadPoolData threadPoolSetter) - : m_threadPoolGetter(threadPoolGetter) - , m_threadPoolSetter(threadPoolSetter) - , m_pageAllocator(desc.m_pageAllocator) - , m_staticDataBlock(nullptr) - , m_numStaticPages(desc.m_numStaticPages) - , m_pageSize(desc.m_pageSize) - , m_minAllocationSize(desc.m_minAllocationSize) - , m_maxAllocationSize(desc.m_maxAllocationSize) - , m_isDynamic(desc.m_isDynamic) -{ -# if AZ_TRAIT_OS_HAS_CRITICAL_SECTION_SPIN_COUNT - // In memory allocation case (usually tools) we might have high contention, - // using spin lock will improve performance. - SetCriticalSectionSpinCount(m_mutex.native_handle(), 4000); -# endif - - if (m_pageAllocator == nullptr) - { - m_pageAllocator = &AllocatorInstance::Get(); // use the SystemAllocator if no page allocator is provided - } - if (m_numStaticPages) - { - // We store the page struct at the end of the block - char* memBlock = reinterpret_cast(m_pageAllocator->Allocate(m_pageSize*m_numStaticPages, m_pageSize, 0, "AZSystem::ThreadPoolSchemaImpl::Page static array", __FILE__, __LINE__)); - m_staticDataBlock = memBlock; - size_t pageDataSize = m_pageSize - sizeof(Page); - for (unsigned int i = 0; i < m_numStaticPages; ++i) - { - Page* page = new(memBlock+pageDataSize)Page(m_threadPoolGetter()); - page->m_bin = 0xffffffff; - PushFreePage(page); - memBlock += m_pageSize; - } - } -} - -//========================================================================= -// ~ThreadPoolSchemaImpl -// [9/15/2009] -//========================================================================= -ThreadPoolSchemaImpl::~ThreadPoolSchemaImpl() -{ - // clean up all the thread data. - // IMPORTANT: We assume/rely that all threads (except the calling one) are or will - // destroyed before you create another instance of the pool allocation. - // This should generally be ok since the all allocators are singletons. - { - AZStd::lock_guard lock(m_mutex); - if (!m_threads.empty()) - { - for (size_t i = 0; i < m_threads.size(); ++i) - { - if (m_threads[i]) - { - // Force free all pages - delete m_threads[i]; - } - } - - /// reset the variable for the owner thread. - m_threadPoolSetter(nullptr); + m_pageAllocator->DeAllocate(m_staticDataBlock); } } - GarbageCollect(); - - if (m_staticDataBlock) + //========================================================================= + // Allocate + // [9/15/2009] + //========================================================================= + PoolSchema::pointer_type PoolSchemaImpl::Allocate(PoolSchema::size_type byteSize, PoolSchema::size_type alignment, int flags) { - Page* page; - { - AZStd::lock_guard lock(m_mutex); - while (!m_freePages.empty()) - { - page = &m_freePages.front(); - m_freePages.pop_front(); - AZ_Assert(IsInStaticBlock(page), "All dynamic pages should be free by now!"); - } - } - - char* memBlock = reinterpret_cast(m_staticDataBlock); - size_t pageDataSize = m_pageSize - sizeof(Page); - for (unsigned int i = 0; i < m_numStaticPages; ++i) - { - page = reinterpret_cast(memBlock+pageDataSize); - page->~Page(); - memBlock += m_pageSize; - } - m_pageAllocator->DeAllocate(m_staticDataBlock); - } -} - -//========================================================================= -// Allocate -// [9/15/2009] -//========================================================================= -ThreadPoolSchema::pointer_type -ThreadPoolSchemaImpl::Allocate(ThreadPoolSchema::size_type byteSize, ThreadPoolSchema::size_type alignment, int flags) -{ - (void)flags; - - ThreadPoolData* threadData = m_threadPoolGetter(); - - if (threadData == nullptr) - { - threadData = aznew ThreadPoolData(this, m_pageSize, m_minAllocationSize, m_maxAllocationSize); - m_threadPoolSetter(threadData); - { - AZStd::lock_guard lock(m_mutex); - m_threads.push_back(threadData); - } - } - else - { - // deallocate elements if they were freed from other threads - Page::FakeNodeLF* fakeLFNode; - while ((fakeLFNode = threadData->m_freedElements.pop())!=nullptr) - { - threadData->m_allocator.DeAllocate(fakeLFNode); - } + // AZ_Warning("Memory",m_ownerThread==AZStd::this_thread::get_id(),"You can't allocation from a different context/thread, use + // ThreadPoolAllocator!"); + (void)flags; + void* address = m_allocator.Allocate(byteSize, alignment); + return address; } - return threadData->m_allocator.Allocate(byteSize, alignment); -} + //========================================================================= + // DeAllocate + // [9/15/2009] + //========================================================================= + void PoolSchemaImpl::DeAllocate(PoolSchema::pointer_type ptr) + { + // AZ_Warning("Memory",m_ownerThread==AZStd::this_thread::get_id(),"You can't deallocate from a different context/thread, use + // ThreadPoolAllocator!"); + m_allocator.DeAllocate(ptr); + } -//========================================================================= -// DeAllocate -// [9/15/2009] -//========================================================================= -void -ThreadPoolSchemaImpl::DeAllocate(ThreadPoolSchema::pointer_type ptr) -{ - Page* page = PageFromAddress(ptr); - if (page==nullptr) + //========================================================================= + // AllocationSize + // [11/22/2010] + //========================================================================= + PoolSchema::size_type PoolSchemaImpl::AllocationSize(PoolSchema::pointer_type ptr) { - AZ_Error("Memory", false, "Address 0x%08x is not in the ThreadPool!", ptr); - return; + // AZ_Warning("Memory",m_ownerThread==AZStd::this_thread::get_id(),"You can't use PoolAllocator from a different context/thread, use + // ThreadPoolAllocator!"); + return m_allocator.AllocationSize(ptr); } - AZ_Assert(page->m_threadData!=nullptr, ("We must have valid page thread data for the page!")); - ThreadPoolData* threadData = m_threadPoolGetter(); - if (threadData == page->m_threadData) - { - // we can free here - threadData->m_allocator.DeAllocate(ptr); - } - else - { - // push this element to be deleted from it's own thread! - // cast the pointer to a fake lock free node - Page::FakeNodeLF* fakeLFNode = reinterpret_cast(ptr); -#ifdef AZ_DEBUG_BUILD - // we need to reset the fakeLFNode because we share the memory. - // otherwise we will assert the node is in the list - fakeLFNode->m_next = 0; -#endif - page->m_threadData->m_freedElements.push(*fakeLFNode); - } -} -//========================================================================= -// AllocationSize -// [11/22/2010] -//========================================================================= -ThreadPoolSchema::size_type -ThreadPoolSchemaImpl::AllocationSize(ThreadPoolSchema::pointer_type ptr) -{ - Page* page = PageFromAddress(ptr); - if (page==nullptr) + //========================================================================= + // Pop + // [9/15/2009] + //========================================================================= + AZ_FORCE_INLINE PoolSchemaImpl::Page* PoolSchemaImpl::PopFreePage() { - return 0; - } - AZ_Assert(page->m_threadData!=nullptr, ("We must have valid page thread data for the page!")); - return page->m_threadData->m_allocator.AllocationSize(ptr); -} - -//========================================================================= -// PopFreePage -// [9/15/2009] -//========================================================================= -AZ_INLINE ThreadPoolSchemaImpl::Page* -ThreadPoolSchemaImpl::PopFreePage() -{ - Page* page; - { - AZStd::lock_guard lock(m_mutex); - if (m_freePages.empty()) - { - page = nullptr; - } - else + Page* page = nullptr; + if (!m_freePages.empty()) { page = &m_freePages.front(); m_freePages.pop_front(); } + return page; } - if (page) - { -# ifdef AZ_DEBUG_BUILD - AZ_Assert(page->m_threadData == 0, "If we stored the free page properly we should have null here!"); -# endif - // store the current thread data, used when we free elements - page->m_threadData = m_threadPoolGetter(); - } - return page; -} -//========================================================================= -// PushFreePage -// [9/15/2009] -//========================================================================= -AZ_INLINE void -ThreadPoolSchemaImpl::PushFreePage(Page* page) -{ -#ifdef AZ_DEBUG_BUILD - page->m_threadData = 0; -#endif + //========================================================================= + // Push + // [9/15/2009] + //========================================================================= + AZ_INLINE void PoolSchemaImpl::PushFreePage(Page* page) { - AZStd::lock_guard lock(m_mutex); m_freePages.push_front(*page); } -} -//========================================================================= -// GarbageCollect -// [9/15/2009] -//========================================================================= -void -ThreadPoolSchemaImpl::GarbageCollect() -{ - if (!m_isDynamic) + //========================================================================= + // PurgePages + // [9/11/2009] + //========================================================================= + void PoolSchemaImpl::GarbageCollect() { - return; // we have the memory statically allocated, can't collect garbage. + // if( m_ownerThread == AZStd::this_thread::get_id() ) + { + if (m_isDynamic) + { + m_allocator.GarbageCollect(); + + Bucket::PageListType staticPages; + while (!m_freePages.empty()) + { + Page* page = &m_freePages.front(); + m_freePages.pop_front(); + if (IsInStaticBlock(page)) + { + staticPages.push_front(*page); + } + else + { + FreePage(page); + } + } + + while (!staticPages.empty()) + { + Page* page = &staticPages.front(); + staticPages.pop_front(); + m_freePages.push_front(*page); + } + } + } } - FreePagesType staticPages; - AZStd::lock_guard lock(m_mutex); - while (!m_freePages.empty()) + //========================================================================= + // SetupFreeList + // [9/09/2009] + //========================================================================= + AZ_FORCE_INLINE void PoolSchemaImpl::Page::SetupFreeList(size_t elementSize, size_t pageDataBlockSize) { - Page* page = &m_freePages.front(); - m_freePages.pop_front(); - if (IsInStaticBlock(page)) + char* pageData = reinterpret_cast(this) - pageDataBlockSize; + m_freeList.clear(); + // setup free list + size_t numElements = pageDataBlockSize / elementSize; + for (unsigned int i = 0; i < numElements; ++i) { - staticPages.push_front(*page); + char* address = pageData + i * elementSize; + Page::FakeNode* node = new (address) Page::FakeNode(); + m_freeList.push_back(*node); + } + } + + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // ThreadPoolSchema + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + + //========================================================================= + // ThreadPoolSchema + // [9/15/2009] + //========================================================================= + ThreadPoolSchema::ThreadPoolSchema(GetThreadPoolData getThreadPoolData, SetThreadPoolData setThreadPoolData) + : m_impl(nullptr) + , m_threadPoolGetter(getThreadPoolData) + , m_threadPoolSetter(setThreadPoolData) + { + } + + //========================================================================= + // ~ThreadPoolSchema + // [9/15/2009] + //========================================================================= + ThreadPoolSchema::~ThreadPoolSchema() + { + AZ_Assert(m_impl == nullptr, "You did not destroy the thread pool schema!"); + delete m_impl; + } + + //========================================================================= + // Create + // [9/15/2009] + //========================================================================= + bool ThreadPoolSchema::Create(const Descriptor& desc) + { + AZ_Assert(m_impl == nullptr, "PoolSchema already created!"); + if (m_impl == nullptr) + { + m_impl = aznew ThreadPoolSchemaImpl(desc, m_threadPoolGetter, m_threadPoolSetter); + } + return (m_impl != nullptr); + } + + //========================================================================= + // Destroy + // [9/15/2009] + //========================================================================= + bool ThreadPoolSchema::Destroy() + { + delete m_impl; + m_impl = nullptr; + return true; + } + //========================================================================= + // Allocate + // [9/15/2009] + //========================================================================= + ThreadPoolSchema::pointer_type ThreadPoolSchema::Allocate( + size_type byteSize, + size_type alignment, + int flags, + const char* name, + const char* fileName, + int lineNum, + unsigned int suppressStackRecord) + { + (void)flags; + (void)name; + (void)fileName; + (void)lineNum; + (void)suppressStackRecord; + return m_impl->Allocate(byteSize, alignment); + } + + //========================================================================= + // DeAllocate + // [9/15/2009] + //========================================================================= + void ThreadPoolSchema::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) + { + (void)byteSize; + (void)alignment; + m_impl->DeAllocate(ptr); + } + + //========================================================================= + // Resize + // [10/14/2018] + //========================================================================= + ThreadPoolSchema::size_type ThreadPoolSchema::Resize(pointer_type ptr, size_type newSize) + { + (void)ptr; + (void)newSize; + return 0; // unsupported + } + + //========================================================================= + // ReAllocate + // [10/14/2018] + //========================================================================= + ThreadPoolSchema::pointer_type ThreadPoolSchema::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) + { + (void)ptr; + (void)newSize; + (void)newAlignment; + AZ_Assert(false, "unsupported"); + + return ptr; + } + + //========================================================================= + // AllocationSize + // [11/22/2010] + //========================================================================= + ThreadPoolSchema::size_type ThreadPoolSchema::AllocationSize(pointer_type ptr) + { + return m_impl->AllocationSize(ptr); + } + + //========================================================================= + // DeAllocate + // [9/15/2009] + //========================================================================= + void ThreadPoolSchema::GarbageCollect() + { + m_impl->GarbageCollect(); + } + + auto ThreadPoolSchema::GetMaxContiguousAllocationSize() const -> size_type + { + return m_impl->m_maxAllocationSize; + } + + //========================================================================= + // NumAllocatedBytes + // [11/1/2010] + //========================================================================= + ThreadPoolSchema::size_type ThreadPoolSchema::NumAllocatedBytes() const + { + size_type bytesAllocated = 0; + { + AZStd::lock_guard lock(m_impl->m_mutex); + for (size_t i = 0; i < m_impl->m_threads.size(); ++i) + { + bytesAllocated += m_impl->m_threads[i]->m_allocator.m_numBytesAllocated; + } + } + return bytesAllocated; + } + + //========================================================================= + // Capacity + // [11/1/2010] + //========================================================================= + ThreadPoolSchema::size_type ThreadPoolSchema::Capacity() const + { + return m_impl->m_numStaticPages * m_impl->m_pageSize; + } + + //========================================================================= + // GetPageAllocator + // [11/17/2010] + //========================================================================= + IAllocatorAllocate* ThreadPoolSchema::GetSubAllocator() + { + return m_impl->m_pageAllocator; + } + + //========================================================================= + // ThreadPoolSchemaImpl + // [9/15/2009] + //========================================================================= + ThreadPoolSchemaImpl::ThreadPoolSchemaImpl( + const ThreadPoolSchema::Descriptor& desc, + ThreadPoolSchema::GetThreadPoolData threadPoolGetter, + ThreadPoolSchema::SetThreadPoolData threadPoolSetter) + : m_threadPoolGetter(threadPoolGetter) + , m_threadPoolSetter(threadPoolSetter) + , m_pageAllocator(desc.m_pageAllocator) + , m_staticDataBlock(nullptr) + , m_numStaticPages(desc.m_numStaticPages) + , m_pageSize(desc.m_pageSize) + , m_minAllocationSize(desc.m_minAllocationSize) + , m_maxAllocationSize(desc.m_maxAllocationSize) + , m_isDynamic(desc.m_isDynamic) + { +#if AZ_TRAIT_OS_HAS_CRITICAL_SECTION_SPIN_COUNT + // In memory allocation case (usually tools) we might have high contention, + // using spin lock will improve performance. + SetCriticalSectionSpinCount(m_mutex.native_handle(), 4000); +#endif + + if (m_pageAllocator == nullptr) + { + m_pageAllocator = &AllocatorInstance::Get(); // use the SystemAllocator if no page allocator is provided + } + if (m_numStaticPages) + { + // We store the page struct at the end of the block + char* memBlock = reinterpret_cast(m_pageAllocator->Allocate( + m_pageSize * m_numStaticPages, m_pageSize, 0, "AZSystem::ThreadPoolSchemaImpl::Page static array", __FILE__, __LINE__)); + m_staticDataBlock = memBlock; + size_t pageDataSize = m_pageSize - sizeof(Page); + for (unsigned int i = 0; i < m_numStaticPages; ++i) + { + Page* page = new (memBlock + pageDataSize) Page(m_threadPoolGetter()); + page->m_bin = 0xffffffff; + PushFreePage(page); + memBlock += m_pageSize; + } + } + } + + //========================================================================= + // ~ThreadPoolSchemaImpl + // [9/15/2009] + //========================================================================= + ThreadPoolSchemaImpl::~ThreadPoolSchemaImpl() + { + // clean up all the thread data. + // IMPORTANT: We assume/rely that all threads (except the calling one) are or will + // destroyed before you create another instance of the pool allocation. + // This should generally be ok since the all allocators are singletons. + { + AZStd::lock_guard lock(m_mutex); + if (!m_threads.empty()) + { + for (size_t i = 0; i < m_threads.size(); ++i) + { + if (m_threads[i]) + { + // Force free all pages + delete m_threads[i]; + } + } + + /// reset the variable for the owner thread. + m_threadPoolSetter(nullptr); + } + } + + GarbageCollect(); + + if (m_staticDataBlock) + { + Page* page; + { + AZStd::lock_guard lock(m_mutex); + while (!m_freePages.empty()) + { + page = &m_freePages.front(); + m_freePages.pop_front(); + AZ_Assert(IsInStaticBlock(page), "All dynamic pages should be free by now!"); + } + } + + char* memBlock = reinterpret_cast(m_staticDataBlock); + size_t pageDataSize = m_pageSize - sizeof(Page); + for (unsigned int i = 0; i < m_numStaticPages; ++i) + { + page = reinterpret_cast(memBlock + pageDataSize); + page->~Page(); + memBlock += m_pageSize; + } + m_pageAllocator->DeAllocate(m_staticDataBlock); + } + } + + //========================================================================= + // Allocate + // [9/15/2009] + //========================================================================= + ThreadPoolSchema::pointer_type ThreadPoolSchemaImpl::Allocate( + ThreadPoolSchema::size_type byteSize, ThreadPoolSchema::size_type alignment, int flags) + { + (void)flags; + + ThreadPoolData* threadData = m_threadPoolGetter(); + + if (threadData == nullptr) + { + threadData = aznew ThreadPoolData(this, m_pageSize, m_minAllocationSize, m_maxAllocationSize); + m_threadPoolSetter(threadData); + { + AZStd::lock_guard lock(m_mutex); + m_threads.push_back(threadData); + } } else { - FreePage(page); + // deallocate elements if they were freed from other threads + Page::FakeNodeLF* fakeLFNode; + while ((fakeLFNode = threadData->m_freedElements.pop()) != nullptr) + { + threadData->m_allocator.DeAllocate(fakeLFNode); + } + } + + return threadData->m_allocator.Allocate(byteSize, alignment); + } + + //========================================================================= + // DeAllocate + // [9/15/2009] + //========================================================================= + void ThreadPoolSchemaImpl::DeAllocate(ThreadPoolSchema::pointer_type ptr) + { + Page* page = PageFromAddress(ptr); + if (page == nullptr) + { + AZ_Error("Memory", false, "Address 0x%08x is not in the ThreadPool!", ptr); + return; + } + AZ_Assert(page->m_threadData != nullptr, ("We must have valid page thread data for the page!")); + ThreadPoolData* threadData = m_threadPoolGetter(); + if (threadData == page->m_threadData) + { + // we can free here + threadData->m_allocator.DeAllocate(ptr); + } + else + { + // push this element to be deleted from it's own thread! + // cast the pointer to a fake lock free node + Page::FakeNodeLF* fakeLFNode = reinterpret_cast(ptr); +#ifdef AZ_DEBUG_BUILD + // we need to reset the fakeLFNode because we share the memory. + // otherwise we will assert the node is in the list + fakeLFNode->m_next = 0; +#endif + page->m_threadData->m_freedElements.push(*fakeLFNode); } } - while (!staticPages.empty()) - { - Page* page = &staticPages.front(); - staticPages.pop_front(); - m_freePages.push_front(*page); - } -} -//========================================================================= -// SetupFreeList -// [9/15/2009] -//========================================================================= -inline void -ThreadPoolSchemaImpl::Page::SetupFreeList(size_t elementSize, size_t pageDataBlockSize) -{ - char* pageData = reinterpret_cast(this) - pageDataBlockSize; - m_freeList.clear(); - // setup free list - size_t numElements = pageDataBlockSize / elementSize; - for (size_t i = 0; i < numElements; ++i) + //========================================================================= + // AllocationSize + // [11/22/2010] + //========================================================================= + ThreadPoolSchema::size_type ThreadPoolSchemaImpl::AllocationSize(ThreadPoolSchema::pointer_type ptr) { - char* address = pageData+i*elementSize; - Page::FakeNode* node = new(address) Page::FakeNode(); - m_freeList.push_back(*node); + Page* page = PageFromAddress(ptr); + if (page == nullptr) + { + return 0; + } + AZ_Assert(page->m_threadData != nullptr, ("We must have valid page thread data for the page!")); + return page->m_threadData->m_allocator.AllocationSize(ptr); } -} -//========================================================================= -// ThreadPoolData::ThreadPoolData -// [9/15/2009] -//========================================================================= -ThreadPoolData::ThreadPoolData(ThreadPoolSchemaImpl* alloc, size_t pageSize, size_t minAllocationSize, size_t maxAllocationSize) - : m_allocator(alloc, pageSize, minAllocationSize, maxAllocationSize) -{} - -//========================================================================= -// ThreadPoolData::~ThreadPoolData -// [9/15/2009] -//========================================================================= -ThreadPoolData::~ThreadPoolData() -{ - // deallocate elements if they were freed from other threads - ThreadPoolSchemaImpl::Page::FakeNodeLF* fakeLFNode; - while ((fakeLFNode = m_freedElements.pop())!=nullptr) + //========================================================================= + // PopFreePage + // [9/15/2009] + //========================================================================= + AZ_INLINE ThreadPoolSchemaImpl::Page* ThreadPoolSchemaImpl::PopFreePage() { - m_allocator.DeAllocate(fakeLFNode); + Page* page; + { + AZStd::lock_guard lock(m_mutex); + if (m_freePages.empty()) + { + page = nullptr; + } + else + { + page = &m_freePages.front(); + m_freePages.pop_front(); + } + } + if (page) + { +#ifdef AZ_DEBUG_BUILD + AZ_Assert(page->m_threadData == 0, "If we stored the free page properly we should have null here!"); +#endif + // store the current thread data, used when we free elements + page->m_threadData = m_threadPoolGetter(); + } + return page; } -} + + //========================================================================= + // PushFreePage + // [9/15/2009] + //========================================================================= + AZ_INLINE void ThreadPoolSchemaImpl::PushFreePage(Page* page) + { +#ifdef AZ_DEBUG_BUILD + page->m_threadData = 0; +#endif + { + AZStd::lock_guard lock(m_mutex); + m_freePages.push_front(*page); + } + } + + //========================================================================= + // GarbageCollect + // [9/15/2009] + //========================================================================= + void ThreadPoolSchemaImpl::GarbageCollect() + { + if (!m_isDynamic) + { + return; // we have the memory statically allocated, can't collect garbage. + } + + FreePagesType staticPages; + AZStd::lock_guard lock(m_mutex); + while (!m_freePages.empty()) + { + Page* page = &m_freePages.front(); + m_freePages.pop_front(); + if (IsInStaticBlock(page)) + { + staticPages.push_front(*page); + } + else + { + FreePage(page); + } + } + while (!staticPages.empty()) + { + Page* page = &staticPages.front(); + staticPages.pop_front(); + m_freePages.push_front(*page); + } + } + + //========================================================================= + // SetupFreeList + // [9/15/2009] + //========================================================================= + inline void ThreadPoolSchemaImpl::Page::SetupFreeList(size_t elementSize, size_t pageDataBlockSize) + { + char* pageData = reinterpret_cast(this) - pageDataBlockSize; + m_freeList.clear(); + // setup free list + size_t numElements = pageDataBlockSize / elementSize; + for (size_t i = 0; i < numElements; ++i) + { + char* address = pageData + i * elementSize; + Page::FakeNode* node = new (address) Page::FakeNode(); + m_freeList.push_back(*node); + } + } + + //========================================================================= + // ThreadPoolData::ThreadPoolData + // [9/15/2009] + //========================================================================= + ThreadPoolData::ThreadPoolData(ThreadPoolSchemaImpl* alloc, size_t pageSize, size_t minAllocationSize, size_t maxAllocationSize) + : m_allocator(alloc, pageSize, minAllocationSize, maxAllocationSize) + { + } + + //========================================================================= + // ThreadPoolData::~ThreadPoolData + // [9/15/2009] + //========================================================================= + ThreadPoolData::~ThreadPoolData() + { + // deallocate elements if they were freed from other threads + ThreadPoolSchemaImpl::Page::FakeNodeLF* fakeLFNode; + while ((fakeLFNode = m_freedElements.pop()) != nullptr) + { + m_allocator.DeAllocate(fakeLFNode); + } + } + +} // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/Memory/PoolSchema.h b/Code/Framework/AzCore/AzCore/Memory/PoolSchema.h index cfc5e3ea07..ef38dcf24f 100644 --- a/Code/Framework/AzCore/AzCore/Memory/PoolSchema.h +++ b/Code/Framework/AzCore/AzCore/Memory/PoolSchema.h @@ -5,8 +5,7 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT * */ -#ifndef AZ_POOL_ALLOCATION_SCHEME_H -#define AZ_POOL_ALLOCATION_SCHEME_H +#pragma once #include @@ -164,8 +163,3 @@ namespace AZ template AZ_THREAD_LOCAL ThreadPoolData* ThreadPoolSchemaHelper::m_threadData = 0; } - -#endif // AZ_POOL_ALLOCATION_SCHEME_H -#pragma once - - diff --git a/Code/Framework/AzCore/AzCore/Memory/SystemAllocator.cpp b/Code/Framework/AzCore/AzCore/Memory/SystemAllocator.cpp index ddcc046b8e..15cf5de8bc 100644 --- a/Code/Framework/AzCore/AzCore/Memory/SystemAllocator.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/SystemAllocator.cpp @@ -21,8 +21,8 @@ #define AZCORE_SYSTEM_ALLOCATOR_HEAP 3 #if !defined(AZCORE_SYSTEM_ALLOCATOR) - // define the default - #define AZCORE_SYSTEM_ALLOCATOR AZCORE_SYSTEM_ALLOCATOR_HPHA +// define the default +#define AZCORE_SYSTEM_ALLOCATOR AZCORE_SYSTEM_ALLOCATOR_HPHA #endif #if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA @@ -35,12 +35,11 @@ #error "Invalid allocator selected for SystemAllocator" #endif - -using namespace AZ; - -////////////////////////////////////////////////////////////////////////// -// Globals - we use global storage for the first memory schema, since we can't use dynamic memory! -static bool g_isSystemSchemaUsed = false; +namespace AZ +{ + ////////////////////////////////////////////////////////////////////////// + // Globals - we use global storage for the first memory schema, since we can't use dynamic memory! + static bool g_isSystemSchemaUsed = false; #if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA static AZStd::aligned_storage::value>::type g_systemSchema; #elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC @@ -49,268 +48,275 @@ static bool g_isSystemSchemaUsed = false; static AZStd::aligned_storage::value>::type g_systemSchema; #endif -////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// -//========================================================================= -// SystemAllocator -// [9/2/2009] -//========================================================================= -SystemAllocator::SystemAllocator() - : AllocatorBase(this, "SystemAllocator", "Fundamental generic memory allocator") - , m_isCustom(false) - , m_allocator(nullptr) - , m_ownsOSAllocator(false) -{ -} - -//========================================================================= -// ~SystemAllocator -//========================================================================= -SystemAllocator::~SystemAllocator() -{ - if (IsReady()) + //========================================================================= + // SystemAllocator + // [9/2/2009] + //========================================================================= + SystemAllocator::SystemAllocator() + : AllocatorBase(this, "SystemAllocator", "Fundamental generic memory allocator") + , m_isCustom(false) + , m_allocator(nullptr) + , m_ownsOSAllocator(false) { - Destroy(); - } -} - -//========================================================================= -// ~Create -// [9/2/2009] -//========================================================================= -bool -SystemAllocator::Create(const Descriptor& desc) -{ - AZ_Assert(IsReady() == false, "System allocator was already created!"); - if (IsReady()) - { - return false; } - m_desc = desc; - - if (!AllocatorInstance::IsReady()) + //========================================================================= + // ~SystemAllocator + //========================================================================= + SystemAllocator::~SystemAllocator() { - m_ownsOSAllocator = true; - AllocatorInstance::Create(); - } - bool isReady = false; - if (desc.m_custom) - { - m_isCustom = true; - m_allocator = desc.m_custom; - isReady = true; - } - else - { - m_isCustom = false; -#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA - HphaSchema::Descriptor heapDesc; - heapDesc.m_pageSize = desc.m_heap.m_pageSize; - heapDesc.m_poolPageSize = desc.m_heap.m_poolPageSize; - AZ_Assert(desc.m_heap.m_numFixedMemoryBlocks <= 1, "We support max1 memory block at the moment!"); - if (desc.m_heap.m_numFixedMemoryBlocks > 0) + if (IsReady()) { - heapDesc.m_fixedMemoryBlock = desc.m_heap.m_fixedMemoryBlocks[0]; - heapDesc.m_fixedMemoryBlockByteSize = desc.m_heap.m_fixedMemoryBlocksByteSize[0]; + Destroy(); } - heapDesc.m_subAllocator = desc.m_heap.m_subAllocator; - heapDesc.m_isPoolAllocations = desc.m_heap.m_isPoolAllocations; - // Fix SystemAllocator from growing in small chunks - heapDesc.m_systemChunkSize = desc.m_heap.m_systemChunkSize; -#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC - MallocSchema::Descriptor heapDesc; -#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP - HeapSchema::Descriptor heapDesc; - memcpy(heapDesc.m_memoryBlocks, desc.m_heap.m_memoryBlocks, sizeof(heapDesc.m_memoryBlocks)); - memcpy(heapDesc.m_memoryBlocksByteSize, desc.m_heap.m_memoryBlocksByteSize, sizeof(heapDesc.m_memoryBlocksByteSize)); - heapDesc.m_numMemoryBlocks = desc.m_heap.m_numMemoryBlocks; -#endif - if (&AllocatorInstance::Get() == this) // if we are the system allocator - { - AZ_Assert(!g_isSystemSchemaUsed, "AZ::SystemAllocator MUST be created first! It's the source of all allocations!"); + } -#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA - m_allocator = new(&g_systemSchema)HphaSchema(heapDesc); -#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC - m_allocator = new(&g_systemSchema)MallocSchema(heapDesc); -#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP - m_allocator = new(&g_systemSchema)HeapSchema(heapDesc); -#endif - g_isSystemSchemaUsed = true; + //========================================================================= + // ~Create + // [9/2/2009] + //========================================================================= + bool SystemAllocator::Create(const Descriptor& desc) + { + AZ_Assert(IsReady() == false, "System allocator was already created!"); + if (IsReady()) + { + return false; + } + + m_desc = desc; + + if (!AllocatorInstance::IsReady()) + { + m_ownsOSAllocator = true; + AllocatorInstance::Create(); + } + bool isReady = false; + if (desc.m_custom) + { + m_isCustom = true; + m_allocator = desc.m_custom; isReady = true; } else { - // this class should be inheriting from SystemAllocator - AZ_Assert(AllocatorInstance::IsReady(), "System allocator must be created before any other allocator! They allocate from it."); + m_isCustom = false; +#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA + HphaSchema::Descriptor heapDesc; + heapDesc.m_pageSize = desc.m_heap.m_pageSize; + heapDesc.m_poolPageSize = desc.m_heap.m_poolPageSize; + AZ_Assert(desc.m_heap.m_numFixedMemoryBlocks <= 1, "We support max1 memory block at the moment!"); + if (desc.m_heap.m_numFixedMemoryBlocks > 0) + { + heapDesc.m_fixedMemoryBlock = desc.m_heap.m_fixedMemoryBlocks[0]; + heapDesc.m_fixedMemoryBlockByteSize = desc.m_heap.m_fixedMemoryBlocksByteSize[0]; + } + heapDesc.m_subAllocator = desc.m_heap.m_subAllocator; + heapDesc.m_isPoolAllocations = desc.m_heap.m_isPoolAllocations; + // Fix SystemAllocator from growing in small chunks + heapDesc.m_systemChunkSize = desc.m_heap.m_systemChunkSize; +#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC + MallocSchema::Descriptor heapDesc; +#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP + HeapSchema::Descriptor heapDesc; + memcpy(heapDesc.m_memoryBlocks, desc.m_heap.m_memoryBlocks, sizeof(heapDesc.m_memoryBlocks)); + memcpy(heapDesc.m_memoryBlocksByteSize, desc.m_heap.m_memoryBlocksByteSize, sizeof(heapDesc.m_memoryBlocksByteSize)); + heapDesc.m_numMemoryBlocks = desc.m_heap.m_numMemoryBlocks; +#endif + if (&AllocatorInstance::Get() == this) // if we are the system allocator + { + AZ_Assert(!g_isSystemSchemaUsed, "AZ::SystemAllocator MUST be created first! It's the source of all allocations!"); #if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA - m_allocator = azcreate(HphaSchema, (heapDesc), SystemAllocator); + m_allocator = new (&g_systemSchema) HphaSchema(heapDesc); #elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC - m_allocator = azcreate(MallocSchema, (heapDesc), SystemAllocator); + m_allocator = new (&g_systemSchema) MallocSchema(heapDesc); #elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP - m_allocator = azcreate(HeapSchema, (heapDesc), SystemAllocator); + m_allocator = new (&g_systemSchema) HeapSchema(heapDesc); #endif - if (m_allocator == nullptr) - { - isReady = false; + g_isSystemSchemaUsed = true; + isReady = true; } else { - isReady = true; + // this class should be inheriting from SystemAllocator + AZ_Assert( + AllocatorInstance::IsReady(), + "System allocator must be created before any other allocator! They allocate from it."); + +#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA + m_allocator = azcreate(HphaSchema, (heapDesc), SystemAllocator); +#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC + m_allocator = azcreate(MallocSchema, (heapDesc), SystemAllocator); +#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP + m_allocator = azcreate(HeapSchema, (heapDesc), SystemAllocator); +#endif + if (m_allocator == nullptr) + { + isReady = false; + } + else + { + isReady = true; + } } } + + return isReady; } - return isReady; -} - -//========================================================================= -// Allocate -// [9/2/2009] -//========================================================================= -void -SystemAllocator::Destroy() -{ - if (g_isSystemSchemaUsed) + //========================================================================= + // Allocate + // [9/2/2009] + //========================================================================= + void SystemAllocator::Destroy() { - int dummy; - (void)dummy; - } - - if (!m_isCustom) - { - if ((void*)m_allocator == (void*)&g_systemSchema) + if (g_isSystemSchemaUsed) { + int dummy; + (void)dummy; + } + + if (!m_isCustom) + { + if ((void*)m_allocator == (void*)&g_systemSchema) + { #if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA - static_cast(m_allocator)->~HphaSchema(); + static_cast(m_allocator)->~HphaSchema(); #elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC - static_cast(m_allocator)->~MallocSchema(); + static_cast(m_allocator)->~MallocSchema(); #elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP - static_cast(m_allocator)->~HeapSchema(); + static_cast(m_allocator)->~HeapSchema(); #endif - g_isSystemSchemaUsed = false; + g_isSystemSchemaUsed = false; + } + else + { + azdestroy(m_allocator); + } } - else + + if (m_ownsOSAllocator) { - azdestroy(m_allocator); + AllocatorInstance::Destroy(); + m_ownsOSAllocator = false; } } - if (m_ownsOSAllocator) + AllocatorDebugConfig SystemAllocator::GetDebugConfig() { - AllocatorInstance::Destroy(); - m_ownsOSAllocator = false; - } -} - -AllocatorDebugConfig SystemAllocator::GetDebugConfig() -{ - return AllocatorDebugConfig() - .StackRecordLevels(m_desc.m_stackRecordLevels) - .UsesMemoryGuards(!m_isCustom) - .MarksUnallocatedMemory(!m_isCustom) - .ExcludeFromDebugging(!m_desc.m_allocationRecords); -} - -IAllocatorAllocate* SystemAllocator::GetSchema() -{ - return m_allocator; -} - -//========================================================================= -// Allocate -// [9/2/2009] -//========================================================================= -SystemAllocator::pointer_type -SystemAllocator::Allocate(size_type byteSize, size_type alignment, int flags, const char* name, const char* fileName, int lineNum, unsigned int suppressStackRecord) -{ - if (byteSize == 0) - { - return nullptr; - } - AZ_Assert(byteSize > 0, "You can not allocate 0 bytes!"); - AZ_Assert((alignment & (alignment - 1)) == 0, "Alignment must be power of 2!"); - - byteSize = MemorySizeAdjustedUp(byteSize); - SystemAllocator::pointer_type address = m_allocator->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord + 1); - - if (address == nullptr) - { - // Free all memory we can and try again! - AllocatorManager::Instance().GarbageCollect(); - - address = m_allocator->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord + 1); + return AllocatorDebugConfig() + .StackRecordLevels(m_desc.m_stackRecordLevels) + .UsesMemoryGuards(!m_isCustom) + .MarksUnallocatedMemory(!m_isCustom) + .ExcludeFromDebugging(!m_desc.m_allocationRecords); } - if (address == nullptr) + IAllocatorAllocate* SystemAllocator::GetSchema() { - byteSize = MemorySizeAdjustedDown(byteSize); // restore original size + return m_allocator; } - AZ_Assert(address != nullptr, "SystemAllocator: Failed to allocate %d bytes aligned on %d (flags: 0x%08x) %s : %s (%d)!", byteSize, alignment, flags, name ? name : "(no name)", fileName ? fileName : "(no file name)", lineNum); + //========================================================================= + // Allocate + // [9/2/2009] + //========================================================================= + SystemAllocator::pointer_type SystemAllocator::Allocate( + size_type byteSize, + size_type alignment, + int flags, + const char* name, + const char* fileName, + int lineNum, + unsigned int suppressStackRecord) + { + if (byteSize == 0) + { + return nullptr; + } + AZ_Assert(byteSize > 0, "You can not allocate 0 bytes!"); + AZ_Assert((alignment & (alignment - 1)) == 0, "Alignment must be power of 2!"); - AZ_PROFILE_MEMORY_ALLOC_EX(MemoryReserved, fileName, lineNum, address, byteSize, name); - AZ_MEMORY_PROFILE(ProfileAllocation(address, byteSize, alignment, name, fileName, lineNum, suppressStackRecord + 1)); + byteSize = MemorySizeAdjustedUp(byteSize); + SystemAllocator::pointer_type address = + m_allocator->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord + 1); - return address; -} + if (address == nullptr) + { + // Free all memory we can and try again! + AllocatorManager::Instance().GarbageCollect(); -//========================================================================= -// DeAllocate -// [9/2/2009] -//========================================================================= -void -SystemAllocator::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) -{ - byteSize = MemorySizeAdjustedUp(byteSize); - AZ_PROFILE_MEMORY_FREE(MemoryReserved, ptr); - AZ_MEMORY_PROFILE(ProfileDeallocation(ptr, byteSize, alignment, nullptr)); - m_allocator->DeAllocate(ptr, byteSize, alignment); -} + address = m_allocator->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord + 1); + } -//========================================================================= -// ReAllocate -// [9/13/2011] -//========================================================================= -SystemAllocator::pointer_type -SystemAllocator::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) -{ - newSize = MemorySizeAdjustedUp(newSize); + if (address == nullptr) + { + byteSize = MemorySizeAdjustedDown(byteSize); // restore original size + } - AZ_MEMORY_PROFILE(ProfileReallocationBegin(ptr, newSize)); - AZ_PROFILE_MEMORY_FREE(MemoryReserved, ptr); - pointer_type newAddress = m_allocator->ReAllocate(ptr, newSize, newAlignment); - AZ_PROFILE_MEMORY_ALLOC(MemoryReserved, newAddress, newSize, "SystemAllocator realloc"); - AZ_MEMORY_PROFILE(ProfileReallocationEnd(ptr, newAddress, newSize, newAlignment)); + AZ_Assert( + address != nullptr, "SystemAllocator: Failed to allocate %d bytes aligned on %d (flags: 0x%08x) %s : %s (%d)!", byteSize, + alignment, flags, name ? name : "(no name)", fileName ? fileName : "(no file name)", lineNum); - return newAddress; -} + AZ_PROFILE_MEMORY_ALLOC_EX(MemoryReserved, fileName, lineNum, address, byteSize, name); + AZ_MEMORY_PROFILE(ProfileAllocation(address, byteSize, alignment, name, fileName, lineNum, suppressStackRecord + 1)); -//========================================================================= -// Resize -// [8/12/2011] -//========================================================================= -SystemAllocator::size_type -SystemAllocator::Resize(pointer_type ptr, size_type newSize) -{ - newSize = MemorySizeAdjustedUp(newSize); - size_type resizedSize = m_allocator->Resize(ptr, newSize); + return address; + } - AZ_MEMORY_PROFILE(ProfileResize(ptr, resizedSize)); + //========================================================================= + // DeAllocate + // [9/2/2009] + //========================================================================= + void SystemAllocator::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) + { + byteSize = MemorySizeAdjustedUp(byteSize); + AZ_PROFILE_MEMORY_FREE(MemoryReserved, ptr); + AZ_MEMORY_PROFILE(ProfileDeallocation(ptr, byteSize, alignment, nullptr)); + m_allocator->DeAllocate(ptr, byteSize, alignment); + } - return MemorySizeAdjustedDown(resizedSize); -} + //========================================================================= + // ReAllocate + // [9/13/2011] + //========================================================================= + SystemAllocator::pointer_type SystemAllocator::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) + { + newSize = MemorySizeAdjustedUp(newSize); -//========================================================================= -// -// [8/12/2011] -//========================================================================= -SystemAllocator::size_type -SystemAllocator::AllocationSize(pointer_type ptr) -{ - size_type allocSize = MemorySizeAdjustedDown(m_allocator->AllocationSize(ptr)); + AZ_MEMORY_PROFILE(ProfileReallocationBegin(ptr, newSize)); + AZ_PROFILE_MEMORY_FREE(MemoryReserved, ptr); + pointer_type newAddress = m_allocator->ReAllocate(ptr, newSize, newAlignment); + AZ_PROFILE_MEMORY_ALLOC(MemoryReserved, newAddress, newSize, "SystemAllocator realloc"); + AZ_MEMORY_PROFILE(ProfileReallocationEnd(ptr, newAddress, newSize, newAlignment)); - return allocSize; -} + return newAddress; + } + + //========================================================================= + // Resize + // [8/12/2011] + //========================================================================= + SystemAllocator::size_type SystemAllocator::Resize(pointer_type ptr, size_type newSize) + { + newSize = MemorySizeAdjustedUp(newSize); + size_type resizedSize = m_allocator->Resize(ptr, newSize); + + AZ_MEMORY_PROFILE(ProfileResize(ptr, resizedSize)); + + return MemorySizeAdjustedDown(resizedSize); + } + + //========================================================================= + // + // [8/12/2011] + //========================================================================= + SystemAllocator::size_type SystemAllocator::AllocationSize(pointer_type ptr) + { + size_type allocSize = MemorySizeAdjustedDown(m_allocator->AllocationSize(ptr)); + + return allocSize; + } + +} // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/Memory/SystemAllocator.h b/Code/Framework/AzCore/AzCore/Memory/SystemAllocator.h index c02ada5843..c730b8dde6 100644 --- a/Code/Framework/AzCore/AzCore/Memory/SystemAllocator.h +++ b/Code/Framework/AzCore/AzCore/Memory/SystemAllocator.h @@ -5,8 +5,7 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT * */ -#ifndef AZCORE_SYS_ALLOCATOR_H -#define AZCORE_SYS_ALLOCATOR_H +#pragma once #include @@ -120,7 +119,5 @@ namespace AZ }; } -#endif // AZCORE_SYS_ALLOCATOR_H -#pragma once