birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Making allocator use the MallocSchema so we can take full advantage of ASan

Browse Source 
Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com>
This commit is contained in:
Esteban Papp
2021-08-12 16:44:59 -07:00
parent a087fc06a9
commit 89e1b6db25
7 changed files with 138 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Framework/AzCore/AzCore/IO/Path/Path.inl
+1
View File
@@ -10,6 +10,7 @@
#include <AzCore/std/containers/array.h>
#include <AzCore/std/string/wildcard.h>
#include <AzCore/Casting/numeric_cast.h>
// extern instantiations of Path templates to prevent implicit instantiations
namespace AZ::IO
Code/Framework/AzCore/AzCore/Memory/SystemAllocator.cpp
+36 -30
View File
@@ -17,15 +17,23 @@
#include <AzCore/Debug/Profiler.h>
#define AZCORE_SYS_ALLOCATOR_HPPA // If you disable this make sure you start building the heapschema.cpp
//#define AZCORE_SYS_ALLOCATOR_MALLOC
#define AZCORE_SYSTEM_ALLOCATOR_HPHA 1
#define AZCORE_SYSTEM_ALLOCATOR_MALLOC 2
#define AZCORE_SYSTEM_ALLOCATOR_HEAP 3
#ifdef AZCORE_SYS_ALLOCATOR_HPPA
# include <AzCore/Memory/HphaSchema.h>
#elif defined(AZCORE_SYS_ALLOCATOR_MALLOC)
#include <AzCore/Memory/MallocSchema.h>
#if !defined(AZCORE_SYSTEM_ALLOCATOR)
// define the default
#define AZCORE_SYSTEM_ALLOCATOR AZCORE_SYSTEM_ALLOCATOR_HPHA
#endif
#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA
#include <AzCore/Memory/HphaSchema.h>
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC
#include <AzCore/Memory/MallocSchema.h>
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP
#include <AzCore/Memory/HeapSchema.h>
#else
# include <AzCore/Memory/HeapSchema.h>
#error "Invalid allocator selected for SystemAllocator"
#endif
@@ -34,12 +42,12 @@ using namespace AZ;
//////////////////////////////////////////////////////////////////////////
// Globals - we use global storage for the first memory schema, since we can't use dynamic memory!
static bool g_isSystemSchemaUsed = false;
#ifdef AZCORE_SYS_ALLOCATOR_HPPA
static AZStd::aligned_storage<sizeof(HphaSchema), AZStd::alignment_of<HphaSchema>::value>::type g_systemSchema;
#elif defined(AZCORE_SYS_ALLOCATOR_MALLOC)
static AZStd::aligned_storage<sizeof(MallocSchema), AZStd::alignment_of<MallocSchema>::value>::type g_systemSchema;
#else
static AZStd::aligned_storage<sizeof(HeapSchema), AZStd::alignment_of<HeapSchema>::value>::type g_systemSchema;
#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA
static AZStd::aligned_storage<sizeof(HphaSchema), AZStd::alignment_of<HphaSchema>::value>::type g_systemSchema;
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC
static AZStd::aligned_storage<sizeof(MallocSchema), AZStd::alignment_of<MallocSchema>::value>::type g_systemSchema;
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP
static AZStd::aligned_storage<sizeof(HeapSchema), AZStd::alignment_of<HeapSchema>::value>::type g_systemSchema;
#endif
//////////////////////////////////////////////////////////////////////////
@@ -97,9 +105,9 @@ SystemAllocator::Create(const Descriptor& desc)
else
{
m_isCustom = false;
#ifdef AZCORE_SYS_ALLOCATOR_HPPA
HphaSchema::Descriptor heapDesc;
heapDesc.m_pageSize = desc.m_heap.m_pageSize;
#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA
HphaSchema::Descriptor heapDesc;
heapDesc.m_pageSize = desc.m_heap.m_pageSize;
heapDesc.m_poolPageSize = desc.m_heap.m_poolPageSize;
AZ_Assert(desc.m_heap.m_numFixedMemoryBlocks <= 1, "We support max1 memory block at the moment!");
if (desc.m_heap.m_numFixedMemoryBlocks > 0)
@@ -111,11 +119,10 @@ SystemAllocator::Create(const Descriptor& desc)
heapDesc.m_isPoolAllocations = desc.m_heap.m_isPoolAllocations;
// Fix SystemAllocator from growing in small chunks
heapDesc.m_systemChunkSize = desc.m_heap.m_systemChunkSize;
#elif defined(AZCORE_SYS_ALLOCATOR_MALLOC)
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC
MallocSchema::Descriptor heapDesc;
#else
HeapSchema::Descriptor heapDesc;
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP
HeapSchema::Descriptor heapDesc;
memcpy(heapDesc.m_memoryBlocks, desc.m_heap.m_memoryBlocks, sizeof(heapDesc.m_memoryBlocks));
memcpy(heapDesc.m_memoryBlocksByteSize, desc.m_heap.m_memoryBlocksByteSize, sizeof(heapDesc.m_memoryBlocksByteSize));
heapDesc.m_numMemoryBlocks = desc.m_heap.m_numMemoryBlocks;
@@ -124,11 +131,11 @@ SystemAllocator::Create(const Descriptor& desc)
{
AZ_Assert(!g_isSystemSchemaUsed, "AZ::SystemAllocator MUST be created first! It's the source of all allocations!");
#ifdef AZCORE_SYS_ALLOCATOR_HPPA
#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA
m_allocator = new(&g_systemSchema)HphaSchema(heapDesc);
#elif defined(AZCORE_SYS_ALLOCATOR_MALLOC)
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC
m_allocator = new(&g_systemSchema)MallocSchema(heapDesc);
#else
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP
m_allocator = new(&g_systemSchema)HeapSchema(heapDesc);
#endif
g_isSystemSchemaUsed = true;
@@ -139,14 +146,13 @@ SystemAllocator::Create(const Descriptor& desc)
// this class should be inheriting from SystemAllocator
AZ_Assert(AllocatorInstance<SystemAllocator>::IsReady(), "System allocator must be created before any other allocator! They allocate from it.");
#ifdef AZCORE_SYS_ALLOCATOR_HPPA
#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA
m_allocator = azcreate(HphaSchema, (heapDesc), SystemAllocator);
#elif defined(AZCORE_SYS_ALLOCATOR_MALLOC)
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC
m_allocator = azcreate(MallocSchema, (heapDesc), SystemAllocator);
#else
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP
m_allocator = azcreate(HeapSchema, (heapDesc), SystemAllocator);
#endif
if (m_allocator == NULL)
{
isReady = false;
@@ -178,11 +184,11 @@ SystemAllocator::Destroy()
{
if ((void*)m_allocator == (void*)&g_systemSchema)
{
#ifdef AZCORE_SYS_ALLOCATOR_HPPA
#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA
static_cast<HphaSchema*>(m_allocator)->~HphaSchema();
#elif defined(AZCORE_SYS_ALLOCATOR_MALLOC)
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC
static_cast<MallocSchema*>(m_allocator)->~MallocSchema();
#else
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HEAP
static_cast<HeapSchema*>(m_allocator)->~HeapSchema();
#endif
g_isSystemSchemaUsed = false;
Code/Framework/AzCore/CMakeLists.txt
+9
View File
@@ -54,6 +54,15 @@ ly_add_source_properties(
VALUES ${LY_PAL_TOOLS_DEFINES}
)
if(LY_BUILD_WITH_ADDRESS_SANITIZER)
# Default to use Malloc schema so ASan works well
ly_add_source_properties(
SOURCES AzCore/Memory/SystemAllocator.cpp
PROPERTY COMPILE_DEFINITIONS
VALUES AZCORE_SYSTEM_ALLOCATOR=AZCORE_SYSTEM_ALLOCATOR_MALLOC
)
endif()
################################################################################
# Tests
################################################################################
Code/Framework/AzFramework/AzFramework/Physics/Common/PhysicsTypes.h
+1
View File
@@ -13,6 +13,7 @@
#include <AzCore/std/containers/variant.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/smart_ptr/shared_ptr.h>
#include <AzCore/Casting/numeric_cast.h>
namespace Physics
{
Code/Legacy/CryCommon/LegacyAllocator.cpp
+79
View File
@@ -0,0 +1,79 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <CryCommon/LegacyAllocator.h>
namespace AZ
{
LegacyAllocator::pointer_type LegacyAllocator::Allocate(size_type byteSize, size_type alignment, int flags, const char* name, const char* fileName, int lineNum, unsigned int suppressStackRecord)
{
if (alignment == 0)
{
// Some STL containers, like std::vector, are assuming a specific minimum alignment. seems to have a requirement
// Take a look at _Allocate_manually_vector_aligned in xmemory0
alignment = sizeof(void*) * 2;
}
pointer_type ptr = m_schema->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord);
AZ_PROFILE_MEMORY_ALLOC_EX(AZ::Debug::ProfileCategory::MemoryReserved, fileName, lineNum, ptr, byteSize, name ? name : GetName());
AZ_MEMORY_PROFILE(ProfileAllocation(ptr, byteSize, alignment, name, fileName, lineNum, suppressStackRecord));
AZ_Assert(ptr || byteSize == 0, "OOM - Failed to allocate %zu bytes from LegacyAllocator", byteSize);
return ptr;
}
// DeAllocate with file/line, to track when allocs were freed from Cry
void LegacyAllocator::DeAllocate(pointer_type ptr, [[maybe_unused]] const char* file, [[maybe_unused]] const int line, size_type byteSize, size_type alignment)
{
AZ_PROFILE_MEMORY_FREE_EX(AZ::Debug::ProfileCategory::MemoryReserved, file, line, ptr);
AZ_MEMORY_PROFILE(ProfileDeallocation(ptr, byteSize, alignment, nullptr));
m_schema->DeAllocate(ptr, byteSize, alignment);
}
// Realloc with file/line, because Cry uses realloc(nullptr) and realloc(ptr, 0) to mimic malloc/free
LegacyAllocator::pointer_type LegacyAllocator::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment, [[maybe_unused]] const char* file, [[maybe_unused]] const int line)
{
if (newAlignment == 0)
{
// Some STL containers, like std::vector, are assuming a specific minimum alignment. seems to have a requirement
// Take a look at _Allocate_manually_vector_aligned in xmemory0
newAlignment = sizeof(void*) * 2;
}
AZ_MEMORY_PROFILE(ProfileReallocationBegin(ptr, newSize));
AZ_PROFILE_MEMORY_FREE_EX(AZ::Debug::ProfileCategory::MemoryReserved, file, line, ptr);
pointer_type newPtr = m_schema->ReAllocate(ptr, newSize, newAlignment);
AZ_PROFILE_MEMORY_ALLOC_EX(AZ::Debug::ProfileCategory::MemoryReserved, file, line, newPtr, newSize, "LegacyAllocator Realloc");
AZ_MEMORY_PROFILE(ProfileReallocationEnd(ptr, newPtr, newSize, newAlignment));
AZ_Assert(newPtr || newSize == 0, "OOM - Failed to reallocate %zu bytes from LegacyAllocator", newSize);
return newPtr;
}
void LegacyAllocator::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment)
{
AZ_MEMORY_PROFILE(ProfileDeallocation(ptr, 0, 0, nullptr));
Base::DeAllocate(ptr, byteSize, alignment);
}
LegacyAllocator::pointer_type LegacyAllocator::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment)
{
if (newAlignment == 0)
{
// Some STL containers, like std::vector, are assuming a specific minimum alignment. seems to have a requirement
// Take a look at _Allocate_manually_vector_aligned in xmemory0
newAlignment = sizeof(void*) * 2;
}
AZ_MEMORY_PROFILE(ProfileReallocationBegin(ptr, newSize));
pointer_type newPtr = Base::ReAllocate(ptr, newSize, newAlignment);
AZ_MEMORY_PROFILE(ProfileReallocationEnd(ptr, newPtr, newSize, newAlignment));
AZ_Assert(newPtr || newSize == 0, "OOM - Failed to reallocate %zu bytes from LegacyAllocator", newSize);
return newPtr;
}
}
Code/Legacy/CryCommon/LegacyAllocator.h
+11 -146
View File
@@ -11,118 +11,36 @@
#include <AzCore/Memory/AllocatorBase.h>
#include <AzCore/Memory/HphaSchema.h>
#define AZCORE_SYS_ALLOCATOR_HPPA
//#define AZCORE_SYS_ALLOCATOR_MALLOC
#ifdef AZCORE_SYS_ALLOCATOR_HPPA
# include <AzCore/Memory/HphaSchema.h>
#elif defined(AZCORE_SYS_ALLOCATOR_MALLOC)
# include <AzCore/Memory/MallocSchema.h>
#else
# include <AzCore/Memory/HeapSchema.h>
#endif
namespace AZ
{
#ifdef AZCORE_SYS_ALLOCATOR_HPPA
typedef AZ::HphaSchema LegacyAllocatorSchema;
#elif defined(AZCORE_SYS_ALLOCATOR_MALLOC)
typedef AZ::MallocSchema LegacyAllocatorSchema;
#else
typedef AZ::HeapSchema LegacyAllocatorSchema;
#endif
struct LegacyAllocatorDescriptor
: public LegacyAllocatorSchema::Descriptor
{
LegacyAllocatorDescriptor()
{
// pull 32MB from the OS at a time
#ifdef AZCORE_SYS_ALLOCATOR_HPPA
m_systemChunkSize = 32 * 1024 * 1024;
#endif
}
};
class LegacyAllocator
: public SimpleSchemaAllocator<AZ::HphaSchema, LegacyAllocatorDescriptor>
: public SimpleSchemaAllocator<AZ::HphaSchema, AZ::HphaSchema::Descriptor>
{
public:
AZ_TYPE_INFO(LegacyAllocator, "{17FC25A4-92D9-48C5-BB85-7F860FCA2C6F}");
using Descriptor = LegacyAllocatorDescriptor;
using Base = SimpleSchemaAllocator<AZ::HphaSchema, LegacyAllocatorDescriptor>;
using Descriptor = AZ::HphaSchema::Descriptor;
using Base = SimpleSchemaAllocator<AZ::HphaSchema, Descriptor>;
using pointer_type = typename Base::pointer_type;
using size_type = typename Base::size_type;
using difference_type = typename Base::difference_type;
LegacyAllocator()
: Base("LegacyAllocator", "Allocator for Legacy CryEngine systems")
{
}
pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = 0, const char* fileName = 0, int lineNum = 0, unsigned int suppressStackRecord = 0) override
{
if (alignment == 0)
{
// Some STL containers, like std::vector, are assuming a specific minimum alignment. seems to have a requirement
// Take a look at _Allocate_manually_vector_aligned in xmemory0
alignment = sizeof(void*) * 2;
}
pointer_type ptr = m_schema->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord);
AZ_PROFILE_MEMORY_ALLOC_EX(AZ::Debug::ProfileCategory::MemoryReserved, fileName, lineNum, ptr, byteSize, name ? name : GetName());
AZ_MEMORY_PROFILE(ProfileAllocation(ptr, byteSize, alignment, name, fileName, lineNum, suppressStackRecord));
AZ_Assert(ptr || byteSize == 0, "OOM - Failed to allocate %zu bytes from LegacyAllocator", byteSize);
return ptr;
}
pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = 0, const char* fileName = 0, int lineNum = 0, unsigned int suppressStackRecord = 0) override;
// DeAllocate with file/line, to track when allocs were freed from Cry
void DeAllocate(pointer_type ptr, [[maybe_unused]] const char* file, [[maybe_unused]] const int line, size_type byteSize = 0, size_type alignment = 0)
{
AZ_PROFILE_MEMORY_FREE_EX(AZ::Debug::ProfileCategory::MemoryReserved, file, line, ptr);
AZ_MEMORY_PROFILE(ProfileDeallocation(ptr, byteSize, alignment, nullptr));
m_schema->DeAllocate(ptr, byteSize, alignment);
}
void DeAllocate(pointer_type ptr, const char* file, const int line, size_type byteSize = 0, size_type alignment = 0);
// Realloc with file/line, because Cry uses realloc(nullptr) and realloc(ptr, 0) to mimic malloc/free
pointer_type ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment, [[maybe_unused]] const char* file, [[maybe_unused]] const int line)
{
if (newAlignment == 0)
{
// Some STL containers, like std::vector, are assuming a specific minimum alignment. seems to have a requirement
// Take a look at _Allocate_manually_vector_aligned in xmemory0
newAlignment = sizeof(void*) * 2;
}
pointer_type ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment, const char* file, const int line);
AZ_MEMORY_PROFILE(ProfileReallocationBegin(ptr, newSize));
AZ_PROFILE_MEMORY_FREE_EX(AZ::Debug::ProfileCategory::MemoryReserved, file, line, ptr);
pointer_type newPtr = m_schema->ReAllocate(ptr, newSize, newAlignment);
AZ_PROFILE_MEMORY_ALLOC_EX(AZ::Debug::ProfileCategory::MemoryReserved, file, line, newPtr, newSize, "LegacyAllocator Realloc");
AZ_MEMORY_PROFILE(ProfileReallocationEnd(ptr, newPtr, newSize, newAlignment));
AZ_Assert(newPtr || newSize == 0, "OOM - Failed to reallocate %zu bytes from LegacyAllocator", newSize);
return newPtr;
}
void DeAllocate(pointer_type ptr, size_type byteSize = 0, size_type alignment = 0) override;
void DeAllocate(pointer_type ptr, size_type byteSize = 0, size_type alignment = 0) override
{
AZ_MEMORY_PROFILE(ProfileDeallocation(ptr, 0, 0, nullptr));
Base::DeAllocate(ptr, byteSize, alignment);
}
pointer_type ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) override
{
if (newAlignment == 0)
{
// Some STL containers, like std::vector, are assuming a specific minimum alignment. seems to have a requirement
// Take a look at _Allocate_manually_vector_aligned in xmemory0
newAlignment = sizeof(void*) * 2;
}
AZ_MEMORY_PROFILE(ProfileReallocationBegin(ptr, newSize));
pointer_type newPtr = Base::ReAllocate(ptr, newSize, newAlignment);
AZ_MEMORY_PROFILE(ProfileReallocationEnd(ptr, newPtr, newSize, newAlignment));
AZ_Assert(newPtr || newSize == 0, "OOM - Failed to reallocate %zu bytes from LegacyAllocator", newSize);
return newPtr;
}
pointer_type ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) override;
};
using StdLegacyAllocator = AZStdAlloc<LegacyAllocator>;
@@ -133,57 +51,4 @@ namespace AZ
class AllocatorInstance<LegacyAllocator> : public Internal::AllocatorInstanceBase<LegacyAllocator>
{
};
#if defined(AZ_PLATFORM_PROVO) || defined(AZ_PLATFORM_JASPER)
struct GlobalAllocatorDescriptor
: public AZ::HphaSchema::Descriptor
{
GlobalAllocatorDescriptor()
{
// pull 1MB from the OS at a time
m_systemChunkSize = 1024 * 1024;
}
};
class GlobalAllocator
: public SimpleSchemaAllocator<AZ::HphaSchema, GlobalAllocatorDescriptor>
{
public:
AZ_TYPE_INFO(GlobalAllocator, "{BC7861DA-AF7F-4FFD-A2F5-BAD89BDD77FD}");
using Descriptor = GlobalAllocatorDescriptor;
using Base = SimpleSchemaAllocator<AZ::HphaSchema, GlobalAllocatorDescriptor>;
GlobalAllocator()
: Base("GlobalAllocator", "Allocator for untracked new/delete/malloc/free")
{
}
//---------------------------------------------------------------------
// IAllocatorAllocate
//---------------------------------------------------------------------
pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = 0, const char* fileName = 0, int lineNum = 0, unsigned int suppressStackRecord = 0) override
{
// Note: We cannot put the asserts in the AllocateBase class because various allocators depend on allocations failing from some heap classes.
pointer_type ptr = Base::Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord);
AZ_Assert(ptr, "OOM - Failed to allocate %zu bytes from GlobalAllocator", byteSize);
return ptr;
}
pointer_type ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) override
{
pointer_type newPtr = Base::ReAllocate(ptr, newSize, newAlignment);
AZ_Assert(newPtr, "OOM - Failed to reallocate %zu bytes from GlobalAllocator", newSize);
return newPtr;
}
};
// Specialize for the GlobalAllocator to provide one per module that does not use the
// environment for its storage
template <>
class AllocatorInstance<GlobalAllocator> : public Internal::AllocatorInstanceBase<GlobalAllocator, AllocatorStorage::ModuleStoragePolicy<GlobalAllocator, false>>
{
};
#endif
}
Code/Legacy/CryCommon/crycommon_files.cmake
+1
View File
@@ -92,6 +92,7 @@ set(FILES
CryVersion.h
FrameProfiler.h
HeapAllocator.h
LegacyAllocator.cpp
LegacyAllocator.h
MetaUtils.h
MiniQueue.h