Initial commit

This commit is contained in:
alexpete
2021-03-05 11:26:34 -08:00
commit a10351f38d
27091 changed files with 5521199 additions and 0 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,311 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_ALEMBICCOMPILER_H
#define CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_ALEMBICCOMPILER_H
#pragma once
#include "IConvertor.h"
#include "GeomCache.h"
#include "../ResourceCompilerPC/PhysWorld.h"
#include <AzCore/std/parallel/atomic.h>
#include <AzCore/std/parallel/mutex.h>
#include <AzCore/std/parallel/condition_variable.h>
#include <Alembic/AbcGeom/Visibility.h> // for Alembic::AbcGeom::ObjectVisibility
#define RC_ABC_AUTOMATIC_UVMAX_DETECTION_VALUE .0f
class ICryXML;
class GeomCacheEncoder;
class IXMLSerializer;
// Used for detecting identical meshes. For two identical meshes all digests must match.
class AlembicMeshDigest
{
friend struct std::hash<AlembicMeshDigest>;
public:
AlembicMeshDigest(Alembic::AbcGeom::IPolyMeshSchema& meshSchema);
bool operator==(const AlembicMeshDigest& digest) const;
private:
bool m_bHasNormals;
bool m_bHasTexcoords;
bool m_bHasColors;
Alembic::AbcGeom::ArraySampleKey m_positionDigest;
Alembic::AbcGeom::ArraySampleKey m_positionIndexDigest;
Alembic::AbcGeom::ArraySampleKey m_normalsDigest;
Alembic::AbcGeom::ArraySampleKey m_texcoordDigest;
Alembic::AbcGeom::ArraySampleKey m_colorsDigest;
};
// Unoptimized vertex used in the compiler
struct AlembicCompilerVertex
{
Vec3 m_position;
Vec3 m_normal;
Vec2 m_texcoords;
Vec4 m_rgba;
};
template<class T>
class AlembicCompilerHash;
template<>
class AlembicCompilerHash<float>
{
public:
uint64 operator()(const float value)
{
uint32 bits = alias_cast<uint32>(value);
bits = (bits == 0x80000000) ? 0 : bits; // -0 == 0
// Magic taken from CityHash64
const uint64 u = bits;
const uint64 kMul = 0x9ddfea08eb382d69ULL;
uint64 a = u * kMul;
a ^= (a >> 47);
a *= kMul;
return a;
}
};
template<>
class AlembicCompilerHash<uint64>
{
public:
uint64 operator()(const uint64 value)
{
return value;
}
};
// Helper function to combine hashes
template<class T>
inline void AlembicCompilerHashCombine(uint64& seed, const T& v)
{
AlembicCompilerHash<T> hasher;
// Magic taken from CityHash64
const uint64 kMul = 0x9ddfea08eb382d69ULL;
uint64 a = (hasher(v) ^ seed) * kMul;
a ^= (a >> 47);
uint64 b = (seed ^ a) * kMul;
b ^= (b >> 47);
seed = b * kMul;
}
template<>
class AlembicCompilerHash<AlembicCompilerVertex>
{
public:
uint64 operator()(const AlembicCompilerVertex& vertex) const
{
uint64 hash = 0;
AlembicCompilerHashCombine(hash, vertex.m_position[0]);
AlembicCompilerHashCombine(hash, vertex.m_position[1]);
AlembicCompilerHashCombine(hash, vertex.m_position[2]);
AlembicCompilerHashCombine(hash, vertex.m_normal[0]);
AlembicCompilerHashCombine(hash, vertex.m_normal[1]);
AlembicCompilerHashCombine(hash, vertex.m_normal[2]);
AlembicCompilerHashCombine(hash, vertex.m_texcoords[0]);
AlembicCompilerHashCombine(hash, vertex.m_texcoords[1]);
AlembicCompilerHashCombine(hash, vertex.m_rgba[0]);
AlembicCompilerHashCombine(hash, vertex.m_rgba[1]);
AlembicCompilerHashCombine(hash, vertex.m_rgba[2]);
AlembicCompilerHashCombine(hash, vertex.m_rgba[3]);
return hash;
}
};
// std::hash<> specializations for std::unordered_map
namespace std
{
template<>
class hash<AlembicMeshDigest>
{
public:
size_t operator()(const AlembicMeshDigest& digest) const
{
// Just return the position digest. It's very likely that
// if positions match everything else is matching as well
return Alembic::AbcGeom::StdHash(digest.m_positionDigest);
}
};
}
class GeomCacheEncoder;
class AlembicCompiler
: public ICompiler
{
struct FrameData
{
FrameData()
: m_errorCount(0)
, m_pAlembicCompiler(nullptr) {}
FrameData(const FrameData& toBeCopied)
: m_errorCount(0)
, m_pAlembicCompiler(toBeCopied.m_pAlembicCompiler)
{
m_errorCount.store(toBeCopied.m_errorCount);
}
uint m_jobIndex;
uint m_frameIndex;
AZStd::atomic_uint m_errorCount;
Alembic::Abc::chrono_t m_frameTime;
AlembicCompiler* m_pAlembicCompiler;
AABB m_frameAABB;
};
public:
AlembicCompiler(ICryXML* pXMLParser);
virtual ~AlembicCompiler() {}
// ICompiler methods.
virtual void Release();
virtual void BeginProcessing([[maybe_unused]] const IConfig* config) {}
virtual void EndProcessing() {}
virtual IConvertContext* GetConvertContext() { return &m_CC; }
virtual bool Process();
private:
XmlNodeRef ReadConfig(const string& configPath, IXMLSerializer* pXMLSerializer);
string GetOutputFileNameOnly() const;
string GetOutputPath() const;
uint32_t GetIndex(Alembic::AbcGeom::GeometryScope geomScope, const Alembic::Abc::UInt32ArraySamplePtr& normalIndices, size_t currentIndexArraysIndex, int32_t positionIndex);
bool CheckTimeSampling(Alembic::Abc::IArchive& archive);
void OutputTimeSamplingType(const Alembic::Abc::TimeSamplingType& timeSamplingType);
void CheckTimeSamplingRec(const Alembic::Abc::IObject& currentObject);
void CheckTimeSamplingRec(const Alembic::Abc::ICompoundProperty& currentProperty);
// Setup static data for header
bool CompileStaticData(Alembic::Abc::IArchive& archive);
bool CompileStaticDataRec(GeomCache::Node* pParentNode, Alembic::Abc::IObject& currentObject, const QuatTNS& localTransform,
std::vector<Alembic::AbcGeom::IXform> abcXformStack, const bool bParentRemoved, const GeomCacheFile::ETransformType parentTransform);
bool CompileStaticMeshData(GeomCache::Node& node, Alembic::AbcGeom::IPolyMesh& mesh);
bool CompilePhysicsGeometry(GeomCache::Node& node, Alembic::AbcGeom::IPolyMesh& mesh);
void CheckMeshForColors(Alembic::AbcGeom::IPolyMeshSchema& meshSchema, GeomCache::Mesh& mesh) const;
// Prints the node tree
void PrintNodeTreeRec(GeomCache::Node& node, string padding);
// Compile stream of frames and send them to the cache writer
bool CompileAnimationData(Alembic::Abc::IArchive& archive, GeomCacheEncoder& geomCacheEncoder);
// Pushes the completed frames in order to the encoder
void PushCompletedFrames(GeomCacheEncoder& geomCacheEncoder);
// Update transforms
void UpdateTransformsWithErrorHandling();
typedef std::unordered_map<std::string, Alembic::AbcGeom::M44d> TMatrixMap;
typedef std::unordered_map<std::string, Alembic::AbcGeom::ObjectVisibility> TVisibilityMap;
void UpdateTransformsRec(GeomCache::Node& node, const Alembic::Abc::chrono_t frameTime,
AABB& frameAABB, QuatTNS currentTransform, TMatrixMap& matrixMap, TVisibilityMap& visibilityMap, std::string& currentObjectPath);
// Compiles a mesh or update its vertices in a job
void UpdateVertexDataWithErrorHandling(GeomCache::Mesh* mesh);
// Gets mapping from alembic face Id to material ids
std::unordered_map<uint32, uint16> GetMeshMaterialMap(const Alembic::AbcGeom::IPolyMesh& mesh, const Alembic::Abc::chrono_t frameTime);
// Generates a hash for each vertex (all frames are taken into account)
bool ComputeVertexHashes(std::vector<uint64>& abcVertexHashes, const size_t currentFrame, const size_t numAbcIndices, GeomCache::Mesh& mesh,
Alembic::Abc::TimeSampling& meshTimeSampling, size_t numMeshSamples, Alembic::AbcGeom::IPolyMeshSchema& meshSchema, const bool bHasNormals,
const bool bHasTexcoords, const bool bHasColors, const size_t numAbcNormalIndices, const size_t numAbcTexcoordsIndices, const size_t numAbcFaces);
// This is used for the first frame compilation of constant/homogeneous meshes and for each frame for heterogeneous meshes.
bool CompileFullMesh(GeomCache::Mesh& mesh, const size_t currentFrame, const QuatTNS& transform);
// Update vertex data according to the mapping table produced by CompileFullMesh at frameTime. Used for homogeneous meshes.
bool UpdateVertexData(GeomCache::Mesh& mesh, const size_t currentFrame);
// Calculate smoothed normals
void CalculateSmoothNormals(std::vector<AlembicCompilerVertex>& vertices, GeomCache::Mesh& mesh,
const Alembic::Abc::Int32ArraySample& faceCounts, const Alembic::Abc::Int32ArraySample& faceIndices,
const Alembic::Abc::P3fArraySample& positions);
// Computes tangent space, quantizes vertex positions and fills MeshData
bool CompileVertices(std::vector<AlembicCompilerVertex>& vertices, GeomCache::Mesh& mesh, GeomCache::MeshData& meshData, const bool bUpdate);
// Goes through the node tree and update the transform frame dequeues
void AppendTransformFrameDataRec(GeomCache::Node& node, const uint bufferIndex) const;
// Cleans up data structures
void Cleanup();
// The RC XML parser instance
ICryXML* m_pXMLParser;
// Cache root node
GeomCache::Node m_rootNode;
// Context
ConvertContext m_CC;
// Ref count
int m_refCount;
// Flag for 32 bit index format
bool m_b32BitIndices;
// Config flags
bool m_bConvertYUpToZUp;
bool m_bMeshPrediction;
bool m_bUseBFrames;
bool m_bPlaybackFromMemory;
uint m_indexFrameDistance;
GeomCacheFile::EBlockCompressionFormat m_blockCompressionFormat;
double m_positionPrecision;
float m_uvMax;
// Time
std::vector<Alembic::Abc::TimeSampling> m_timeSamplings;
Alembic::Abc::chrono_t m_minTime;
Alembic::Abc::chrono_t m_maxTime;
std::vector<Alembic::Abc::chrono_t> m_frameTimes;
// Stats
AZStd::atomic_long m_numVertexSplits;
LONG m_numExportedMeshes;
LONG m_numSharedMeshNodes;
// For error handling
std::string m_currentObjectPath;
// List of unique meshes
std::vector<GeomCache::Mesh*> m_meshes;
uint m_numAnimatedMeshes;
// For detecting cloned meshes
std::unordered_map<AlembicMeshDigest, std::shared_ptr<GeomCache::Mesh> > m_digestToMeshMap;
// Data for each frame processing
FrameData m_jobGroupData;
// Error count
AZStd::atomic_uint m_errorCount;
};
#endif // CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_ALEMBICCOMPILER_H
@@ -0,0 +1,58 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "ResourceCompilerABC_precompiled.h"
#include "AlembicConvertor.h"
#include "AlembicCompiler.h"
#include <AzCore/Memory/SystemAllocator.h>
AlembicConvertor::AlembicConvertor(ICryXML* pXMLParser, IPakSystem* pPakSystem)
: m_refCount(1)
, m_pXMLParser(pXMLParser)
, m_pPakSystem(pPakSystem)
{
AZ::AllocatorInstance<AZ::SystemAllocator>::Create();
}
AlembicConvertor::~AlembicConvertor()
{
if (AZ::AllocatorInstance<AZ::SystemAllocator>::IsReady())
{
AZ::AllocatorInstance<AZ::SystemAllocator>::Destroy();
}
}
void AlembicConvertor::Release()
{
if (--m_refCount <= 0)
{
delete this;
}
}
ICompiler* AlembicConvertor::CreateCompiler()
{
return new AlembicCompiler(m_pXMLParser);
}
const char* AlembicConvertor::GetExt(int index) const
{
switch (index)
{
case 0:
return "abc";
default:
return 0;
}
}
@@ -0,0 +1,41 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_ALEMBICCONVERTOR_H
#define CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_ALEMBICCONVERTOR_H
#pragma once
#include "IConvertor.h"
class ICryXML;
class AlembicConvertor
: public IConvertor
{
public:
AlembicConvertor(ICryXML* pXMLParser, IPakSystem* pPakSystem);
virtual ~AlembicConvertor();
// IConvertor methods.
virtual void Release();
virtual void DeInit() {}
virtual ICompiler* CreateCompiler();
virtual const char* GetExt(int index) const;
private:
int m_refCount;
ICryXML* m_pXMLParser;
IPakSystem* m_pPakSystem;
};
#endif // CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_ALEMBICCONVERTOR_H
@@ -0,0 +1,47 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
if(NOT PAL_TRAIT_BUILD_HOST_TOOLS)
return()
endif()
ly_get_list_relative_pal_filename(pal_dir ${CMAKE_CURRENT_LIST_DIR}/Platform/${PAL_PLATFORM_NAME})
ly_get_list_relative_pal_filename(common_dir ${CMAKE_CURRENT_LIST_DIR}/Platform/Common)
ly_add_target(
NAME ResourceCompilerABC MODULE
NAMESPACE Legacy
OUTPUT_SUBDIRECTORY rc_plugins
FILES_CMAKE
resourcecompilerabc_files.cmake
PLATFORM_INCLUDE_FILES
${pal_dir}/platform_${PAL_PLATFORM_NAME_LOWERCASE}.cmake
${common_dir}/${PAL_TRAIT_COMPILER_ID}/resourcecompilerabc_${PAL_TRAIT_COMPILER_ID_LOWERCASE}.cmake
INCLUDE_DIRECTORIES
PRIVATE
.
BUILD_DEPENDENCIES
PRIVATE
3rdParty::lz4
3rdParty::mikkelsen
3rdParty::zlib
3rdParty::zstd
Legacy::CryCommonTools
Legacy::Cry3DEngine.MeshCompiler.Static
Legacy::Cry3DEngine.CGF.Static
PUBLIC
3rdParty::alembic
Legacy::CryCommon
Legacy::ResourceCompiler.Static
RUNTIME_DEPENDENCIES
Legacy::CryXML
)
ly_add_dependencies(RC Legacy::ResourceCompilerABC)
@@ -0,0 +1,174 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#pragma once
#include "GeomCacheFileFormat.h"
#include <Alembic/AbcGeom/IPolyMesh.h> // for Alembic::AbcGeom::IPolyMesh
#include <Alembic/AbcGeom/IXform.h> // for Alembic::AbcGeom::IXform
#include <primitives.h>
#include <physinterface.h>
#include <AzCore/std/parallel/atomic.h>
#include <AzCore/std/parallel/mutex.h>
#include <unordered_set>
namespace GeomCache
{
struct MeshData
{
std::vector<GeomCacheFile::Position> m_positions;
std::vector<GeomCacheFile::Texcoords> m_texcoords;
std::vector<GeomCacheFile::QTangent> m_qTangents;
std::vector<GeomCacheFile::Color> m_reds;
std::vector<GeomCacheFile::Color> m_greens;
std::vector<GeomCacheFile::Color> m_blues;
std::vector<GeomCacheFile::Color> m_alphas;
};
struct RawMeshFrame
{
RawMeshFrame()
: m_bDone(false)
, m_bEncoded(false) {}
RawMeshFrame(RawMeshFrame&& toBeCopied)
: m_bDone(toBeCopied.m_bDone)
, m_bEncoded(toBeCopied.m_bEncoded)
, m_meshData(std::move(toBeCopied.m_meshData))
{
m_frameUseCount.store(toBeCopied.m_frameUseCount);
}
bool m_bDone;
bool m_bEncoded;
AZStd::atomic_int m_frameUseCount;
MeshData m_meshData;
};
// Data stored for each mesh
struct Mesh
{
Mesh()
: m_bUsePredictor(false)
, m_firstRawFrameIndex(0) {}
// Variance
GeomCacheFile::EStreams m_constantStreams;
GeomCacheFile::EStreams m_animatedStreams;
// Mesh hash
uint64 m_hash;
// Static mesh AABB
AABB m_aabb;
// The number of required position quantization bits for each axis
uint8 m_positionPrecision[3];
// The abs value of the upper limit of the UV range
float m_uvMax;
// Static mesh data
MeshData m_staticMeshData;
// Compile buffer
RawMeshFrame m_meshDataBuffer;
// Raw animated data frames for encoder
mutable AZStd::mutex m_rawFramesCS;
uint m_firstRawFrameIndex;
std::deque<RawMeshFrame> m_rawFrames;
// Encoded animated data frames for writer
mutable AZStd::mutex m_encodedFramesCS;
std::deque<std::vector<uint8> > m_encodedFrames;
// Material ID -> material indices. Needs to be std::map, because materials need to be sorted by their id.
std::map<uint16, std::vector<uint32> > m_indicesMap;
// Face ID -> Material ID
std::unordered_map<uint32, uint16> m_materialIdMap;
// Predictor data
std::vector<uint16> m_predictorData;
bool m_bUsePredictor;
// Compilation data
bool m_bHasNormals;
bool m_bHasTexcoords;
bool m_bHasColors;
std::string m_colorParamName;
std::vector<uint32> m_abcIndexToGeomCacheIndex; // map from alembic indices to GPU indices
Alembic::AbcGeom::IPolyMesh m_abcMesh; // The alembic poly mesh this originated from
std::vector<bool> m_reflections;
};
struct NodeData
{
bool m_bVisible;
QuatTNS m_transform;
};
// A node in the cache transform hierarchy.
// Can be a plain parent transform, a transform and mesh combined
// or a transform and a physics geometry combined.
struct Node
{
Node()
: m_type(GeomCacheFile::eNodeType_Transform)
, m_transformType(GeomCacheFile::eTransformType_Constant) {}
// Node type
GeomCacheFile::ENodeType m_type;
// Transform type
GeomCacheFile::ETransformType m_transformType;
// Static node data
NodeData m_staticNodeData;
// Compile buffer
NodeData m_nodeDataBuffer;
// Animated data frames for encoder
std::deque<NodeData> m_animatedNodeData;
// Encoded animated data frames for writer
mutable AZStd::mutex m_encodedFramesCS;
std::deque<std::vector<uint8> > m_encodedFrames;
// Mesh (if mesh node)
std::shared_ptr<Mesh> m_pMesh;
// Serialized physics geometry (if physics geometry node)
std::vector<char> m_physicsGeometry;
// Children
std::vector<std::unique_ptr<Node> > m_children;
// The alembic object
Alembic::Abc::IObject m_abcObject;
// The alembic xform stack for this node that will be merged down to one
// transform matrix in the compilation process
std::vector<Alembic::AbcGeom::IXform> m_abcXForms;
// For debug output
string m_name;
};
}
@@ -0,0 +1,121 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "ResourceCompilerABC_precompiled.h"
#include <zlib.h>
#include "GeomCacheBlockCompressor.h"
#include "zlib.h"
#include "lz4.h"
#include "lz4hc.h"
#include <zstd.h>
bool GeomCacheDeflateBlockCompressor::Compress(std::vector<char>& input, std::vector<char>& output)
{
const size_t uncompressedSize = input.size();
// Reserve buffer. zlib maximum overhead is 5 bytes per 32KB block + 6 bytes fixed.
const size_t maxCompressedSize = uncompressedSize + ((uncompressedSize / 32768) + 1) * 5 + 6;
output.resize(maxCompressedSize);
unsigned long compressedSize = 0;
// Deflate
z_stream stream;
int error;
stream.next_in = (Bytef*)&input[0];
stream.avail_in = (uInt)uncompressedSize;
stream.next_out = (Bytef*)&output[0];
stream.avail_out = (uInt)maxCompressedSize;
stream.zalloc = Z_NULL;
stream.zfree = Z_NULL;
stream.opaque = Z_NULL;
error = deflateInit2(&stream, Z_BEST_COMPRESSION, Z_DEFLATED, -MAX_WBITS, 9, Z_DEFAULT_STRATEGY);
if (error != Z_OK)
{
return false;
}
error = deflate(&stream, Z_FINISH);
if (error != Z_STREAM_END)
{
deflateEnd(&stream);
return false;
}
compressedSize = stream.total_out;
error = deflateEnd(&stream);
if (error != Z_OK)
{
return false;
}
if (compressedSize == 0)
{
return false;
}
// Resize output to final, compressed size
output.resize(compressedSize);
return true;
}
bool GeomCacheLZ4HCBlockCompressor::Compress(std::vector<char>& input, std::vector<char>& output)
{
const size_t uncompressedSize = input.size();
// Reserve compress buffer
const size_t maxCompressedSize = LZ4_compressBound(uncompressedSize);
output.resize(maxCompressedSize);
// Compress
int compressedSize = LZ4_compressHC(input.data(), output.data(), uncompressedSize);
if (compressedSize == 0)
{
return false;
}
// Resize output to final, compressed size
output.resize(compressedSize);
return true;
}
bool GeomCacheZStdBlockCompressor::Compress(std::vector<char>& input, std::vector<char>& output)
{
const size_t uncompressedSize = input.size();
// Reserve compress buffer
const size_t maxCompressedSize = ZSTD_compressBound(uncompressedSize);
output.resize(maxCompressedSize);
// Compress
int compressedSize = ZSTD_compress(output.data(), output.size(), input.data(), input.size(), 1);
if (ZSTD_isError(compressedSize))
{
return false;
}
// Resize output to final, compressed size
output.resize(compressedSize);
return true;
}
@@ -0,0 +1,61 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_GEOMCACHEBLOCKCOMPRESSOR_H
#define CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_GEOMCACHEBLOCKCOMPRESSOR_H
#pragma once
class IGeomCacheBlockCompressor
{
public:
virtual ~IGeomCacheBlockCompressor() = default;
virtual bool Compress(std::vector<char>& input, std::vector<char>& output) = 0;
};
// This just passes the input to the output vector and does no compression
class GeomCacheStoreBlockCompressor
: public IGeomCacheBlockCompressor
{
~GeomCacheStoreBlockCompressor() override = default;
virtual bool Compress(std::vector<char>& input, std::vector<char>& output) override
{
input.swap(output);
return true;
}
};
// This is the deflate compressor
class GeomCacheDeflateBlockCompressor
: public IGeomCacheBlockCompressor
{
~GeomCacheDeflateBlockCompressor() override = default;
virtual bool Compress(std::vector<char>& input, std::vector<char>& output) override;
};
// This is the LZ4 HC compressor
class GeomCacheLZ4HCBlockCompressor
: public IGeomCacheBlockCompressor
{
~GeomCacheLZ4HCBlockCompressor() override = default;
virtual bool Compress(std::vector<char>& input, std::vector<char>& output) override;
};
// This is the ZStandard compressor
class GeomCacheZStdBlockCompressor
: public IGeomCacheBlockCompressor
{
virtual bool Compress(std::vector<char>& input, std::vector<char>& output) override;
};
#endif // CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_GEOMCACHEBLOCKCOMPRESSOR_H
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,117 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_GEOMCACHEENCODER_H
#define CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_GEOMCACHEENCODER_H
#pragma once
#include "GeomCache.h"
#include "GeomCacheWriter.h"
#include "StealingThreadPool.h"
#include <AzCore/std/parallel/atomic.h>
#include <AzCore/std/parallel/condition_variable.h>
#include <AzCore/std/parallel/mutex.h>
class GeomCacheEncoder;
struct GeomCacheEncoderFrameInfo
{
GeomCacheEncoderFrameInfo(GeomCacheEncoder* pEncoder, const uint frameIndex,
const Alembic::Abc::chrono_t frameTime, const AABB& aabb, const bool bIsLastFrame)
: m_pEncoder(pEncoder)
, m_bWritten(false)
, m_encodeCountdown(0)
, m_frameIndex(frameIndex)
, m_frameTime(frameTime)
, m_frameAABB(aabb)
, m_bIsLastFrame(bIsLastFrame) {};
// The encoder that created this structure
GeomCacheEncoder* m_pEncoder;
// The ID of this frame
uint m_frameIndex;
// Frame type
GeomCacheFile::EFrameType m_frameType;
// If frame is the last one
bool m_bIsLastFrame;
// The time of this frame
Alembic::AbcCoreAbstract::chrono_t m_frameTime;
// If frame was written already
bool m_bWritten;
// If this counter reaches 0 the frame is ready to be written
uint m_encodeCountdown;
// If this counter reaches 0 the frame can be discarded
uint m_doneCountdown;
// AABB of frame
const AABB m_frameAABB;
};
class GeomCacheEncoder
{
public:
GeomCacheEncoder(GeomCacheWriter& geomCacheWriter, GeomCache::Node& rootNode,
const std::vector<GeomCache::Mesh*>& meshes, const bool bUseBFrames, const uint indexFrameDistance);
void Init();
void AddFrame(const Alembic::Abc::chrono_t frameTime, const AABB& aabb, const bool bIsLastFrame);
static bool OptimizeMeshForCompression(GeomCache::Mesh& mesh, const bool bUseMeshPrediction);
private:
GeomCacheEncoderFrameInfo& GetInfoFromFrameIndex(const uint index);
void CountNodesRec(GeomCache::Node& currentNode);
void EncodeFrame(GeomCacheEncoderFrameInfo* pFrame);
void EncodeAllMeshes(GeomCacheEncoderFrameInfo* pFrame);
void FrameEncodeFinished(GeomCacheEncoderFrameInfo* pFrame);
void EncodeNodesRec(GeomCache::Node& currentNode, GeomCacheEncoderFrameInfo* pFrame);
void EncodeNodeIFrame(const GeomCache::Node& currentNode, const GeomCache::NodeData& rawFrame, std::vector<uint8>& output);
void EncodeMesh(GeomCache::Mesh* pMesh, GeomCacheEncoderFrameInfo* pFrame);
void EncodeMeshIFrame(GeomCache::Mesh& mesh, GeomCache::RawMeshFrame& rawMeshFrame, std::vector<uint8>& output);
void EncodeMeshBFrame(GeomCache::Mesh & mesh, GeomCache::RawMeshFrame & rawMeshFrame, GeomCache::RawMeshFrame * pPrevFrames[2],
GeomCache::RawMeshFrame & floorIndexFrame, GeomCache::RawMeshFrame & ceilIndexFrame, std::vector<uint8> &output);
GeomCacheWriter& m_geomCacheWriter;
// Set to true if encoder should use bi-directional predicted frames
bool m_bUseBFrames;
uint m_indexFrameDistance;
// Number of animated nodes to compile
unsigned int m_numNodes;
// Frame data
uint m_firstInfoFrameIndex;
uint m_nextFrameIndex;
std::deque<std::unique_ptr<GeomCacheEncoderFrameInfo> > m_frames;
// Global scene structure handles from alembic compiler
GeomCache::Node& m_rootNode;
const std::vector<GeomCache::Mesh*>& m_meshes;
};
#endif // CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_GEOMCACHEENCODER_H
@@ -0,0 +1,557 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "ResourceCompilerABC_precompiled.h"
#include "GeomCacheWriter.h"
#include "GeomCacheBlockCompressor.h"
#include <GeomCacheFileFormat.h>
#include <AzCore/std/bind/bind.h>
#include <AzCore/std/chrono/types.h>
#include <AzCore/std/parallel/lock.h>
GeomCacheDiskWriteThread::GeomCacheDiskWriteThread(const string& fileName)
: m_bExit(false)
{
m_fileHandle = nullptr;
azfopen(&m_fileHandle, fileName, "w+b");
}
GeomCacheDiskWriteThread::~GeomCacheDiskWriteThread()
{
assert(m_bExit);
fclose(m_fileHandle);
}
void GeomCacheDiskWriteThread::Write(std::vector<char>& buffer, long offset, int origin)
{
if (buffer.empty() || m_bExit)
{
return;
}
fseek(m_fileHandle, offset, origin);
// Write and clear current read buffer
const size_t bufferSize = buffer.size();
m_bytesWritten += bufferSize;
size_t bytesWritten = fwrite(buffer.data(), 1, bufferSize, m_fileHandle);
//RCLog("Written %Iu/%Iu bytes with error: %d", bytesWritten, bufferSize, ferror(m_fileHandle));
}
void GeomCacheDiskWriteThread::EndThread()
{
m_bExit = true;
RCLog(" Disk write thread exited");
}
uint64 GeomCacheDiskWriteThread::GetCurrentPosition() const
{
#if defined(AZ_PLATFORM_LINUX)
off_t position;
position = ftello(m_fileHandle);
return static_cast<uint64>(position);
#else
fpos_t position;
fgetpos(m_fileHandle, &position);
return static_cast<uint64>(position);
#endif
}
GeomCacheBlockCompressionWriter::GeomCacheBlockCompressionWriter(IGeomCacheBlockCompressor* pBlockCompressor, GeomCacheDiskWriteThread& diskWriteThread)
: m_pBlockCompressor(pBlockCompressor)
, m_diskWriteThread(diskWriteThread)
{
}
GeomCacheBlockCompressionWriter::~GeomCacheBlockCompressionWriter()
{
}
void GeomCacheBlockCompressionWriter::PushData(const void* data, size_t size)
{
size_t writePosition = m_data.size();
m_data.resize(m_data.size() + size);
memcpy(&m_data[writePosition], data, size);
}
DataBlockFileInfo GeomCacheBlockCompressionWriter::WriteBlock(bool bCompress, long offset, int origin)
{
DataBlockFileInfo diskStats;
diskStats.m_size = m_data.size();
if (m_data.empty())
{
return diskStats;
}
// Fill job data
if (bCompress)
{
CompressData();
}
diskStats.m_position = m_diskWriteThread.GetCurrentPosition();
diskStats.m_size = m_data.size();
m_diskWriteThread.Write(m_data, offset, origin);
m_totalBytesWritten += diskStats.m_size;
m_data.clear();
return diskStats;
}
void GeomCacheBlockCompressionWriter::CompressData()
{
// Remember uncompressed size
const uint32 uncompressedSize = m_data.size();
// Compress job data
std::vector<char> compressedData;
m_pBlockCompressor->Compress(m_data, compressedData);
std::vector<char> dataBuffer;
dataBuffer.reserve(sizeof(GeomCacheFile::SCompressedBlockHeader) + compressedData.size());
dataBuffer.resize(sizeof(GeomCacheFile::SCompressedBlockHeader));
GeomCacheFile::SCompressedBlockHeader* pBlockHeader = reinterpret_cast<GeomCacheFile::SCompressedBlockHeader*>(dataBuffer.data());
pBlockHeader->m_uncompressedSize = uncompressedSize;
pBlockHeader->m_compressedSize = compressedData.size();
dataBuffer.insert(dataBuffer.begin() + sizeof(GeomCacheFile::SCompressedBlockHeader), compressedData.begin(), compressedData.end());
m_data = dataBuffer;
}
GeomCacheWriter::GeomCacheWriter(const string& filename, GeomCacheFile::EBlockCompressionFormat compressionFormat,
[[maybe_unused]] const uint numFrames, const bool bPlaybackFromMemory, const bool b32BitIndices)
: m_compressionFormat(compressionFormat)
, m_totalUncompressedAnimationSize(0)
{
m_animationAABB.Reset();
m_fileHeader.m_flags |= bPlaybackFromMemory ? GeomCacheFile::eFileHeaderFlags_PlaybackFromMemory : 0;
m_fileHeader.m_flags |= b32BitIndices ? GeomCacheFile::eFileHeaderFlags_32BitIndices : 0;
m_pDiskWriteThread.reset(new GeomCacheDiskWriteThread(filename));
switch (compressionFormat)
{
case GeomCacheFile::eBlockCompressionFormat_None:
m_pBlockCompressor.reset(new GeomCacheStoreBlockCompressor);
break;
case GeomCacheFile::eBlockCompressionFormat_Deflate:
m_pBlockCompressor.reset(new GeomCacheDeflateBlockCompressor);
break;
case GeomCacheFile::eBlockCompressionFormat_LZ4HC:
m_pBlockCompressor.reset(new GeomCacheLZ4HCBlockCompressor);
break;
case GeomCacheFile::eBlockCompressionFormat_ZSTD:
m_pBlockCompressor.reset(new GeomCacheZStdBlockCompressor);
break;
}
m_pCompressionWriter.reset(new GeomCacheBlockCompressionWriter(m_pBlockCompressor.get(), *m_pDiskWriteThread));
for (uint i = 0; i < m_kNumProgressStatusReports; ++i)
{
m_bShowedStatus[i] = false;
}
}
GeomCacheWriterStats GeomCacheWriter::FinishWriting()
{
GeomCacheWriterStats stats = {0};
// Write frame offsets
RCLog(" Writing frame offsets/sizes...");
if (!WriteFrameInfos())
{
m_pDiskWriteThread->EndThread();
return stats;
}
// Last write header with proper signature, AABB & static mesh data offset
m_fileHeader.m_signature = GeomCacheFile::kFileSignature;
m_fileHeader.m_totalUncompressedAnimationSize = m_totalUncompressedAnimationSize;
for (unsigned int i = 0; i < 3; ++i)
{
m_fileHeader.m_aabbMin[i] = m_animationAABB.min[i];
m_fileHeader.m_aabbMax[i] = m_animationAABB.max[i];
}
m_pCompressionWriter->PushData((void*)&m_fileHeader, sizeof(GeomCacheFile::SHeader));
uint64 currentBytesWritten = m_pCompressionWriter->GetTotalBytesWritten();
m_pCompressionWriter->WriteBlock(false, 0, SEEK_SET);
uint64 animationBytesWritten = m_pCompressionWriter->GetTotalBytesWritten() - currentBytesWritten;
// Destroy compression writer and block compressor
m_pCompressionWriter.reset(nullptr);
m_pBlockCompressor.reset(nullptr);
m_pDiskWriteThread->EndThread();
stats.m_headerDataSize = m_headerWriteSize + m_placeholderForFrameInfos.m_size + m_staticNodeDataSize;
stats.m_staticDataSize = m_staticMeshDataSize;
stats.m_animationDataSize = animationBytesWritten;
stats.m_uncompressedAnimationSize = m_totalUncompressedAnimationSize;
return stats;
}
void GeomCacheWriter::WriteStaticData(const std::vector<Alembic::Abc::chrono_t>& frameTimes, const std::vector<GeomCache::Mesh*>& meshes, const GeomCache::Node& rootNode)
{
RCLog("Writing static data to disk...");
DataBlockFileInfo stats;
// Write header with 0 signature, to avoid the engine to read incomplete caches.
// Correct signature will be written in FinishWriting at the end
m_fileHeader.m_blockCompressionFormat = m_compressionFormat;
m_fileHeader.m_numFrames = frameTimes.size();
m_pCompressionWriter->PushData((void*)&m_fileHeader, sizeof(GeomCacheFile::SHeader));
stats = m_pCompressionWriter->WriteBlock(false);
m_headerWriteSize = stats.m_size;
m_diskInfoForFrames.resize(frameTimes.size());
// Leave space for frame offsets/sizes (don't know them until frames are written)
// and pass future object to get write position for WriteFrameOffsets later
std::vector<GeomCacheFile::SFrameInfo> frameInfos(frameTimes.size());
memset(frameInfos.data(), 0, sizeof(GeomCacheFile::SFrameInfo) * frameInfos.size());
m_pCompressionWriter->PushData(frameInfos.data(), sizeof(GeomCacheFile::SFrameInfo) * frameInfos.size());
m_placeholderForFrameInfos = m_pCompressionWriter->WriteBlock(false);
// Reserve frame type array and store frame times
m_frameTypes.resize(frameTimes.size(), 0);
m_frameTimes = frameTimes;
// Write compressed physics geometries and node data
RCLog(" Writing node data");
WriteNodeStaticDataRec(rootNode, meshes);
stats = m_pCompressionWriter->WriteBlock(true);
m_staticNodeDataSize = stats.m_size;
// Write compressed static mesh data
RCLog(" Writing mesh data (%u meshes)", (uint)meshes.size());
WriteMeshesStaticData(meshes);
stats = m_pCompressionWriter->WriteBlock(true);
m_staticMeshDataSize = stats.m_size;
}
void GeomCacheWriter::WriteFrameTimes(const std::vector<Alembic::Abc::chrono_t>& frameTimes)
{
RCLog(" Writing frame times");
const uint32 numFrameTimes = frameTimes.size();
m_pCompressionWriter->PushData((void*)&numFrameTimes, sizeof(uint32));
// Convert frame times to float
std::vector<float> floatFrameTimes;
floatFrameTimes.reserve(frameTimes.size());
for (auto iter = frameTimes.begin(); iter != frameTimes.end(); ++iter)
{
floatFrameTimes.push_back((float)*iter);
}
// Write out time array
m_pCompressionWriter->PushData(floatFrameTimes.data(), sizeof(float) * floatFrameTimes.size());
}
bool GeomCacheWriter::WriteFrameInfos()
{
const size_t numFrames = m_diskInfoForFrames.size();
std::vector<GeomCacheFile::SFrameInfo> frameInfos(numFrames);
for (size_t i = 0; i < numFrames; ++i)
{
frameInfos[i].m_frameType = m_frameTypes[i];
frameInfos[i].m_frameOffset = m_diskInfoForFrames[i].m_position;
frameInfos[i].m_frameSize = m_diskInfoForFrames[i].m_size;
frameInfos[i].m_frameTime = (float)m_frameTimes[i];
if (frameInfos[i].m_frameOffset == 0 || frameInfos[i].m_frameSize == 0)
{
RCLogError("Invalid frame offset or size");
return false;
}
}
m_pCompressionWriter->PushData(frameInfos.data(), sizeof(GeomCacheFile::SFrameInfo) * frameInfos.size());
// Overwrite space left free for frame infos
m_pCompressionWriter->WriteBlock(false, (long)m_placeholderForFrameInfos.m_position, SEEK_SET);
return true;
}
void GeomCacheWriter::WriteMeshesStaticData(const std::vector<GeomCache::Mesh*>& meshes)
{
const uint32 numMeshes = meshes.size();
m_pCompressionWriter->PushData((void*)&numMeshes, sizeof(uint32));
for (auto iter = meshes.begin(); iter != meshes.end(); ++iter)
{
const GeomCache::Mesh* pMesh = *iter;
const std::string& fullName = pMesh->m_abcMesh.getFullName();
const uint nameLength = fullName.size() + 1;
GeomCacheFile::SMeshInfo meshInfo;
meshInfo.m_constantStreams = static_cast<uint8>(pMesh->m_constantStreams);
meshInfo.m_animatedStreams = static_cast<uint8>(pMesh->m_animatedStreams);
meshInfo.m_positionPrecision[0] = pMesh->m_positionPrecision[0];
meshInfo.m_positionPrecision[1] = pMesh->m_positionPrecision[1];
meshInfo.m_positionPrecision[2] = pMesh->m_positionPrecision[2];
meshInfo.m_uvMax = pMesh->m_uvMax;
meshInfo.m_numVertices = static_cast<uint32>(pMesh->m_staticMeshData.m_positions.size());
meshInfo.m_numMaterials = static_cast<uint32>(pMesh->m_indicesMap.size());
meshInfo.m_flags = pMesh->m_bUsePredictor ? GeomCacheFile::eMeshIFrameFlags_UsePredictor : 0;
meshInfo.m_nameLength = nameLength;
meshInfo.m_hash = pMesh->m_hash;
for (unsigned int i = 0; i < 3; ++i)
{
meshInfo.m_aabbMin[i] = pMesh->m_aabb.min[i];
meshInfo.m_aabbMax[i] = pMesh->m_aabb.max[i];
}
m_pCompressionWriter->PushData((void*)&meshInfo, sizeof(GeomCacheFile::SMeshInfo));
m_pCompressionWriter->PushData(fullName.c_str(), nameLength);
// Write out material IDs
for (auto iter2 = pMesh->m_indicesMap.begin(); iter2 != pMesh->m_indicesMap.end(); ++iter2)
{
const uint16 materialId = iter2->first;
m_pCompressionWriter->PushData((void*)&materialId, sizeof(uint16));
}
}
for (auto iter = meshes.begin(); iter != meshes.end(); ++iter)
{
const GeomCache::Mesh& mesh = **iter;
const GeomCacheFile::EStreams mandatoryStreams = GeomCacheFile::EStreams(GeomCacheFile::eStream_Indices | GeomCacheFile::eStream_Positions
| GeomCacheFile::eStream_Texcoords | GeomCacheFile::eStream_QTangents);
assert (((mesh.m_constantStreams | mesh.m_animatedStreams) & mandatoryStreams) == mandatoryStreams);
WriteMeshStaticData(mesh, mesh.m_constantStreams);
}
}
void GeomCacheWriter::WriteNodeStaticDataRec(const GeomCache::Node& node, const std::vector<GeomCache::Mesh*>& meshes)
{
GeomCacheFile::SNodeInfo fileNode;
fileNode.m_type = static_cast<uint8>(node.m_type);
fileNode.m_transformType = static_cast<uint16>(node.m_transformType);
fileNode.m_bVisible = node.m_staticNodeData.m_bVisible ? 1 : 0;
fileNode.m_meshIndex = std::numeric_limits<uint32>::max();
if (node.m_type == GeomCacheFile::eNodeType_Mesh)
{
auto findIter = std::find(meshes.begin(), meshes.end(), node.m_pMesh.get());
if (findIter != meshes.end())
{
fileNode.m_meshIndex = static_cast<uint32>(findIter - meshes.begin());
}
}
fileNode.m_numChildren = static_cast<uint32>(node.m_children.size());
const std::string& fullName = (node.m_abcObject != NULL) ? node.m_abcObject.getFullName() : "root";
const uint nameLength = fullName.size() + 1;
fileNode.m_nameLength = nameLength;
// Write out node infos
m_pCompressionWriter->PushData((void*)&fileNode, sizeof(GeomCacheFile::SNodeInfo));
m_pCompressionWriter->PushData(fullName.c_str(), nameLength);
// Store full initial pose. We could optimize this by leaving out animated branches without any physics proxies.
STATIC_ASSERT(sizeof(QuatTNS) == 10 * sizeof(float), "QuatTNS size should be 40 bytes");
m_pCompressionWriter->PushData((void*)&node.m_staticNodeData.m_transform, sizeof(QuatTNS));
if (node.m_type == GeomCacheFile::eNodeType_PhysicsGeometry)
{
uint32 geometrySize = node.m_physicsGeometry.size();
m_pCompressionWriter->PushData((void*)&geometrySize, sizeof(uint32));
m_pCompressionWriter->PushData((void*)node.m_physicsGeometry.data(), node.m_physicsGeometry.size());
}
for (auto iter = node.m_children.begin(); iter != node.m_children.end(); ++iter)
{
WriteNodeStaticDataRec(**iter, meshes);
}
}
void GeomCacheWriter::WriteFrame(const uint frameIndex, const AABB& frameAABB, const GeomCacheFile::EFrameType frameType,
const std::vector<GeomCache::Mesh*>& meshes, GeomCache::Node& rootNode)
{
m_animationAABB.Add(frameAABB);
std::vector<uint32> meshOffsets;
GeomCacheFile::SFrameHeader frameHeader;
STATIC_ASSERT((sizeof(GeomCacheFile::SFrameHeader) % 16) == 0, "GeomCacheFile::SFrameHeader size must be a multiple of 16");
m_frameTypes[frameIndex] = frameType;
for (unsigned int i = 0; i < 3; ++i)
{
frameHeader.m_frameAABBMin[i] = frameAABB.min[i];
frameHeader.m_frameAABBMax[i] = frameAABB.max[i];
}
GetFrameData(frameHeader, meshes);
m_pCompressionWriter->PushData((void*)&frameHeader, sizeof(GeomCacheFile::SFrameHeader));
for (auto iter = meshes.begin(); iter != meshes.end(); ++iter)
{
GeomCache::Mesh* mesh = *iter;
WriteMeshFrameData(*mesh);
}
uint32 bytesWritten = 0;
WriteNodeFrameRec(rootNode, meshes, bytesWritten);
// Pad node data to 16 bytes
std::vector<uint8> paddingData(((bytesWritten + 15) & ~15) - bytesWritten, 0);
if (paddingData.size())
{
m_pCompressionWriter->PushData(paddingData.data(), paddingData.size());
}
m_totalUncompressedAnimationSize += m_pCompressionWriter->GetCurrentDataSize();
m_diskInfoForFrames[frameIndex] = m_pCompressionWriter->WriteBlock(true);
const uint fraction = uint((float)m_kNumProgressStatusReports * ((float)(frameIndex + 1) / (float)m_fileHeader.m_numFrames));
if (fraction > 0 && !m_bShowedStatus[fraction - 1])
{
const uint percent = (100 * fraction) / (m_kNumProgressStatusReports);
RCLog(" %u%% processed", percent);
m_bShowedStatus[fraction - 1] = true;
}
}
void GeomCacheWriter::GetFrameData(GeomCacheFile::SFrameHeader& header, const std::vector<GeomCache::Mesh*>& meshes)
{
uint32 dataOffset = 0;
for (auto iter = meshes.begin(); iter != meshes.end(); ++iter)
{
const GeomCache::Mesh& mesh = **iter;
if (mesh.m_animatedStreams != 0)
{
AZStd::lock_guard<AZStd::mutex> jobLock(mesh.m_encodedFramesCS);
dataOffset += mesh.m_encodedFrames.front().size();
}
}
header.m_nodeDataOffset = dataOffset;
}
void GeomCacheWriter::WriteNodeFrameRec(GeomCache::Node& node, const std::vector<GeomCache::Mesh*>& meshes, uint32& bytesWritten)
{
AZStd::lock_guard<AZStd::mutex> jobLock(node.m_encodedFramesCS);
m_pCompressionWriter->PushData(node.m_encodedFrames.front().data(),
node.m_encodedFrames.front().size());
bytesWritten += node.m_encodedFrames.front().size();
node.m_encodedFrames.pop_front();
for (auto iter = node.m_children.begin(); iter != node.m_children.end(); ++iter)
{
WriteNodeFrameRec(**iter, meshes, bytesWritten);
}
}
void GeomCacheWriter::WriteMeshFrameData(GeomCache::Mesh& mesh)
{
if (mesh.m_animatedStreams != 0)
{
AZStd::lock_guard<AZStd::mutex> jobLock(mesh.m_encodedFramesCS);
m_pCompressionWriter->PushData(mesh.m_encodedFrames.front().data(), mesh.m_encodedFrames.front().size());
mesh.m_encodedFrames.pop_front();
}
}
void GeomCacheWriter::WriteMeshStaticData(const GeomCache::Mesh& mesh, GeomCacheFile::EStreams streamMask)
{
if (streamMask & GeomCacheFile::eStream_Indices)
{
for (auto iter = mesh.m_indicesMap.begin(); iter != mesh.m_indicesMap.end(); ++iter)
{
const std::vector<uint32>& indices = iter->second;
const uint32 numIndices = indices.size();
m_pCompressionWriter->PushData(&numIndices, sizeof(uint32));
if ((m_fileHeader.m_flags & GeomCacheFile::eFileHeaderFlags_32BitIndices) != 0)
{
m_pCompressionWriter->PushData(indices.data(), sizeof(uint32) * numIndices);
}
else
{
std::vector<uint16> indices16Bit(indices.size());
std::transform(indices.begin(), indices.end(), indices16Bit.begin(), [](uint32 index) { return static_cast<uint16>(index); });
m_pCompressionWriter->PushData(indices16Bit.data(), sizeof(uint16) * numIndices);
}
}
}
bool bWroteStream = false;
unsigned int numElements = 0;
const GeomCache::MeshData& meshData = mesh.m_staticMeshData;
if (streamMask & GeomCacheFile::eStream_Positions)
{
m_pCompressionWriter->PushData(meshData.m_positions.data(), sizeof(GeomCacheFile::Position) * meshData.m_positions.size());
bWroteStream = true;
numElements = meshData.m_positions.size();
}
if (streamMask & GeomCacheFile::eStream_Texcoords)
{
assert(!bWroteStream || meshData.m_texcoords.size() == numElements);
m_pCompressionWriter->PushData(meshData.m_texcoords.data(), sizeof(GeomCacheFile::Texcoords) * meshData.m_texcoords.size());
bWroteStream = true;
numElements = meshData.m_texcoords.size();
}
if (streamMask & GeomCacheFile::eStream_QTangents)
{
assert(!bWroteStream || meshData.m_qTangents.size() == numElements);
m_pCompressionWriter->PushData(meshData.m_qTangents.data(), sizeof(GeomCacheFile::QTangent) * meshData.m_qTangents.size());
bWroteStream = true;
numElements = meshData.m_qTangents.size();
}
if (streamMask & GeomCacheFile::eStream_Colors)
{
assert(!bWroteStream || meshData.m_reds.size() == numElements);
assert(!bWroteStream || meshData.m_greens.size() == numElements);
assert(!bWroteStream || meshData.m_blues.size() == numElements);
assert(!bWroteStream || meshData.m_alphas.size() == numElements);
m_pCompressionWriter->PushData(meshData.m_reds.data(), sizeof(GeomCacheFile::Color) * meshData.m_reds.size());
m_pCompressionWriter->PushData(meshData.m_greens.data(), sizeof(GeomCacheFile::Color) * meshData.m_greens.size());
m_pCompressionWriter->PushData(meshData.m_blues.data(), sizeof(GeomCacheFile::Color) * meshData.m_blues.size());
m_pCompressionWriter->PushData(meshData.m_alphas.data(), sizeof(GeomCacheFile::Color) * meshData.m_alphas.size());
bWroteStream = true;
numElements = meshData.m_reds.size();
}
if (mesh.m_bUsePredictor)
{
const uint32 predictorDataSize = mesh.m_predictorData.size();
m_pCompressionWriter->PushData(&predictorDataSize, sizeof(uint32));
m_pCompressionWriter->PushData(mesh.m_predictorData.data(), sizeof(uint16) * mesh.m_predictorData.size());
}
}
@@ -0,0 +1,162 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_GEOMCACHEWRITER_H
#define CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_GEOMCACHEWRITER_H
#pragma once
#include "GeomCache.h"
#include "GeomCacheBlockCompressor.h"
#include "StealingThreadPool.h"
#include <AzCore/std/parallel/atomic.h>
#include <AzCore/std/parallel/condition_variable.h>
#include <AzCore/std/parallel/mutex.h>
#include <AzCore/std/parallel/thread.h>
// This contains the position and size of a disk write
struct DataBlockFileInfo
{
DataBlockFileInfo()
: m_position(0)
, m_size(0)
{
}
DataBlockFileInfo(uint64 position, uint32 size)
: m_position(position)
, m_size(size)
{
}
uint64 m_position;
uint32 m_size;
};
// This writes the results from the compression job to disk and will
// not run in the thread pool, because it requires almost no CPU.
class GeomCacheDiskWriteThread
{
public:
GeomCacheDiskWriteThread(const string& fileName);
~GeomCacheDiskWriteThread();
// Flushes the buffers to disk and exits the thread
void EndThread();
// Write with FIFO buffering. This will acquire ownership of the buffer.
void Write(std::vector<char>& buffer, long offset = 0, int origin = SEEK_CUR);
size_t GetBytesWritten() { return m_bytesWritten; }
uint64 GetCurrentPosition() const;
private:
FILE* m_fileHandle;
bool m_bExit;
// Stats
size_t m_bytesWritten;
};
// This class will receive the data from the GeomCacheWriter.
class GeomCacheBlockCompressionWriter
{
public:
GeomCacheBlockCompressionWriter(IGeomCacheBlockCompressor* pBlockCompressor, GeomCacheDiskWriteThread& diskWriteThread);
~GeomCacheBlockCompressionWriter();
// This adds data to the current buffer.
void PushData(const void* data, size_t size);
// Compresses data in the current buffer and writes it to disk. Returns information about the disk write
DataBlockFileInfo WriteBlock(bool bCompress, long offset = 0, int origin = SEEK_CUR);
// Returns the total number of bytes written to disk. This can be less than
// the data pushed to the writer because the data written to disk may have
// been compressed
uint64 GetTotalBytesWritten() const { return m_totalBytesWritten; }
size_t GetCurrentDataSize() const { return m_data.size(); }
private:
void CompressData();
std::vector<char> m_data;
GeomCacheDiskWriteThread& m_diskWriteThread;
IGeomCacheBlockCompressor* m_pBlockCompressor;
uint64 m_totalBytesWritten;
};
struct GeomCacheWriterStats
{
uint64 m_headerDataSize;
uint64 m_staticDataSize;
uint64 m_animationDataSize;
uint64 m_uncompressedAnimationSize;
};
class GeomCacheWriter
{
public:
GeomCacheWriter(const string& filename, GeomCacheFile::EBlockCompressionFormat compressionFormat,
const uint numFrames, const bool bPlaybackFromMemory,
const bool b32BitIndices);
void WriteStaticData(const std::vector<Alembic::Abc::chrono_t>& frameTimes, const std::vector<GeomCache::Mesh*>& meshes, const GeomCache::Node& rootNode);
void WriteFrame(const uint frameIndex, const AABB& frameAABB, const GeomCacheFile::EFrameType frameType,
const std::vector<GeomCache::Mesh*>& meshes, GeomCache::Node& rootNode);
// Flush all buffers, write frame offsets and return the size of the animation stream
GeomCacheWriterStats FinishWriting();
private:
void WriteFrameTimes(const std::vector<Alembic::Abc::chrono_t>& frameTimes);
bool WriteFrameInfos();
void WriteMeshesStaticData(const std::vector<GeomCache::Mesh*>& meshes);
void WriteNodeStaticDataRec(const GeomCache::Node& node, const std::vector<GeomCache::Mesh*>& meshes);
void GetFrameData(GeomCacheFile::SFrameHeader& header, const std::vector<GeomCache::Mesh*>& meshes);
void WriteNodeFrameRec(GeomCache::Node& node, const std::vector<GeomCache::Mesh*>& meshes, uint32& bytesWritten);
void WriteMeshFrameData(GeomCache::Mesh& meshData);
void WriteMeshStaticData(const GeomCache::Mesh& meshData, GeomCacheFile::EStreams streamMask);
std::vector<DataBlockFileInfo> m_diskInfoForFrames;
std::vector<Alembic::Abc::chrono_t> m_frameTimes;
std::vector<uint> m_frameTypes;
GeomCacheFile::EBlockCompressionFormat m_compressionFormat;
GeomCacheFile::SHeader m_fileHeader;
AABB m_animationAABB;
static const uint m_kNumProgressStatusReports = 10;
bool m_bShowedStatus[m_kNumProgressStatusReports];
std::unique_ptr<IGeomCacheBlockCompressor> m_pBlockCompressor;
std::unique_ptr<GeomCacheDiskWriteThread> m_pDiskWriteThread;
std::unique_ptr<GeomCacheBlockCompressionWriter> m_pCompressionWriter;
DataBlockFileInfo m_placeholderForFrameInfos;
uint64 m_headerWriteSize;
uint64 m_staticNodeDataSize;
uint64 m_staticMeshDataSize;
uint64 m_totalUncompressedAnimationSize;
};
#endif // CRYINCLUDE_TOOLS_RC_RESOURCECOMPILERABC_GEOMCACHEWRITER_H
@@ -0,0 +1,12 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(LY_COMPILE_OPTIONS PRIVATE -fexceptions)
@@ -0,0 +1,12 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(LY_COMPILE_OPTIONS PRIVATE /EHsc)
@@ -0,0 +1,10 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
@@ -0,0 +1,16 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(LY_BUILD_DEPENDENCIES
PUBLIC
3rdParty::Qt::Core
)
@@ -0,0 +1,10 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
@@ -0,0 +1,68 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "ResourceCompilerABC_precompiled.h"
#include "IResCompiler.h"
#include "AlembicConvertor.h"
#include "IRCLog.h"
#include "CryLibrary.h"
#include "../../CryXML/ICryXML.h"
#include <AzCore/Utils/Utils.h>
#include <ResourceCompiler/ResourceCompiler.h>
extern "C" DLL_EXPORT void __stdcall RegisterConvertors(IResourceCompiler* pRC)
{
PREVENT_MODULE_AND_ENVIRONMENT_SYMBOL_STRIPPING
SetRCLog(pRC->GetIRCLog());
ICryXML* const pCryXML = LoadICryXML();
if (pCryXML == 0)
{
RCLogError("Loading xml library failed - not registering alembic converter.");
}
else
{
pRC->RegisterConvertor("AlembicCompiler", new AlembicConvertor(pCryXML, pRC->GetPakSystem()));
pRC->RegisterKey("upAxis", "[ABC] Up axis of alembic file\n"
"Z = Use Z as up axis: No conversion\n"
"Y = Use Y as up axis: Convert Y up to Z up (default)");
pRC->RegisterKey("meshPrediction", "[ABC] Use mesh prediction for index frames\n"
"0 = No mesh prediction (default)\n"
"1 = Use mesh prediction");
pRC->RegisterKey("useBFrames", "[ABC] Use bi-directional predicted frames\n"
"0 = Don't use b-frames (default)\n"
"1 = Use b-frames");
pRC->RegisterKey("indexFrameDistance", "[ABC] Index frame distance when using b-frames (default is 15)");
pRC->RegisterKey("blockCompressionFormat", "[ABC] Method used to compress data\n"
"store = No compression\n"
"deflate = Use deflate (zlib) compression (default)");
pRC->RegisterKey("playbackFromMemory", "[ABC] Set flag that resulting cache will be played back from memory\n"
"0 = Do not play back from memory (default)\n"
"1 = Cache plays from memory after loading");
pRC->RegisterKey("positionPrecision", "[ABC] Set the position precision in mm. Higher values usually result in better compression (default is 1)");
pRC->RegisterKey("uvMax", "[ABC] Set the upper value of the UV range. Values above this value will be wrapped.\n"
"0 = use detected per-mesh uvMax values. (default is 0)");
pRC->RegisterKey("skipFilesWithoutBuildConfig", "[ABC] Skip files without build configuration (.CBC)");
}
}
@@ -0,0 +1,14 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "ResourceCompilerABC_precompiled.h"
@@ -0,0 +1,70 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
// stdafx.h : include file for standard system include files,
// or project specific include files that are used frequently, but
// are changed infrequently
//
#pragma once
// When STD disables exceptions, it moves "exception" to stdext
// When this happens, alembic ends up with an underfined symbol : void Alembic::Abc::v11::ErrorHandler::operator()(class stdext::exception&, ...
// Since alembic was compiled with exceptions enabled, (instead of compile it with - D_HAS_EXCEPTIONS = 0), we need to here enable it so that
// symbol doesnt get mixed up
#undef _HAS_EXCEPTIONS
#define _HAS_EXCEPTIONS 1
#include <assert.h>
#define CRY_ASSERT(condition) assert(condition)
#define CRY_ASSERT_TRACE(condition, message) assert(condition)
#define CRY_ASSERT_MESSAGE(condition, message) assert(condition)
// Define this to prevent including CryAssert (there is no proper hook for turning this off, like the above).
#define CRYINCLUDE_CRYCOMMON_CRYASSERT_H
#include <platform.h>
#include <functional>
#include <vector>
#include <map>
#include <stdio.h>
#include <Cry_Math.h>
#include <Cry_Geo.h>
#include "CryFile.h"
#include "IXml.h"
#include "IRCLog.h"
#include <VertexFormats.h>
AZ_PUSH_DISABLE_WARNING(4996, "-Wdeprecated-declarations")
#include <Alembic/Abc/All.h>
AZ_POP_DISABLE_WARNING
#include <Alembic/AbcGeom/All.h>
#include <Alembic/AbcCoreFactory/All.h>
#include <Alembic/AbcCoreHDF5/All.h>
#include <Alembic/AbcCoreOgawa/All.h>
#include <Alembic/Util/All.h>
#include <AzCore/PlatformDef.h>
#include <AzCore/base.h>
#include "SwapEndianness.h"
@@ -0,0 +1,27 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(FILES
AlembicCompiler.cpp
AlembicConvertor.cpp
GeomCacheBlockCompressor.cpp
GeomCacheEncoder.cpp
GeomCacheWriter.cpp
AlembicCompiler.h
AlembicConvertor.h
GeomCache.h
GeomCacheBlockCompressor.h
GeomCacheEncoder.h
GeomCacheWriter.h
ResourceCompilerABC.cpp
ResourceCompilerABC_precompiled.h
ResourceCompilerABC_precompiled.cpp
)