Initial commit

This commit is contained in:
alexpete
2021-03-05 11:26:34 -08:00
commit a10351f38d
27091 changed files with 5521199 additions and 0 deletions
@@ -0,0 +1,63 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
ly_get_list_relative_pal_filename(pal_dir ${CMAKE_CURRENT_LIST_DIR}/Platform/${PAL_PLATFORM_NAME})
ly_get_list_relative_pal_filename(common_dir ${CMAKE_CURRENT_LIST_DIR}/Platform/Common)
ly_get_pal_tool_dirs(pal_tool_dirs ${CMAKE_CURRENT_LIST_DIR}/Platform)
ly_get_pal_tool_dirs(pal_tool_core_server_dirs ${CMAKE_CURRENT_LIST_DIR}/Core/Server/Platform)
include(${pal_dir}/PAL_${PAL_PLATFORM_NAME_LOWERCASE}.cmake)
if(NOT PAL_TRAIT_BUILD_CRYSCOMPILESERVER_SUPPORTED OR NOT PAL_TRAIT_BUILD_HOST_TOOLS)
return()
endif()
set(platform_tools_files)
foreach(enabled_platform ${LY_PAL_TOOLS_ENABLED})
string(TOLOWER ${enabled_platform} enabled_platform_lowercase)
ly_get_list_relative_pal_filename(pal_tool_dir ${CMAKE_CURRENT_LIST_DIR}/Platform/${enabled_platform})
list(APPEND platform_tools_files ${pal_tool_dir}/pal_tools_${enabled_platform_lowercase}.cmake)
endforeach()
ly_add_target(
NAME CrySCompileServer EXECUTABLE
NAMESPACE Legacy
FILES_CMAKE
cryscompileserver_files.cmake
PLATFORM_INCLUDE_FILES
Platform/${PAL_PLATFORM_NAME}/platform_${PAL_PLATFORM_NAME_LOWERCASE}.cmake
${platform_tools_files}
${common_dir}/${PAL_TRAIT_COMPILER_ID}/cryscompileserver_${PAL_TRAIT_COMPILER_ID_LOWERCASE}.cmake
INCLUDE_DIRECTORIES
PUBLIC
.
External
PRIVATE
${pal_tool_dirs}
${pal_tool_core_server_dirs}
BUILD_DEPENDENCIES
PRIVATE
3rdParty::zlib
AZ::AzCore
AZ::AzFramework
)
ly_add_source_properties(
SOURCES
Core/Server/CrySimpleJobCompile.cpp
CrySCompileServer.cpp
PROPERTY COMPILE_DEFINITIONS
VALUES ${LY_PAL_TOOLS_DEFINES}
)
ly_add_source_properties(
SOURCES Core/Server/CrySimpleServer.cpp
PROPERTY COMPILE_DEFINITIONS
VALUES ${LY_PAL_TOOLS_DEFINES}
)
@@ -0,0 +1,39 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_CRYSCOMPILESERVER_CORE_COMMON_H
#define CRYINCLUDE_CRYSCOMPILESERVER_CORE_COMMON_H
#pragma once
#include <AzCore/base.h>
#include <AzCore/PlatformDef.h>
#include <AzCore/PlatformIncl.h>
#if defined(AZ_PLATFORM_WINDOWS)
# if !defined(_WIN32_WINNT)
# define _WIN32_WINNT 0x0501
# endif
// Windows platform requires either a long or an unsigned long/uint64 for the
// Interlock instructions.
typedef long AtomicCountType;
#else
// Linux/Mac platforms don't support a long for the atomic types, only int32 or
// int64 (no unsigned support).
typedef int32_t AtomicCountType;
#endif
#endif // CRYINCLUDE_CRYSCOMPILESERVER_CORE_COMMON_H
@@ -0,0 +1,73 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "StdTypes.hpp"
#include "Error.hpp"
#include <tinyxml/tinyxml.h>
#include "Server/CrySimpleErrorLog.hpp"
#include "Server/CrySimpleJob.hpp"
#include <AzCore/base.h>
#include <AzCore/Debug/Trace.h>
#include <time.h>
ICryError::ICryError(EErrorType t)
: m_eType(t)
, m_numDupes(0)
{
}
void logmessage(const char* text, ...)
{
va_list arg;
va_start(arg, text);
char szBuffer[256];
char* error = szBuffer;
int bufferlen = sizeof(szBuffer) - 1;
memset(szBuffer, 0, sizeof(szBuffer));
long req = CCrySimpleJob::GlobalRequestNumber();
int ret = azsnprintf(error, bufferlen, "%8ld | ", req);
if (ret <= 0)
{
return;
}
error += ret;
bufferlen -= ret;
time_t ltime;
time(&ltime);
tm today;
#if defined(AZ_PLATFORM_WINDOWS)
localtime_s(&today, &ltime);
#else
localtime_r(&ltime, &today);
#endif
ret = (int)strftime(error, bufferlen, "%d/%m %H:%M:%S | ", &today);
if (ret <= 0)
{
return;
}
error += ret;
bufferlen -= ret;
int count = vsnprintf(error, bufferlen, text, arg);
AZ_TracePrintf(0, szBuffer);
va_end(arg);
}
@@ -0,0 +1,95 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __DXPSERROR__
#define __DXPSERROR__
#include <string>
#include <set>
#include "STLHelper.hpp"
// printf wrapper to format things nicely
void logmessage(const char* text, ...);
class ICryError
{
public:
enum EErrorType
{
SIMPLE_ERROR = 0,
COMPILE_ERROR,
};
enum EOutputFormatType
{
OUTPUT_EMAIL = 0,
OUTPUT_TTY,
OUTPUT_HASH,
};
ICryError(EErrorType t);
virtual ~ICryError() {};
EErrorType GetType() const { return m_eType; }
tdHash Hash() const { return CSTLHelper::Hash(GetErrorName() + GetErrorDetails(OUTPUT_HASH)); };
virtual bool Compare(const ICryError* err) const
{
if (GetType() != err->GetType())
{
return GetType() < err->GetType();
}
return Hash() < err->Hash();
};
virtual bool CanMerge([[maybe_unused]] const ICryError* err) const { return true; }
virtual void AddDuplicate([[maybe_unused]] ICryError* err) { m_numDupes++; }
uint32_t NumDuplicates() const { return m_numDupes; }
virtual void SetUniqueID([[maybe_unused]] int uniqueID) {}
virtual bool HasFile() const { return false; };
virtual void AddCCs([[maybe_unused]] std::set<std::string>& ccs) const {}
virtual std::string GetErrorName() const = 0;
virtual std::string GetErrorDetails(EOutputFormatType outputType) const = 0;
virtual std::string GetFilename() const { return "NoFile"; }
virtual std::string GetFileContents() const { return ""; }
private:
EErrorType m_eType;
uint32_t m_numDupes;
};
class CSimpleError
: public ICryError
{
public:
CSimpleError(const std::string& in_text)
: ICryError(SIMPLE_ERROR)
, m_text(in_text) {}
virtual ~CSimpleError() {}
virtual std::string GetErrorName() const { return m_text; };
virtual std::string GetErrorDetails([[maybe_unused]] EOutputFormatType outputType) const { return m_text; };
private:
std::string m_text;
};
#define CrySimple_ERROR(X) throw new CSimpleError(X)
#define CrySimple_SECURE_START try{
#define CrySimple_SECURE_END }catch (const ICryError* err) {printf(err->GetErrorName().c_str()); delete err; }
#endif
@@ -0,0 +1,334 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CSCMD5__
#define __CSCMD5__
/*
* This code implements the MD5 message-digest algorithm.
* The algorithm is due to Ron Rivest. This code was
* written by Colin Plumb in 1993, no copyright is claimed.
* This code is in the public domain; do with it what you wish.
*
* Equivalent code is available from RSA Data Security, Inc.
* This code has been tested against that, and is equivalent,
* except that you don't need to include two pages of legalese
* with every copy.
*
* To compute the message digest of a chunk of bytes, declare an
* MD5Context structure, pass it to MD5Init, call MD5Update as
* needed on buffers full of bytes, and then call MD5Final, which
* will fill a supplied 16-byte array with the digest.
*/
/* This code was modified in 1997 by Jim Kingdon of Cyclic Software to
not require an integer type which is exactly 32 bits. This work
draws on the changes for the same purpose by Tatu Ylonen
<ylo@cs.hut.fi> as part of SSH, but since I didn't actually use
that code, there is no copyright issue. I hereby disclaim
copyright in any changes I have made; this code remains in the
public domain. */
/* Note regarding cvs_* namespace: this avoids potential conflicts
with libraries such as some versions of Kerberos. No particular
need to worry about whether the system supplies an MD5 library, as
this file is only about 3k of object code. */
struct cvs_MD5Context
{
uint32_t buf[4];
uint32_t bits[2];
unsigned char in[64];
};
void cvs_MD5Init(struct cvs_MD5Context* context);
void cvs_MD5Update(struct cvs_MD5Context* context, unsigned char const* buf, unsigned len);
void cvs_MD5Final(unsigned char digest[16], struct cvs_MD5Context* context);
void cvs_MD5Transform(uint32_t buf[4], const unsigned char in[64]);
/* Little-endian byte-swapping routines. Note that these do not
depend on the size of datatypes such as uint32_t, nor do they require
us to detect the endianness of the machine we are running on. It
is possible they should be macros for speed, but I would be
surprised if they were a performance bottleneck for MD5. */
uint32_t getu32 (const unsigned char* addr)
{
return (((((unsigned long)addr[3] << 8) | addr[2]) << 8) | addr[1]) << 8 | addr[0];
}
void putu32(uint32_t data, unsigned char* addr)
{
addr[0] = (unsigned char)data;
addr[1] = (unsigned char)(data >> 8);
addr[2] = (unsigned char)(data >> 16);
addr[3] = (unsigned char)(data >> 24);
}
/*
* Start MD5 accumulation. Set bit count to 0 and buffer to mysterious
* initialization constants.
*/
void cvs_MD5Init(cvs_MD5Context& rCtx)
{
rCtx.buf[0] = 0x67452301;
rCtx.buf[1] = 0xefcdab89;
rCtx.buf[2] = 0x98badcfe;
rCtx.buf[3] = 0x10325476;
rCtx.bits[0] = 0;
rCtx.bits[1] = 0;
}
/*
* Update context to reflect the concatenation of another buffer full
* of bytes.
*/
void cvs_MD5Update(cvs_MD5Context& rCtx, unsigned char const* buf, uint32_t len)
{
uint32_t t;
/* Update bitcount */
t = rCtx.bits[0];
if ((rCtx.bits[0] = (t + ((uint32_t)len << 3)) & 0xffffffff) < t)
{
rCtx.bits[1]++; /* Carry from low to high */
}
rCtx.bits[1] += len >> 29;
t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */
/* Handle any leading odd-sized chunks */
if (t)
{
unsigned char* p = rCtx.in + t;
t = 64 - t;
if (len < t)
{
memcpy(p, buf, len);
return;
}
memcpy(p, buf, t);
cvs_MD5Transform (rCtx.buf, rCtx.in);
buf += t;
len -= t;
}
/* Process data in 64-byte chunks */
while (len >= 64)
{
memcpy(rCtx.in, buf, 64);
cvs_MD5Transform (rCtx.buf, rCtx.in);
buf += 64;
len -= 64;
}
/* Handle any remaining bytes of data. */
memcpy(rCtx.in, buf, len);
}
/*
* Final wrapup - pad to 64-byte boundary with the bit pattern
* 1 0* (64-bit count of bits processed, MSB-first)
*/
void cvs_MD5Final(unsigned char digest[16], cvs_MD5Context& rCtx)
{
unsigned count;
uint8_t* p;
/* Compute number of bytes mod 64 */
count = (rCtx.bits[0] >> 3) & 0x3F;
/* Set the first char of padding to 0x80. This is safe since there is
always at least one byte free */
p = rCtx.in + count;
*p++ = 0x80;
/* Bytes of padding needed to make 64 bytes */
count = 64 - 1 - count;
/* Pad out to 56 mod 64 */
if (count < 8)
{
/* Two lots of padding: Pad the first block to 64 bytes */
memset(p, 0, count);
cvs_MD5Transform (rCtx.buf, rCtx.in);
/* Now fill the next block with 56 bytes */
memset(rCtx.in, 0, 56);
}
else
{
/* Pad block to 56 bytes */
memset(p, 0, count - 8);
}
/* Append length in bits and transform */
putu32(rCtx.bits[0], rCtx.in + 56);
putu32(rCtx.bits[1], rCtx.in + 60);
cvs_MD5Transform (rCtx.buf, rCtx.in);
putu32(rCtx.buf[0], digest);
putu32(rCtx.buf[1], digest + 4);
putu32(rCtx.buf[2], digest + 8);
putu32(rCtx.buf[3], digest + 12);
//memset(&rCtx,0,sizeof(rCtx)); // In case it's sensitive
}
/* The four core functions - F1 is optimized somewhat */
/* #define F1(x, y, z) (x & y | ~x & z) */
#define F1(x, y, z) (z ^ (x & (y ^ z)))
#define F2(x, y, z) F1(z, x, y)
#define F3(x, y, z) (x ^ y ^ z)
#define F4(x, y, z) (y ^ (x | ~z))
/* This is the central step in the MD5 algorithm. */
#define MD5STEP(f, w, x, y, z, data, s) \
(w += f(x, y, z) + data, w &= 0xffffffff, w = w << s | w >> (32 - s), w += x)
/*
* The core of the MD5 algorithm, this alters an existing MD5 hash to
* reflect the addition of 16 longwords of new data. MD5Update blocks
* the data and converts bytes into longwords for this routine.
*/
void cvs_MD5Transform(uint32_t buf[4], const unsigned char inraw[64])
{
uint32_t a, b, c, d;
uint32_t in[16];
int i;
for (i = 0; i < 16; ++i)
{
in[i] = getu32 (inraw + 4 * i);
}
a = buf[0];
b = buf[1];
c = buf[2];
d = buf[3];
MD5STEP(F1, a, b, c, d, in[ 0] + 0xd76aa478, 7);
MD5STEP(F1, d, a, b, c, in[ 1] + 0xe8c7b756, 12);
MD5STEP(F1, c, d, a, b, in[ 2] + 0x242070db, 17);
MD5STEP(F1, b, c, d, a, in[ 3] + 0xc1bdceee, 22);
MD5STEP(F1, a, b, c, d, in[ 4] + 0xf57c0faf, 7);
MD5STEP(F1, d, a, b, c, in[ 5] + 0x4787c62a, 12);
MD5STEP(F1, c, d, a, b, in[ 6] + 0xa8304613, 17);
MD5STEP(F1, b, c, d, a, in[ 7] + 0xfd469501, 22);
MD5STEP(F1, a, b, c, d, in[ 8] + 0x698098d8, 7);
MD5STEP(F1, d, a, b, c, in[ 9] + 0x8b44f7af, 12);
MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17);
MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22);
MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7);
MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12);
MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17);
MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22);
MD5STEP(F2, a, b, c, d, in[ 1] + 0xf61e2562, 5);
MD5STEP(F2, d, a, b, c, in[ 6] + 0xc040b340, 9);
MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14);
MD5STEP(F2, b, c, d, a, in[ 0] + 0xe9b6c7aa, 20);
MD5STEP(F2, a, b, c, d, in[ 5] + 0xd62f105d, 5);
MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9);
MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14);
MD5STEP(F2, b, c, d, a, in[ 4] + 0xe7d3fbc8, 20);
MD5STEP(F2, a, b, c, d, in[ 9] + 0x21e1cde6, 5);
MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9);
MD5STEP(F2, c, d, a, b, in[ 3] + 0xf4d50d87, 14);
MD5STEP(F2, b, c, d, a, in[ 8] + 0x455a14ed, 20);
MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5);
MD5STEP(F2, d, a, b, c, in[ 2] + 0xfcefa3f8, 9);
MD5STEP(F2, c, d, a, b, in[ 7] + 0x676f02d9, 14);
MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20);
MD5STEP(F3, a, b, c, d, in[ 5] + 0xfffa3942, 4);
MD5STEP(F3, d, a, b, c, in[ 8] + 0x8771f681, 11);
MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16);
MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23);
MD5STEP(F3, a, b, c, d, in[ 1] + 0xa4beea44, 4);
MD5STEP(F3, d, a, b, c, in[ 4] + 0x4bdecfa9, 11);
MD5STEP(F3, c, d, a, b, in[ 7] + 0xf6bb4b60, 16);
MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23);
MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4);
MD5STEP(F3, d, a, b, c, in[ 0] + 0xeaa127fa, 11);
MD5STEP(F3, c, d, a, b, in[ 3] + 0xd4ef3085, 16);
MD5STEP(F3, b, c, d, a, in[ 6] + 0x04881d05, 23);
MD5STEP(F3, a, b, c, d, in[ 9] + 0xd9d4d039, 4);
MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11);
MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16);
MD5STEP(F3, b, c, d, a, in[ 2] + 0xc4ac5665, 23);
MD5STEP(F4, a, b, c, d, in[ 0] + 0xf4292244, 6);
MD5STEP(F4, d, a, b, c, in[ 7] + 0x432aff97, 10);
MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15);
MD5STEP(F4, b, c, d, a, in[ 5] + 0xfc93a039, 21);
MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6);
MD5STEP(F4, d, a, b, c, in[ 3] + 0x8f0ccc92, 10);
MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15);
MD5STEP(F4, b, c, d, a, in[ 1] + 0x85845dd1, 21);
MD5STEP(F4, a, b, c, d, in[ 8] + 0x6fa87e4f, 6);
MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10);
MD5STEP(F4, c, d, a, b, in[ 6] + 0xa3014314, 15);
MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21);
MD5STEP(F4, a, b, c, d, in[ 4] + 0xf7537e82, 6);
MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10);
MD5STEP(F4, c, d, a, b, in[ 2] + 0x2ad7d2bb, 15);
MD5STEP(F4, b, c, d, a, in[ 9] + 0xeb86d391, 21);
buf[0] += a;
buf[1] += b;
buf[2] += c;
buf[3] += d;
}
/*
#include <stdio.h>
int
main (int argc, char **argv)
{
struct cvs_MD5Context context;
unsigned char checksum[16];
int i;
int j;
if (argc < 2)
{
fprintf (stderr, "usage: %s string-to-hash\n", argv[0]);
exit (1);
}
for (j = 1; j < argc; ++j)
{
printf ("MD5 (\"%s\") = ", argv[j]);
cvs_MD5Init (&context);
cvs_MD5Update (&context, argv[j], strlen (argv[j]));
cvs_MD5Final (checksum, &context);
for (i = 0; i < 16; i++)
{
printf ("%02x", (unsigned int) checksum[i]);
}
printf ("\n");
}
return 0;
}
*/
#endif
@@ -0,0 +1,420 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "Mailer.h"
#include "WindowsAPIImplementation.h"
#include <AzCore/PlatformDef.h>
#include <assert.h>
#if defined(AZ_PLATFORM_MAC)
#include <netdb.h>
#include <unistd.h>
#elif defined(AZ_PLATFORM_WINDOWS)
#include <ws2tcpip.h>
#endif
#include <AzCore/base.h>
#include <AzCore/IO/SystemFile.h>
#pragma comment(lib,"ws2_32.lib")
namespace // helpers
{
static const char cb64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
void Base64EncodeBlock(const unsigned char* in, unsigned char* out)
{
out[0] = cb64[in[0] >> 2];
out[1] = cb64[((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4)];
out[2] = cb64[((in[1] & 0x0f) << 2) | ((in[2] & 0xc0) >> 6)];
out[3] = cb64[in[2] & 0x3f];
}
void Base64EncodeBlock(const unsigned char* in, unsigned char* out, int len)
{
out[0] = cb64[in[0] >> 2];
out[1] = cb64[((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4)];
out[2] = (unsigned char) (len > 1 ? cb64[((in[1] & 0x0f) << 2) | ((in[2] & 0xc0) >> 6)] : '=');
out[3] = (unsigned char) (len > 2 ? cb64[in[2] & 0x3f] : '=');
}
void Base64Encode(const unsigned char* pSrc, const size_t srcLen, unsigned char* pDst, [[maybe_unused]] const size_t dstLen)
{
assert(dstLen >= 4 * ((srcLen + 2) / 3));
size_t len = srcLen;
for (; len > 2; len -= 3, pSrc += 3, pDst += 4)
{
Base64EncodeBlock(pSrc, pDst);
}
if (len > 0)
{
unsigned char in[3];
in[0] = pSrc[0];
in[1] = len > 1 ? pSrc[1] : 0;
in[2] = 0;
Base64EncodeBlock(in, pDst, (int) len);
}
}
std::string Base64EncodeString(const std::string& in)
{
const size_t srcLen = in.size();
const size_t dstLen = 4 * ((srcLen + 2) / 3);
std::string out(dstLen, 0);
Base64Encode((const unsigned char*) in.c_str(), srcLen, (unsigned char*) out.c_str(), dstLen);
return out;
}
const char* ExtractFileName(const char* filepath)
{
for (const char* p = filepath + strlen(filepath) - 1; p >= filepath; --p)
{
if (*p == '\\' || *p == '/')
{
return p + 1;
}
}
return filepath;
}
}
AZStd::atomic_long CSMTPMailer::ms_OpenSockets = {0};
CSMTPMailer::CSMTPMailer(const tstr& username, const tstr& password, const tstr& server, int port)
: m_server(server)
, m_username(username)
, m_password(password)
, m_port(port)
, m_winSockAvail(false)
, m_response()
{
#if defined(AZ_PLATFORM_WINDOWS)
WSADATA wd;
m_winSockAvail = WSAStartup(MAKEWORD(1, 1), &wd) == 0;
if (!m_winSockAvail)
{
m_response += "Error: Unable to initialize WinSock 1.1\n";
}
#endif
}
CSMTPMailer::~CSMTPMailer()
{
#if defined(AZ_PLATFORM_WINDOWS)
if (m_winSockAvail)
{
WSACleanup();
}
#endif
}
void CSMTPMailer::ReceiveLine(SOCKET connection)
{
char buf[1025];
int ret = recv(connection, buf, sizeof(buf) - 1, 0);
if (ret == SOCKET_ERROR)
{
ret = azsnprintf(buf, sizeof(buf), "Error: WinSock error %d during recv()\n", WSAGetLastError());
if (ret == sizeof(buf) || ret < 0)
{
buf[sizeof(buf) - 1] = '\0';
}
}
else
{
buf[ret] = 0;
}
m_response += buf;
}
void CSMTPMailer::SendLine(SOCKET connection, const char* format, ...) const
{
char buf[2049];
va_list args;
va_start(args, format);
int len = azvsnprintf(buf, sizeof(buf), format, args);
if (len == sizeof(buf) || len < 0)
{
buf[sizeof(buf) - 1] = '\0';
len = sizeof(buf) - 1;
}
va_end(args);
send(connection, buf, len, 0);
}
void CSMTPMailer::SendRaw(SOCKET connection, const char* data, size_t dataLen) const
{
send(connection, data, (int) dataLen, 0);
}
void CSMTPMailer::SendFile(SOCKET connection, const tattachment& filepath, const char* boundary) const
{
AZ::IO::SystemFile inputFile;
const bool wasSuccessful = inputFile.Open(filepath.second.c_str(), AZ::IO::SystemFile::SF_OPEN_READ_ONLY);
if (wasSuccessful)
{
SendLine(connection, "--%s\r\n", boundary);
SendLine(connection, "Content-Type: application/octet-stream\r\n");
SendLine(connection, "Content-Transfer-Encoding: base64\r\n");
SendLine(connection, "Content-Disposition: attachment; filename=\"%s\"\r\n", filepath.first.c_str());
SendLine(connection, "\r\n");
AZ::IO::SystemFile::SizeType fileSize = inputFile.Length();
while (fileSize)
{
const int DEF_BLOCK_SIZE = 128; // 72
char in[3 * DEF_BLOCK_SIZE];
size_t blockSize = fileSize > sizeof(in) ? sizeof(in) : fileSize;
inputFile.Read(blockSize, in);
char out[4 * DEF_BLOCK_SIZE];
Base64Encode((unsigned char*) in, blockSize, (unsigned char*) out, sizeof(out));
SendRaw(connection, out, 4 * ((blockSize + 2) / 3));
SendLine(connection, "\r\n"); // seems to get sent faster if you split up the data lines
fileSize -= blockSize;
}
}
}
SOCKET CSMTPMailer::Open(const char* host, unsigned short port, sockaddr_in& serverAddress)
{
SOCKET connection = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (connection == INVALID_SOCKET)
{
m_response += "Error: Failed to create socket\n";
return 0;
}
struct addrinfo* addressInfo{};
char portBuffer[16];
struct addrinfo hints{};
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
azsnprintf(portBuffer, AZStd::size(portBuffer), "%hu", port);
int addrInfoReturnCode = getaddrinfo(host, portBuffer, &hints, &addressInfo);
if(addrInfoReturnCode != 0)
{
char buf[1025];
int ret = azsnprintf(buf, sizeof(buf), "Error: Host %s not found\n", host);
if (ret == sizeof(buf) || ret < 0)
{
buf[sizeof(buf) - 1] = '\0';
}
m_response += buf;
closesocket(connection);
return 0;
}
if (addressInfo)
{
++ms_OpenSockets;
serverAddress = *reinterpret_cast<sockaddr_in*>(addressInfo->ai_addr);
}
return connection;
}
void CSMTPMailer::AddReceivers(SOCKET connection, const tstrcol& receivers)
{
for (tstrcol::const_iterator it = receivers.begin(), itEnd = receivers.end(); it != itEnd; ++it)
{
if (!(*it).empty())
{
SendLine(connection, "rcpt to: %s\r\n", (*it).c_str());
ReceiveLine(connection);
}
}
}
void CSMTPMailer::AssignReceivers(SOCKET connection, const char* receiverTag, const tstrcol& receivers)
{
tstrcol::const_iterator it = receivers.begin();
tstrcol::const_iterator itEnd = receivers.end();
while (it != itEnd && (*it).empty())
{
++it;
}
if (it != itEnd)
{
tstr out(receiverTag);
out += *it;
++it;
for (; it != itEnd; ++it)
{
if (!(*it).empty())
{
out += "; ";
out += *it;
}
}
out += "\r\n";
SendLine(connection, out.c_str());
}
}
void CSMTPMailer::SendAttachments(SOCKET connection, const tattachlist& attachments, const char* boundary)
{
for (tattachlist::const_iterator it = attachments.begin(), itEnd = attachments.end(); it != itEnd; ++it)
{
if (!(*it).first.empty() && !(*it).second.empty())
{
SendFile(connection, *it, boundary);
}
}
}
bool CSMTPMailer::IsEmpty(const tstrcol& col) const
{
if (!col.empty())
{
for (tstrcol::const_iterator it = col.begin(), itEnd = col.end(); it != itEnd; ++it)
{
if (!(*it).empty())
{
return false;
}
}
}
return true;
}
bool CSMTPMailer::Send(const tstr& from, const tstrcol& to, const tstrcol& cc, const tstrcol& bcc, const tstr& subject, const tstr& body, const tattachlist& attachments)
{
if (!m_winSockAvail)
{
return false;
}
if (from.empty() || IsEmpty(to))
{
return false;
}
sockaddr_in serverAddress;
SOCKET connection = Open(m_server.c_str(), m_port, serverAddress); // SMTP telnet (usually port 25)
if (connection == INVALID_SOCKET)
{
return false;
}
if (connect(connection, (sockaddr*) &serverAddress, sizeof(serverAddress)) != SOCKET_ERROR)
{
ReceiveLine(connection);
SendLine(connection, "helo localhost\r\n");
ReceiveLine(connection);
if (!m_username.empty() && !m_password.empty())
{
SendLine(connection, "auth login\r\n"); // most servers should implement this (todo: otherwise fall back to PLAIN or CRAM-MD5 (requiring EHLO))
ReceiveLine(connection);
SendLine(connection, "%s\r\n", Base64EncodeString(m_username).c_str());
ReceiveLine(connection);
SendLine(connection, "%s\r\n", Base64EncodeString(m_password).c_str());
ReceiveLine(connection);
}
SendLine(connection, "mail from: %s\r\n", from.c_str());
ReceiveLine(connection);
AddReceivers(connection, to);
AddReceivers(connection, cc);
AddReceivers(connection, bcc);
SendLine(connection, "data\r\n");
ReceiveLine(connection);
SendLine(connection, "From: %s\r\n", from.c_str());
AssignReceivers(connection, "To: ", to);
AssignReceivers(connection, "Cc: ", cc);
AssignReceivers(connection, "Bcc: ", bcc);
SendLine(connection, "Subject: %s\r\n", subject.c_str());
static const char boundary[] = "------a95ed0b485e4a9b0fd4ff93f50ad06ca"; // beware, boundary should not clash with text content of message body!
SendLine(connection, "MIME-Version: 1.0\r\n");
SendLine(connection, "Content-Type: multipart/mixed; boundary=\"%s\"\r\n", boundary);
SendLine(connection, "\r\n");
SendLine(connection, "This is a multi-part message in MIME format.\r\n");
SendLine(connection, "--%s\r\n", boundary);
SendLine(connection, "Content-Type: text/plain; charset=iso-8859-1; format=flowed\r\n"); // the used charset should support the commonly used special characters of western languages
SendLine(connection, "Content-Transfer-Encoding: 7bit\r\n");
SendLine(connection, "\r\n");
SendRaw(connection, body.c_str(), body.size());
SendLine(connection, "\r\n");
SendAttachments(connection, attachments, boundary);
SendLine(connection, "--%s--\r\n", boundary);
SendLine(connection, "\r\n.\r\n");
ReceiveLine(connection);
SendLine(connection, "quit\r\n");
ReceiveLine(connection);
}
else
{
char buf[1025];
int ret = azsnprintf(buf, sizeof(buf), "Error: Failed to connect to %s:%d\n", m_server.c_str(), m_port);
if (ret == sizeof(buf) || ret < 0)
{
buf[sizeof(buf) - 1] = '\0';
}
m_response += buf;
return false;
}
closesocket(connection);
--ms_OpenSockets;
return true;
}
const char* CSMTPMailer::GetResponse() const
{
return m_response.c_str();
}
@@ -0,0 +1,72 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef CRYINCLUDE_CRYSCOMPILESERVER_CORE_MAILER_H
#define CRYINCLUDE_CRYSCOMPILESERVER_CORE_MAILER_H
#pragma once
#include "Common.h"
#include "Server/CrySimpleSock.hpp"
#include <string>
#include <set>
#include <list>
#include <memory>
#include <AzCore/std/parallel/atomic.h>
class CSMTPMailer
{
public:
typedef std::string tstr;
typedef std::set<tstr> tstrcol;
typedef std::pair<std::string, std::string> tattachment;
typedef std::list<tattachment> tattachlist;
static const int DEFAULT_PORT = 25;
static AZStd::atomic_long ms_OpenSockets;
public:
CSMTPMailer(const tstr& username, const tstr& password, const tstr& server, int port = DEFAULT_PORT);
~CSMTPMailer();
bool Send(const tstr& from, const tstrcol& to, const tstrcol& cc, const tstrcol& bcc, const tstr& subject, const tstr& body, const tattachlist& attachments);
const char* GetResponse() const;
static long GetOpenSockets() { return ms_OpenSockets; }
private:
void ReceiveLine(SOCKET connection);
void SendLine(SOCKET connection, const char* format, ...) const;
void SendRaw(SOCKET connection, const char* data, size_t dataLen) const;
void SendFile(SOCKET connection, const tattachment& file, const char* boundary) const;
SOCKET Open(const char* host, unsigned short port, sockaddr_in& serverAddress);
void AddReceivers(SOCKET connection, const tstrcol& receivers);
void AssignReceivers(SOCKET connection, const char* receiverTag, const tstrcol& receivers);
void SendAttachments(SOCKET connection, const tattachlist& attachments, const char* boundary);
bool IsEmpty(const tstrcol& col) const;
private:
std::unique_ptr<CCrySimpleSock> m_socket;
tstr m_server;
tstr m_username;
tstr m_password;
int m_port;
bool m_winSockAvail;
tstr m_response;
};
#endif // CRYINCLUDE_CRYSCOMPILESERVER_CORE_MAILER_H
@@ -0,0 +1,344 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "StdTypes.hpp"
#include "Error.hpp"
#include "STLHelper.hpp"
#include <AzCore/PlatformDef.h>
#include <AzCore/PlatformIncl.h>
#include <AzCore/Debug/Trace.h>
#include <AzCore/IO/SystemFile.h>
#include <AzCore/Casting/lossy_cast.h>
#if defined(AZ_PLATFORM_WINDOWS)
#include <io.h>
#endif
#include "MD5.hpp"
#include <assert.h>
#include <zlib.h>
#include <algorithm>
#include <functional>
#include <iterator>
#include <cstdio>
void CSTLHelper::Log(const std::string& rLog)
{
const std::string Output = rLog + "\n";
logmessage(Output.c_str());
}
void CSTLHelper::Tokenize(tdEntryVec& rRet, const std::string& Tokens, const std::string& Separator)
{
rRet.clear();
std::string::size_type Pt;
std::string::size_type Start = 0;
std::string::size_type SSize = Separator.size();
while ((Pt = Tokens.find(Separator, Start)) != std::string::npos)
{
std::string SubStr = Tokens.substr(Start, Pt - Start);
rRet.push_back(SubStr);
Start = Pt + SSize;
}
rRet.push_back(Tokens.substr(Start));
}
void CSTLHelper::Replace(std::string& rRet, const std::string& rSrc, const std::string& rToReplace, const std::string& rReplacement)
{
std::vector<uint8_t> Out;
std::vector<uint8_t> In(rSrc.c_str(), rSrc.c_str() + rSrc.size() + 1);
Replace(Out, In, rToReplace, rReplacement);
rRet = std::string(reinterpret_cast<char*>(&Out[0]));
}
void CSTLHelper::Replace(std::vector<uint8_t>& rRet, const std::vector<uint8_t>& rTokenSrc, const std::string& rToReplace, const std::string& rReplacement)
{
rRet.clear();
size_t SSize = rToReplace.size();
for (size_t a = 0, Size = rTokenSrc.size(); a < Size; a++)
{
if (a + SSize < Size && strncmp((const char*)&rTokenSrc[a], rToReplace.c_str(), SSize) == 0)
{
for (size_t b = 0, RSize = rReplacement.size(); b < RSize; b++)
{
rRet.push_back(rReplacement.c_str()[b]);
}
a += SSize - 1;
}
else
{
rRet.push_back(rTokenSrc[a]);
}
}
}
tdToken CSTLHelper::SplitToken(const std::string& rToken, const std::string& rSeparator)
{
#undef min
using namespace std;
string Token;
Remove(Token, rToken, ' ');
string::size_type Pt = Token.find(rSeparator);
return tdToken(Token.substr(0, Pt), Token.substr(std::min(Pt + 1, Token.size())));
}
void CSTLHelper::Splitizer(tdTokenList& rTokenList, const tdEntryVec& rFilter, const std::string& rSeparator)
{
rTokenList.clear();
for (size_t a = 0, Size = rFilter.size(); a < Size; a++)
{
rTokenList.push_back(SplitToken(rFilter[a], rSeparator));
}
}
void CSTLHelper::Trim(std::string& rStr, const std::string& charsToTrim)
{
std::string::size_type Pt1 = rStr.find_first_not_of(charsToTrim);
if (Pt1 == std::string::npos)
{
// At this point the string could be empty or it could only contain 'charsToTrim' characters.
// In case it's the later then trim should be applied by leaving the string empty.
rStr = "";
return;
}
std::string::size_type Pt2 = rStr.find_last_not_of(charsToTrim) + 1;
Pt2 = Pt2 - Pt1;
rStr = rStr.substr(Pt1, Pt2);
}
void CSTLHelper::Remove(std::string& rTokenDst, const std::string& rTokenSrc, const char C)
{
using namespace std;
AZ_PUSH_DISABLE_WARNING(4996, "-Wdeprecated-declarations")
remove_copy_if(rTokenSrc.begin(), rTokenSrc.end(), back_inserter(rTokenDst), [C](char token) { return token == C; });
AZ_POP_DISABLE_WARNING
}
bool CSTLHelper::ToFile(const std::string& rFileName, const std::vector<uint8_t>& rOut)
{
if (rOut.size() == 0)
{
return false;
}
AZ::IO::SystemFile outputFile;
const bool wasSuccessful = outputFile.Open(rFileName.c_str(), AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY | AZ::IO::SystemFile::SF_OPEN_CREATE);
if (wasSuccessful == false)
{
AZ_Error("ShaderCompiler", wasSuccessful, "CSTLHelper::ToFile Could not create file: %s", rFileName.c_str());
return false;
}
outputFile.Write(&rOut[0], rOut.size());
return true;
}
bool CSTLHelper::FromFile(const std::string& rFileName, std::vector<uint8_t>& rIn)
{
AZ::IO::SystemFile inputFile;
bool wasSuccess = inputFile.Open(rFileName.c_str(), AZ::IO::SystemFile::SF_OPEN_READ_WRITE);
if (!wasSuccess)
{
return false;
}
AZ::IO::SystemFile::SizeType fileSize = inputFile.Length();
if (fileSize <= 0)
{
return false;
}
size_t nNumRead = 0;
rIn.resize(fileSize);
AZ::IO::SystemFile::SizeType actualReadAmount = inputFile.Read(fileSize, &rIn[0]);
return actualReadAmount == fileSize;
}
bool CSTLHelper::ToFileCompressed(const std::string& rFileName, const std::vector<uint8_t>& rOut)
{
std::vector<uint8_t> buf;
unsigned long sourceLen = (unsigned long)rOut.size();
unsigned long destLen = compressBound(sourceLen) + 16;
buf.resize(destLen);
compress(buf.data(), &destLen, &rOut[0], sourceLen);
if (destLen > 0)
{
AZ::IO::SystemFile outputFile;
const bool wasSuccessful = outputFile.Open(rFileName.c_str(), AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY | AZ::IO::SystemFile::SF_OPEN_CREATE);
AZ_Error("ShaderCompiler", wasSuccessful, "Could not create compressed file: %s", rFileName.c_str());
if (wasSuccessful == false)
{
return false;
}
AZ::IO::SystemFile::SizeType bytesWritten = outputFile.Write(&sourceLen, sizeof(sourceLen));
AZ_Error("ShaderCompiler", bytesWritten == sizeof(sourceLen), "Could not save out size of compressed data to %s", rFileName.c_str());
if (bytesWritten != sizeof(sourceLen))
{
return false;
}
bytesWritten = outputFile.Write(buf.data(), destLen);
AZ_Error("ShaderCompiler", bytesWritten == destLen, "Could not save out compressed data to %s", rFileName.c_str());
if (bytesWritten != destLen)
{
return false;
}
return true;
}
else
{
return false;
}
}
bool CSTLHelper::FromFileCompressed(const std::string& rFileName, std::vector<uint8_t>& rIn)
{
std::vector<uint8_t> buf;
AZ::IO::SystemFile inputFile;
const bool wasSuccessful = inputFile.Open(rFileName.c_str(), AZ::IO::SystemFile::SF_OPEN_READ_ONLY);
AZ_Error("ShaderCompiler", wasSuccessful, "Could not read: ", rFileName.c_str());
if (!wasSuccessful)
{
return false;
}
AZ::IO::SystemFile::SizeType FileLen = inputFile.Length();
AZ_Error("ShaderCompiler", FileLen > 0, "Error getting file-size of ", rFileName.c_str());
if (FileLen <= 0)
{
return false;
}
unsigned long uncompressedLen = 0;
// Possible, expected, loss of data from u64 to u32. Zlib supports only unsigned long
unsigned long sourceLen = azlossy_caster((FileLen - 4));
buf.resize(sourceLen);
AZ::IO::SystemFile::SizeType bytesReadIn = inputFile.Read(sizeof(uncompressedLen), &uncompressedLen);
AZ_Warning("ShaderCompiler", bytesReadIn == sizeof(uncompressedLen), "Expected to read in %d but read in %d from file %s", sizeof(uncompressedLen), bytesReadIn, rFileName.c_str());
bytesReadIn = inputFile.Read(buf.size(), buf.data());
AZ_Warning("ShaderCompiler", bytesReadIn == buf.size(), "Expected to read in %d but read in %d from file %s", buf.size(), bytesReadIn, rFileName.c_str());
unsigned long nUncompressedBytes = uncompressedLen;
rIn.resize(uncompressedLen);
int nRes = uncompress(rIn.data(), &nUncompressedBytes, buf.data(), sourceLen);
return nRes == Z_OK && nUncompressedBytes == uncompressedLen;
}
//////////////////////////////////////////////////////////////////////////
bool CSTLHelper::AppendToFile(const std::string& rFileName, const std::vector<uint8_t>& rOut)
{
AZ::IO::SystemFile outputFile;
int openMode = AZ::IO::SystemFile::SF_OPEN_APPEND;
if (!AZ::IO::SystemFile::Exists(rFileName.c_str()))
{
openMode = AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY;
}
const bool wasSuccessful = outputFile.Open(rFileName.c_str(), openMode);
AZ_Error("ShaderCompiler", wasSuccessful, "Could not open file for appending: %s", rFileName.c_str());
if (wasSuccessful == false)
{
return false;
}
AZ::IO::SystemFile::SizeType bytesWritten = outputFile.Write(rOut.data(), rOut.size());
AZ_Warning("ShaderCompiler", bytesWritten == rOut.size(), "Did not write out all the data to the file: %s", rFileName.c_str());
return true;
}
//////////////////////////////////////////////////////////////////////////
tdHash CSTLHelper::Hash(const uint8_t* pData, const size_t Size)
{
tdHash CheckSum;
cvs_MD5Context MD5Context;
cvs_MD5Init(MD5Context);
cvs_MD5Update(MD5Context, pData, static_cast<uint32_t>(Size));
cvs_MD5Final(CheckSum.hash, MD5Context);
return CheckSum;
}
static char C2A[17] = "0123456789ABCDEF";
std::string CSTLHelper::Hash2String(const tdHash& rHash)
{
std::string Ret;
for (size_t a = 0, Size = std::min<size_t>(sizeof(rHash.hash), 16u); a < Size; a++)
{
const uint8_t C1 = rHash[a] & 0xf;
const uint8_t C2 = rHash[a] >> 4;
Ret += C2A[C1];
Ret += C2A[C2];
}
return Ret;
}
tdHash CSTLHelper::String2Hash(const std::string& rStr)
{
assert(rStr.size() == 32);
tdHash Ret;
for (size_t a = 0, Size = std::min<size_t>(rStr.size(), 32u); a < Size; a += 2)
{
const uint8_t C1 = rStr.c_str()[a];
const uint8_t C2 = rStr.c_str()[a + 1];
Ret[a >> 1] = C1 - (C1 >= '0' && C1 <= '9' ? '0' : 'A' - 10);
Ret[a >> 1] |= (C2 - (C2 >= '0' && C2 <= '9' ? '0' : 'A' - 10)) << 4;
}
return Ret;
}
//////////////////////////////////////////////////////////////////////////
bool CSTLHelper::Compress(const std::vector<uint8_t>& rIn, std::vector<uint8_t>& rOut)
{
unsigned long destLen, sourceLen = (unsigned long)rIn.size();
destLen = compressBound(sourceLen) + 16;
rOut.resize(destLen + 4);
compress(&rOut[4], &destLen, &rIn[0], sourceLen);
rOut.resize(destLen + 4);
*(uint32_t*)(&rOut[0]) = sourceLen;
return true;
}
bool CSTLHelper::Uncompress(const std::vector<uint8_t>& rIn, std::vector<uint8_t>& rOut)
{
unsigned long sourceLen = (unsigned long)rIn.size() - 4;
unsigned long nUncompressed = *(uint32_t*)(&rIn[0]);
unsigned long nUncompressedBytes = nUncompressed;
rOut.resize(nUncompressed);
int nRes = uncompress(&rOut[0], &nUncompressedBytes, &rIn[4], sourceLen);
return nRes == Z_OK && nUncompressed == nUncompressedBytes;
}
@@ -0,0 +1,112 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __STLHELPER__
#define __STLHELPER__
#include <vector>
#include <string>
typedef std::vector<std::string> tdEntryVec;
typedef std::pair<std::string, std::string> tdToken;
typedef std::vector<tdToken> tdTokenList;
typedef std::vector<uint8_t> tdDataVector;
//typedef std::vector<uint8_t> tdHash;
struct tdHash
{
uint8_t hash[16];
inline bool operator<(const tdHash& other) const { return memcmp(hash, other.hash, sizeof(hash)) < 0; }
inline bool operator>(const tdHash& other) const { return memcmp(hash, other.hash, sizeof(hash)) > 0; }
inline bool operator==(const tdHash& other) const { return memcmp(hash, other.hash, sizeof(hash)) == 0; }
inline uint8_t& operator[](size_t nIndex) { return hash[nIndex]; }
inline const uint8_t& operator[](size_t nIndex) const { return hash[nIndex]; }
};
class CSTLHelper
{
static tdHash Hash(const uint8_t* pData, const size_t Size);
public:
static void Tokenize(tdEntryVec& rRet, const std::string& Tokens, const std::string& Separator);
static tdToken SplitToken(const std::string& rToken, const std::string& rSeparator);
static void Splitizer(tdTokenList& rTokenList, const tdEntryVec& rFilter, const std::string& rSeparator);
static void Trim(std::string& rStr, const std::string& charsToTrim);
static void Remove(std::string& rTokenDst, const std::string& rTokenSrc, const char C);
static void Replace(std::vector<uint8_t>& rRet, const std::vector<uint8_t>& rTokenSrc, const std::string& rToReplace, const std::string& rReplacement);
static void Replace(std::string& rRet, const std::string& rSrc, const std::string& rToReplace, const std::string& rReplacement);
static bool ToFile(const std::string& rFileName, const std::vector<uint8_t>& rOut);
static bool FromFile(const std::string& rFileName, std::vector<uint8_t>& rIn);
static bool AppendToFile(const std::string& rFileName, const std::vector<uint8_t>& rOut);
static bool ToFileCompressed(const std::string& rFileName, const std::vector<uint8_t>& rOut);
static bool FromFileCompressed(const std::string& rFileName, std::vector<uint8_t>& rIn);
static bool Compress(const std::vector<uint8_t>& rIn, std::vector<uint8_t>& rOut);
static bool Uncompress(const std::vector<uint8_t>& rIn, std::vector<uint8_t>& rOut);
static void EndianSwizzleU64(uint64_t& S)
{
uint8_t* pT = reinterpret_cast<uint8_t*>(&S);
uint8_t T;
T = pT[0];
pT[0] = pT[7];
pT[7] = T;
T = pT[1];
pT[1] = pT[6];
pT[6] = T;
T = pT[2];
pT[2] = pT[5];
pT[5] = T;
T = pT[3];
pT[3] = pT[4];
pT[4] = T;
}
static void EndianSwizzleU32(uint32_t& S)
{
uint8_t* pT = reinterpret_cast<uint8_t*>(&S);
uint8_t T;
T = pT[0];
pT[0] = pT[3];
pT[3] = T;
T = pT[1];
pT[1] = pT[2];
pT[2] = T;
}
static void EndianSwizzleU16(uint16_t& S)
{
uint8_t* pT = reinterpret_cast<uint8_t*>(&S);
uint8_t T;
T = pT[0];
pT[0] = pT[1];
pT[1] = T;
}
static void Log(const std::string& rLog);
static tdHash Hash(const std::string& rStr) { return Hash(reinterpret_cast<const uint8_t*>(rStr.c_str()), rStr.size()); }
static tdHash Hash(const std::vector<uint8_t>& rData) { return Hash(&rData[0], rData.size()); }
static tdHash Hash(const std::vector<uint8_t>& rData, size_t Size) { return Hash(&rData[0], Size); }
static std::string Hash2String(const tdHash& rHash);
static tdHash String2Hash(const std::string& rStr);
};
#define CRYSIMPLE_LOG(X) CSTLHelper::Log(X)
#endif
@@ -0,0 +1,310 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySimpleCache.hpp"
#include "CrySimpleServer.hpp"
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <AzCore/IO/SystemFile.h>
#include <AzCore/std/time.h>
#include <AzCore/std/algorithm.h>
enum EFileEntryHeaderFlags
{
EFEHF_NONE = (0 << 0),
EFEHF_REFERENCE = (1 << 0),
};
#pragma pack(push, 1)
struct SFileEntryHeader
{
char signature[4]; // entry signature.
uint32_t dataSize; // Size of entry data.
uint32_t flags; // Flags
uint8_t hash[16]; // Hash code for the data.
};
#pragma pack(pop)
static const int MAX_DATA_SIZE = 1024 * 1024;
CCrySimpleCache& CCrySimpleCache::Instance()
{
static CCrySimpleCache g_Cache;
return g_Cache;
}
void CCrySimpleCache::Init()
{
CCrySimpleMutexAutoLock Lock(m_Mutex);
m_CachingEnabled = false;
m_Hit = 0;
m_Miss = 0;
}
std::string CCrySimpleCache::CreateFileName(const tdHash& rHash) const
{
std::string Name;
Name = CSTLHelper::Hash2String(rHash);
char Tmp[4] = "012";
Tmp[0] = Name.c_str()[0];
Tmp[1] = Name.c_str()[1];
Tmp[2] = Name.c_str()[2];
return SEnviropment::Instance().m_CachePath + Tmp + "/" + Name;
}
bool CCrySimpleCache::Find(const tdHash& rHash, tdDataVector& rData)
{
if (!m_CachingEnabled)
{
return false;
}
CCrySimpleMutexAutoLock Lock(m_Mutex);
tdEntries::iterator it = m_Entries.find(rHash);
if (it != m_Entries.end())
{
tdData::iterator dataIt = m_Data.find(it->second);
if (dataIt == m_Data.end())
{
m_Miss++;
return false;
}
m_Hit++;
rData = dataIt->second;
return true;
}
m_Miss++;
return false;
}
void CCrySimpleCache::Add(const tdHash& rHash, const tdDataVector& rData)
{
if (!m_CachingEnabled)
{
return;
}
if (rData.size() > 0)
{
SFileEntryHeader hdr;
memcpy(hdr.signature, "SHDR", 4);
hdr.dataSize = (uint32_t)rData.size();
hdr.flags = EFEHF_NONE;
memcpy(hdr.hash, &rHash, sizeof(hdr.hash));
const uint8_t* pData = &rData[0];
tdHash DataHash = CSTLHelper::Hash(rData);
{
CCrySimpleMutexAutoLock Lock(m_Mutex);
m_Entries[rHash] = DataHash;
if (m_Data.find(DataHash) == m_Data.end())
{
m_Data[DataHash] = rData;
}
else
{
hdr.flags |= EFEHF_REFERENCE;
hdr.dataSize = sizeof(tdHash);
pData = reinterpret_cast<const uint8_t*>(&DataHash);
}
}
tdDataVector buf;
buf.resize(sizeof(hdr) + hdr.dataSize);
memcpy(&buf[0], &hdr, sizeof(hdr));
memcpy(&buf[sizeof(hdr)], pData, hdr.dataSize);
tdDataVector* pPendingCacheEntry = new tdDataVector(buf);
{
CCrySimpleMutexAutoLock LockFile(m_FileMutex);
m_PendingCacheEntries.push_back(pPendingCacheEntry);
if (m_PendingCacheEntries.size() > 10000)
{
printf("Warning: Too many pending entries not saved to disk!!!");
}
}
}
}
//////////////////////////////////////////////////////////////////////////
bool CCrySimpleCache::LoadCacheFile(const std::string& filename)
{
AZ::u64 startTimeInMillis = AZStd::GetTimeUTCMilliSecond();
printf("Loading shader cache from %s\n", filename.c_str());
tdDataVector rData;
tdHash hash;
bool bLoadedOK = true;
uint32_t Loaded = 0;
uint32_t num = 0;
uint64_t nFilePos = 0;
uint64_t nFilePos2 = 0;
//////////////////////////////////////////////////////////////////////////
AZ::IO::SystemFile cacheFile;
const bool wasSuccessful = cacheFile.Open(filename.c_str(), AZ::IO::SystemFile::SF_OPEN_READ_ONLY);
if (!wasSuccessful)
{
return false;
}
AZ::IO::SystemFile::SizeType fileSize = cacheFile.Length();
uint64_t SizeAdded = 0;
uint64_t SizeAddedCount = 0;
uint64_t SizeSaved = 0;
uint64_t SizeSavedCount = 0;
while (nFilePos < fileSize)
{
SFileEntryHeader hdr;
AZ::IO::SystemFile::SizeType bytesReadIn = cacheFile.Read(sizeof(hdr), &hdr);
if (bytesReadIn != sizeof(hdr))
{
break;
}
if (memcmp(hdr.signature, "SHDR", 4) != 0)
{
// Bad Entry!
bLoadedOK = false;
printf("\nSkipping Invalid cache entry %d\n at file position: %llu because signature is bad", num, nFilePos);
break;
}
if (hdr.dataSize > MAX_DATA_SIZE || hdr.dataSize == 0)
{
// Too big entry, probably invalid.
bLoadedOK = false;
printf("\nSkipping Invalid cache entry %d\n at file position: %llu because data size is too big", num, nFilePos);
break;
}
rData.resize(hdr.dataSize);
bytesReadIn = cacheFile.Read(hdr.dataSize, rData.data());
if (bytesReadIn != hdr.dataSize)
{
break;
}
memcpy(&hash, hdr.hash, sizeof(hdr.hash));
if (hdr.flags & EFEHF_REFERENCE)
{
if (hdr.dataSize != sizeof(tdHash))
{
// Too big entry, probably invalid.
bLoadedOK = false;
printf("\nSkipping Invalid cache entry %d\n at file position: %llu, was flagged as cache reference but size was %d", num, nFilePos, hdr.dataSize);
break;
}
bool bSkip = false;
tdHash DataHash = *reinterpret_cast<tdHash*>(&rData[0]);
tdData::iterator it = m_Data.find(DataHash);
if (it == m_Data.end())
{
// Too big entry, probably invalid.
bSkip = true; // don't abort reading whole file just yet - skip only this entry
printf("\nSkipping Invalid cache entry %d\n at file position: %llu, data-hash references to not existing data ", num, nFilePos);
}
if (!bSkip)
{
m_Entries[hash] = DataHash;
SizeSaved += it->second.size();
SizeSavedCount++;
}
}
else
{
tdHash DataHash = CSTLHelper::Hash(rData);
m_Entries[hash] = DataHash;
if (m_Data.find(DataHash) == m_Data.end())
{
SizeAdded += rData.size();
m_Data[DataHash] = rData;
SizeAddedCount++;
}
else
{
SizeSaved += rData.size();
SizeSavedCount++;
}
}
if (num % 1000 == 0)
{
AZ::u64 endTimeInMillis = AZStd::GetTimeUTCMilliSecond();
Loaded = static_cast<uint32_t>(nFilePos * 100 / fileSize);
printf("\rLoad:%3u%% %6uk t=%llus Compress: (Count)%llu%% %lluk:%lluk (MB)%llu%% %lluMB:%lluMB", Loaded, num / 1000u, (endTimeInMillis - startTimeInMillis),
SizeAddedCount / AZStd::GetMax((SizeAddedCount + SizeSavedCount) / 100ull, 1ull),
SizeAddedCount / 1000, SizeSavedCount / 1000,
SizeAdded / AZStd::GetMax((SizeAdded + SizeSaved) / 100ull, 1ull),
SizeAdded / (MAX_DATA_SIZE), SizeSaved / (MAX_DATA_SIZE));
}
num++;
nFilePos += hdr.dataSize + sizeof(SFileEntryHeader);
}
printf("\n%d shaders loaded from cache\n", num);
return bLoadedOK;
}
void CCrySimpleCache::Finalize()
{
m_CachingEnabled = true;
printf("\n caching enabled\n");
}
//////////////////////////////////////////////////////////////////////////
void CCrySimpleCache::ThreadFunc_SavePendingCacheEntries()
{
// Check pending entries and save them to disk.
bool bListEmpty = false;
do
{
tdDataVector* pPendingCacheEntry = 0;
{
CCrySimpleMutexAutoLock LockFile(m_FileMutex);
if (!m_PendingCacheEntries.empty())
{
pPendingCacheEntry = m_PendingCacheEntries.front();
m_PendingCacheEntries.pop_front();
}
//CSTLHelper::AppendToFile( SEnviropment::Instance().m_CachePath+"Cache.dat",buf );
bListEmpty = m_PendingCacheEntries.empty();
}
if (pPendingCacheEntry)
{
CSTLHelper::AppendToFile(SEnviropment::Instance().m_CachePath + "Cache.dat", *pPendingCacheEntry);
delete pPendingCacheEntry;
}
} while (!bListEmpty);
}
@@ -0,0 +1,68 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLECACHE__
#define __CRYSIMPLECACHE__
#include "CrySimpleMutex.hpp"
#include <Core/STLHelper.hpp>
#include <map>
#include <vector>
#include <list>
/*class CCrySimpleCacheEntry
{
tdCache
public:
protected:
private:
};*/
typedef std::map<tdHash, tdHash> tdEntries;
typedef std::map<tdHash, tdDataVector> tdData;
class CCrySimpleCache
{
volatile bool m_CachingEnabled;
int m_Hit;
int m_Miss;
tdEntries m_Entries;
tdData m_Data;
CCrySimpleMutex m_Mutex;
CCrySimpleMutex m_FileMutex;
std::list<tdDataVector*> m_PendingCacheEntries;
std::string CreateFileName(const tdHash& rHash) const;
public:
void Init();
bool Find(const tdHash& rHash, tdDataVector& rData);
void Add(const tdHash& rHash, const tdDataVector& rData);
bool LoadCacheFile(const std::string& filename);
void Finalize();
void ThreadFunc_SavePendingCacheEntries();
static CCrySimpleCache& Instance();
std::list<tdDataVector*>& PendingCacheEntries(){return m_PendingCacheEntries; }
int Hit() const{return m_Hit; }
int Miss() const{return m_Miss; }
int EntryCount() const{return static_cast<int>(m_Entries.size()); }
};
#endif
@@ -0,0 +1,274 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySimpleErrorLog.hpp"
#include "CrySimpleServer.hpp"
#include "CrySimpleJob.hpp"
#include <Core/Common.h>
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <Core/Mailer.h>
#include <tinyxml/tinyxml.h>
#include <AzCore/std/time.h>
#include <AzCore/IO/SystemFile.h>
#include <string>
#include <map>
#include <algorithm>
#ifdef _MSC_VER
#include <process.h>
#include <direct.h>
#endif
#ifdef UNIX
#include <pthread.h>
#endif
static unsigned int volatile g_bSendingMail = false;
static unsigned int volatile g_nMailNum = 0;
CCrySimpleErrorLog& CCrySimpleErrorLog::Instance()
{
static CCrySimpleErrorLog g_Cache;
return g_Cache;
}
CCrySimpleErrorLog::CCrySimpleErrorLog()
{
m_lastErrorTime = 0;
}
void CCrySimpleErrorLog::Init()
{
}
bool CCrySimpleErrorLog::Add(ICryError* err)
{
CCrySimpleMutexAutoLock Lock(m_LogMutex);
if (m_Log.size() > 150)
{
// too many, just throw this error away
return false; // no ownership of this error
}
m_Log.push_back(err);
m_lastErrorTime = AZStd::GetTimeUTCMilliSecond();
return true; // take ownership of this error
}
inline bool CmpError(ICryError* a, ICryError* b)
{
return a->Compare(b);
}
void CCrySimpleErrorLog::SendMail()
{
CSMTPMailer::tstrcol Rcpt;
std::string mailBody;
tdEntryVec RcptVec;
CSTLHelper::Tokenize(RcptVec, SEnviropment::Instance().m_FailEMail, ";");
for (size_t i = 0; i < RcptVec.size(); i++)
{
Rcpt.insert(RcptVec[i]);
}
tdErrorList tempLog;
{
CCrySimpleMutexAutoLock Lock(m_LogMutex);
m_Log.swap(tempLog);
}
#if defined(_MSC_VER)
{
char compName[256];
DWORD size = ARRAYSIZE(compName);
typedef BOOL (WINAPI * FP_GetComputerNameExA)(COMPUTER_NAME_FORMAT, LPSTR, LPDWORD);
FP_GetComputerNameExA pGetComputerNameExA = (FP_GetComputerNameExA) GetProcAddress(LoadLibrary("kernel32.dll"), "GetComputerNameExA");
if (pGetComputerNameExA)
{
pGetComputerNameExA(ComputerNamePhysicalDnsFullyQualified, compName, &size);
}
else
{
GetComputerName(compName, &size);
}
mailBody += std::string("Report sent from ") + compName + "...\n\n";
}
#endif
{
bool dedupe = SEnviropment::Instance().m_DedupeErrors;
std::vector<ICryError*> errors;
if (!dedupe)
{
for (tdErrorList::const_iterator it = tempLog.begin(); it != tempLog.end(); ++it)
{
errors.push_back(*it);
}
}
else
{
std::map<tdHash, ICryError*> uniqErrors;
for (tdErrorList::const_iterator it = tempLog.begin(); it != tempLog.end(); ++it)
{
ICryError* err = *it;
tdHash hash = err->Hash();
std::map<tdHash, ICryError*>::iterator uniq = uniqErrors.find(hash);
if (uniq != uniqErrors.end())
{
uniq->second->AddDuplicate(err);
delete err;
}
else
{
uniqErrors[hash] = err;
}
}
for (std::map<tdHash, ICryError*>::iterator it = uniqErrors.begin(); it != uniqErrors.end(); ++it)
{
errors.push_back(it->second);
}
}
std::string body = mailBody;
CSMTPMailer::tstrcol cc;
CSMTPMailer::tattachlist Attachment;
std::sort(errors.begin(), errors.end(), CmpError);
int a = 0;
for (uint32_t i = 0; i < errors.size(); i++)
{
ICryError* err = errors[i];
err->SetUniqueID(a + 1);
// doesn't have to be related to any job/error,
// we just use it to differentiate "1-IlluminationPS.txt" from "1-IlluminationPS.txt"
long req = CCrySimpleJob::GlobalRequestNumber();
if (err->HasFile())
{
char Filename[1024];
azsprintf(Filename, "%d-req%ld-%s", a + 1, req, err->GetFilename().c_str());
char DispFilename[1024];
azsprintf(DispFilename, "%d-%s", a + 1, err->GetFilename().c_str());
std::string sErrorFile = SEnviropment::Instance().m_ErrorPath + Filename;
std::vector<uint8_t> bytes;
std::string text = err->GetFileContents();
bytes.resize(text.size() + 1);
std::copy(text.begin(), text.end(), bytes.begin());
while (bytes.size() && bytes[bytes.size() - 1] == 0)
{
bytes.pop_back();
}
CrySimple_SECURE_START
CSTLHelper::ToFile(sErrorFile, bytes);
Attachment.push_back(CSMTPMailer::tattachment(DispFilename, sErrorFile));
CrySimple_SECURE_END
}
body += std::string("=============================================================\n");
body += err->GetErrorDetails(ICryError::OUTPUT_EMAIL) + "\n";
err->AddCCs(cc);
a++;
if (i == errors.size() - 1 || !err->CanMerge(errors[i + 1]))
{
CSMTPMailer::tstrcol bcc;
CSMTPMailer mail("", "", SEnviropment::Instance().m_MailServer);
bool res = mail.Send(SEnviropment::Instance().m_FailEMail, Rcpt, cc, bcc, err->GetErrorName(), body, Attachment);
a = 0;
body = mailBody;
cc.clear();
for (CSMTPMailer::tattachlist::iterator attach = Attachment.begin(); attach != Attachment.end(); ++attach)
{
AZ::IO::SystemFile::Delete(attach->second.c_str());
}
Attachment.clear();
}
delete err;
}
}
g_bSendingMail = false;
}
//////////////////////////////////////////////////////////////////////////
void CCrySimpleErrorLog::Tick()
{
if (SEnviropment::Instance().m_MailInterval == 0)
{
return;
}
AZ::u64 lastError = 0;
bool forceFlush = false;
{
CCrySimpleMutexAutoLock Lock(m_LogMutex);
if (m_Log.size() == 0)
{
return;
}
// log has gotten pretty big, force a flush to avoid losing any errors
if (m_Log.size() > 100)
{
forceFlush = true;
}
lastError = m_lastErrorTime;
}
AZ::u64 t = AZStd::GetTimeUTCMilliSecond();
if (forceFlush || t < lastError || (t - lastError) > SEnviropment::Instance().m_MailInterval * 1000)
{
if (!g_bSendingMail)
{
g_bSendingMail = true;
g_nMailNum++;
logmessage("Sending Errors Mail %d\n", g_nMailNum);
CCrySimpleErrorLog::Instance().SendMail();
}
}
}
@@ -0,0 +1,42 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLEERRORLOG__
#define __CRYSIMPLEERRORLOG__
#include "CrySimpleMutex.hpp"
#include <vector>
#include <list>
class ICryError;
typedef std::list<ICryError*> tdErrorList;
class CCrySimpleErrorLog
{
CCrySimpleMutex m_LogMutex; // protects both below variables
tdErrorList m_Log; // error log
AZ::u64 m_lastErrorTime; // last time an error came in (we mail out a little after we've stopped receiving errors)
void Init();
void SendMail();
CCrySimpleErrorLog();
public:
bool Add(ICryError* err);
void Tick();
static CCrySimpleErrorLog& Instance();
};
#endif
@@ -0,0 +1,33 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLEFILEGUARD__
#define __CRYSIMPLEFILEGUARD__
#include <Core/Common.h>
#include <string>
class CCrySimpleFileGuard
{
std::string m_FileName;
public:
CCrySimpleFileGuard(const std::string& rFileName)
: m_FileName(rFileName)
{
}
~CCrySimpleFileGuard()
{
remove(m_FileName.c_str());
}
};
#endif
@@ -0,0 +1,228 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySimpleHTTP.hpp"
#include "CrySimpleSock.hpp"
#include "CrySimpleJobCompile.hpp"
#include "CrySimpleServer.hpp"
#include "CrySimpleCache.hpp"
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <tinyxml/tinyxml.h>
#include <AzCore/Jobs/Job.h>
#include <AzCore/std/algorithm.h>
#include <AzCore/std/string/string.h>
#include <assert.h>
#include <memory>
//////////////////////////////////////////////////////////////////////////
class CHTTPRequest
{
CCrySimpleSock* m_pSock;
public:
CHTTPRequest(CCrySimpleSock* pSock)
: m_pSock(pSock){}
~CHTTPRequest(){delete m_pSock; }
CCrySimpleSock* Socket(){return m_pSock; }
};
#define HTML_HEADER "HTTP/1.1 200 OK\n\
Server: Shader compile server %s\n\
Content-Length: %zu\n\
Content-Language: de (nach RFC 3282 sowie RFC 1766)\n\
Content-Type: text/html\n\
Connection: close\n\
\n\
<html><title>shader compile server %s</title><body>"
#define TABLE_START "<TABLE BORDER=0 CELLSPACING=0 CELLPADDING=2 WIDTH=640>\n\
<TR bgcolor=lightgrey><TH align=left>Description</TH><TH WIDTH=5></TH><TH>Value</TH><TH>Max</TH>\n\
<TH WIDTH=10>&nbsp;</TH><TH align=center>%</TH></TR>\n"
#define TABLE_INFO "<TR><TD>%s</TD><TD>&nbsp;</TD><TD align=left>%s</TD><TD align=center></TD><TD>&nbsp;</TD><TD valign=middle>\n\
</TD></TR>\n\
</TD></TR>\n"
#define TABLE_BAR "<TR><TD>%s</TD><TD>&nbsp;</TD><TD align=center>%d</TD><TD align=center>%d</TD><TD>&nbsp;</TD><TD valign=middle>\n\
<TABLE><TR><TD bgcolor=darkred style=\"width: %d;\" ></TD>\n\
<TD><FONT SIZE=1>%d%%</FONT></TD></TR>\n\
</TABLE></TD></TR>\n\
</TD></TR>\n"
#define TABLE_END "</TABLE>"
std::string CreateBar(const std::string& rName, int Value, int Max, int Percentage)
{
AZStd::string formattedString = AZStd::string::format(TABLE_BAR, rName.c_str(), Value, Max, Percentage, Percentage);
return formattedString.c_str();
}
std::string CreateInfoText(const std::string& rName, const std::string& rValue)
{
AZStd::string formattedString = AZStd::string::format(TABLE_INFO, rName.c_str(), rValue.c_str());
return formattedString.c_str();
}
std::string CreateInfoText(const std::string& rName, int Value)
{
char Text[64];
azsprintf(Text, "%d", Value);
return CreateInfoText(rName, Text);
}
class HttpProcessRequestJob
: public AZ::Job
{
public:
HttpProcessRequestJob(CHTTPRequest* request)
: AZ::Job(true, nullptr)
, m_request(request) { }
protected:
void Process() override
{
#if defined(AZ_PLATFORM_WINDOWS)
FILETIME IdleTime0, IdleTime1;
FILETIME KernelTime0, KernelTime1;
FILETIME UserTime0, UserTime1;
int Ret0 = GetSystemTimes(&IdleTime0, &KernelTime0, &UserTime0);
Sleep(100);
int Ret1 = GetSystemTimes(&IdleTime1, &KernelTime1, &UserTime1);
const int Idle = IdleTime1.dwLowDateTime - IdleTime0.dwLowDateTime;
const int Kernel = KernelTime1.dwLowDateTime - KernelTime0.dwLowDateTime;
const int User = UserTime1.dwLowDateTime - UserTime0.dwLowDateTime;
//const int Idle = IdleTime1.dwHighDateTime-IdleTime0.dwHighDateTime;
//const int Kernel = KernelTime1.dwHighDateTime-KernelTime0.dwHighDateTime;
//const int User = UserTime1.dwHighDateTime-UserTime0.dwHighDateTime;
const int Total = Kernel + User;
#else
int Ret0 = 0;
int Ret1 = 0;
int Total = 0;
int Idle = 0;
#endif
std::string Ret = TABLE_START;
Ret += CreateInfoText("<b>Load</b>:", "");
if (Ret0 && Ret1 && Total)
{
Ret += CreateBar("CPU-Usage", Total - Idle, Total, 100 - Idle * 100 / Total);
}
Ret += CreateBar("CompileTasks", CCrySimpleJobCompile::GlobalCompileTasks(),
CCrySimpleJobCompile::GlobalCompileTasksMax(),
CCrySimpleJobCompile::GlobalCompileTasksMax() ?
CCrySimpleJobCompile::GlobalCompileTasks() * 100 /
CCrySimpleJobCompile::GlobalCompileTasksMax() : 0);
Ret += CreateInfoText("<b>Setup</b>:", "");
Ret += CreateInfoText("Root", SEnviropment::Instance().m_Root);
Ret += CreateInfoText("CompilerPath", SEnviropment::Instance().m_CompilerPath);
Ret += CreateInfoText("CachePath", SEnviropment::Instance().m_CachePath);
Ret += CreateInfoText("TempPath", SEnviropment::Instance().m_TempPath);
Ret += CreateInfoText("ErrorPath", SEnviropment::Instance().m_ErrorPath);
Ret += CreateInfoText("ShaderPath", SEnviropment::Instance().m_ShaderPath);
Ret += CreateInfoText("FailEMail", SEnviropment::Instance().m_FailEMail);
Ret += CreateInfoText("MailServer", SEnviropment::Instance().m_MailServer);
Ret += CreateInfoText("port", SEnviropment::Instance().m_port);
Ret += CreateInfoText("MailInterval", SEnviropment::Instance().m_MailInterval);
Ret += CreateInfoText("Caching", SEnviropment::Instance().m_Caching ? "Enabled" : "Disabled");
Ret += CreateInfoText("FallbackServer", SEnviropment::Instance().m_FallbackServer == "" ? "None" : SEnviropment::Instance().m_FallbackServer);
Ret += CreateInfoText("FallbackTreshold", static_cast<int>(SEnviropment::Instance().m_FallbackTreshold));
Ret += CreateInfoText("DumpShaders", static_cast<int>(SEnviropment::Instance().m_DumpShaders));
Ret += CreateInfoText("<b>Cache</b>:", "");
Ret += CreateInfoText("Entries", CCrySimpleCache::Instance().EntryCount());
Ret += CreateBar("Hits", CCrySimpleCache::Instance().Hit(),
CCrySimpleCache::Instance().Hit() + CCrySimpleCache::Instance().Miss(),
CCrySimpleCache::Instance().Hit() * 100 / AZStd::GetMax(1, (CCrySimpleCache::Instance().Hit() + CCrySimpleCache::Instance().Miss())));
Ret += CreateInfoText("Pending Entries", static_cast<int>(CCrySimpleCache::Instance().PendingCacheEntries().size()));
Ret += TABLE_END;
Ret += "</Body></hmtl>";
char Text[sizeof(HTML_HEADER) + 1024];
azsprintf(Text, HTML_HEADER, __DATE__, Ret.size(), __DATE__);
Ret = std::string(Text) + Ret;
m_request->Socket()->Send(Ret);
}
private:
std::unique_ptr<CHTTPRequest> m_request;
};
class HttpServerJob
: public AZ::Job
{
public:
HttpServerJob(CCrySimpleHTTP* simpleHttp)
: AZ::Job(true, nullptr)
, m_simpleHttp(simpleHttp) { }
protected:
void Process()
{
m_simpleHttp->Run();
}
private:
CCrySimpleHTTP* m_simpleHttp;
};
//////////////////////////////////////////////////////////////////////////
CCrySimpleHTTP::CCrySimpleHTTP()
: m_pServerSocket(0)
{
CrySimple_SECURE_START
Init();
CrySimple_SECURE_END
}
void CCrySimpleHTTP::Init()
{
m_pServerSocket = new CCrySimpleSock(61480, SEnviropment::Instance().m_WhitelistAddresses); //http
m_pServerSocket->Listen();
HttpServerJob* serverJob = new HttpServerJob(this);
serverJob->Start();
}
void CCrySimpleHTTP::Run()
{
while (1)
{
// New client message, receive new client socket connection.
CCrySimpleSock* newClientSocket = m_pServerSocket->Accept();
if(!newClientSocket)
{
continue;
}
// HTTP Request Data for new job
CHTTPRequest* pData = new CHTTPRequest(newClientSocket);
HttpProcessRequestJob* requestJob = new HttpProcessRequestJob(pData);
requestJob->Start();
}
}
@@ -0,0 +1,36 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CrySimpleHTTP__
#define __CrySimpleHTTP__
#include <AzCore/std/parallel/atomic.h>
#include <Core/Common.h>
#include <string>
extern bool g_Success;
class CCrySimpleSock;
class CCrySimpleHTTP
{
static AZStd::atomic_long ms_ExceptionCount;
CCrySimpleSock* m_pServerSocket;
void Init();
public:
CCrySimpleHTTP();
void Run();
};
#endif
@@ -0,0 +1,209 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySimpleJob.hpp"
#include "CrySimpleFileGuard.hpp"
#include "CrySimpleServer.hpp"
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <Core/Common.h>
#include <Core/WindowsAPIImplementation.h>
#include <tinyxml/tinyxml.h>
#include <thread>
#include <sstream>
#include <fstream>
AZStd::atomic_long CCrySimpleJob::m_GlobalRequestNumber = {0};
CCrySimpleJob::CCrySimpleJob(uint32_t requestIP)
: m_State(ECSJS_NONE)
, m_RequestIP(requestIP)
{
++m_GlobalRequestNumber;
}
CCrySimpleJob::~CCrySimpleJob()
{
}
bool CCrySimpleJob::ExecuteCommand(const std::string& rCmd, std::string& outError)
{
const bool showStdOuput = false; // For Debug: Set to true if you want the compiler's standard ouput printed out as well.
const bool showStdErrorOuput = SEnviropment::Instance().m_PrintWarnings;
#ifdef _MSC_VER
bool Ret = false;
DWORD ExitCode = 0;
STARTUPINFO StartupInfo;
PROCESS_INFORMATION ProcessInfo;
memset(&StartupInfo, 0, sizeof(StartupInfo));
memset(&ProcessInfo, 0, sizeof(ProcessInfo));
StartupInfo.cb = sizeof(StartupInfo);
std::string Path = "";
std::string::size_type Pt = rCmd.find_first_of(' ');
if (Pt != std::string::npos)
{
std::string First = std::string(rCmd.c_str(), Pt);
std::string::size_type Pt2 = First.find_last_of('/');
if (Pt2 != std::string::npos)
{
Path = std::string(First.c_str(), Pt2);
}
else
{
Pt = std::string::npos;
}
}
HANDLE hReadErr, hWriteErr;
{
CreatePipe(&hReadErr, &hWriteErr, NULL, 0);
SetHandleInformation(hWriteErr, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT);
StartupInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
StartupInfo.hStdOutput = (showStdOuput) ? GetStdHandle(STD_OUTPUT_HANDLE) : NULL;
StartupInfo.hStdError = hWriteErr;
StartupInfo.dwFlags |= STARTF_USESTDHANDLES;
BOOL processCreated = CreateProcess(NULL, (char*)rCmd.c_str(), 0, 0, TRUE, CREATE_DEFAULT_ERROR_MODE, 0, Pt != std::string::npos ? Path.c_str() : 0, &StartupInfo, &ProcessInfo) != false;
if (!processCreated)
{
outError = "Couldn't create process - missing compiler for cmd?: '" + rCmd + "'";
}
else
{
std::string error;
DWORD waitResult = 0;
HANDLE waitHandles[] = { ProcessInfo.hProcess, hReadErr };
while (true)
{
//waitResult = WaitForMultipleObjects(sizeof(waitHandles) / sizeof(waitHandles[0]), waitHandles, FALSE, 1000 );
waitResult = WaitForSingleObject(ProcessInfo.hProcess, 1000);
if (waitResult == WAIT_FAILED)
{
break;
}
DWORD bytesRead, bytesAvailable;
while (PeekNamedPipe(hReadErr, NULL, 0, NULL, &bytesAvailable, NULL) && bytesAvailable)
{
char buff[4096];
ReadFile(hReadErr, buff, sizeof(buff) - 1, &bytesRead, 0);
buff[bytesRead] = '\0';
error += buff;
}
CSTLHelper::Trim(error," \t\r\n");
//if (waitResult == WAIT_OBJECT_0 || waitResult == WAIT_TIMEOUT)
//break;
if (waitResult == WAIT_OBJECT_0)
{
break;
}
}
//if (waitResult != WAIT_TIMEOUT)
{
GetExitCodeProcess(ProcessInfo.hProcess, &ExitCode);
if (ExitCode)
{
Ret = false;
outError = error;
}
else
{
if (showStdErrorOuput && !error.empty())
{
AZ_Printf(0, "\n%s\n", error.c_str());
}
Ret = true;
}
}
/*
else
{
Ret = false;
outError = std::string("Timed out executing compiler: ") + rCmd;
TerminateProcess(ProcessInfo.hProcess, 1);
}
*/
CloseHandle(ProcessInfo.hProcess);
CloseHandle(ProcessInfo.hThread);
}
CloseHandle(hReadErr);
if (hWriteErr)
{
CloseHandle(hWriteErr);
}
}
return Ret;
#endif
#if defined(AZ_PLATFORM_LINUX) || defined(AZ_PLATFORM_MAC)
std::thread::id threadId = std::this_thread::get_id();
std::stringstream threadIdStream;
threadIdStream << threadId;
// Multiple threads could execute a command, therefore the temporary file has to be unique per thread.
std::string stdErrorTempFilename = SEnviropment::Instance().m_TempPath + "stderr_" + threadIdStream.str() + ".log";
CCrySimpleFileGuard FGTmpOutput(stdErrorTempFilename); // Delete file at the end of this function
std::string systemCmd = rCmd;
if(!showStdOuput)
{
// Standard output redirected to null to disable it
systemCmd += " > /dev/null";
}
// Standard error ouput redirected to the temporary file
systemCmd += " 2> \"" + stdErrorTempFilename + "\"";
int ret = system(systemCmd.c_str());
// Obtain standard error output
std::ifstream fileStream(stdErrorTempFilename.c_str());
std::stringstream stdErrorStream;
stdErrorStream << fileStream.rdbuf();
std::string stdErrorString = stdErrorStream.str();
CSTLHelper::Trim(stdErrorString," \t\r\n");
if (ret != 0)
{
outError = stdErrorString;
return false;
}
else
{
if (showStdErrorOuput && !stdErrorString.empty())
{
AZ_Printf(0, "\n%s\n", stdErrorString.c_str());
}
return true;
}
#endif
}
@@ -0,0 +1,74 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLEJOB__
#define __CRYSIMPLEJOB__
#include <Core/Common.h>
#include <string>
#include <vector>
#include <string>
#include <AzCore/std/parallel/atomic.h>
class TiXmlElement;
enum ECrySimpleJobState
{
ECSJS_NONE,
ECSJS_DONE = 1, //this is checked on client side, don't change!
ECSJS_JOBNOTFOUND,
ECSJS_CACHEHIT,
ECSJS_ERROR,
ECSJS_ERROR_COMPILE = 5, //this is checked on client side, don't change!
ECSJS_ERROR_COMPRESS,
ECSJS_ERROR_FILEIO,
ECSJS_ERROR_INVALID_PROFILE,
ECSJS_ERROR_INVALID_PROJECT,
ECSJS_ERROR_INVALID_PLATFORM,
ECSJS_ERROR_INVALID_PROGRAM,
ECSJS_ERROR_INVALID_ENTRY,
ECSJS_ERROR_INVALID_COMPILEFLAGS,
ECSJS_ERROR_INVALID_COMPILER,
ECSJS_ERROR_INVALID_LANGUAGE,
ECSJS_ERROR_INVALID_SHADERREQUESTLINE,
ECSJS_ERROR_INVALID_SHADERLIST,
};
class CCrySimpleJob
{
ECrySimpleJobState m_State;
uint32_t m_RequestIP;
static AZStd::atomic_long m_GlobalRequestNumber;
protected:
virtual bool ExecuteCommand(const std::string& rCmd, std::string& outError);
public:
CCrySimpleJob(uint32_t requestIP);
virtual ~CCrySimpleJob();
virtual bool Execute(const TiXmlElement* pElement) = 0;
void State(ECrySimpleJobState S)
{
if (m_State < ECSJS_ERROR || S >= ECSJS_ERROR)
{
m_State = S;
}
}
ECrySimpleJobState State() const { return m_State; }
const uint32_t& RequestIP() const { return m_RequestIP; }
static long GlobalRequestNumber() { return m_GlobalRequestNumber; }
};
#endif
@@ -0,0 +1,36 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySimpleJobCache.hpp"
#include "CrySimpleCache.hpp"
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <Core/Common.h>
#include <tinyxml/tinyxml.h>
CCrySimpleJobCache::CCrySimpleJobCache(uint32_t requestIP)
: CCrySimpleJob(requestIP)
{
}
void CCrySimpleJobCache::CheckHashID(std::vector<uint8_t>& rVec, size_t Size)
{
m_HashID = CSTLHelper::Hash(rVec, Size);
if (CCrySimpleCache::Instance().Find(m_HashID, rVec))
{
State(ECSJS_CACHEHIT);
logmessage("\r"); // Just update cache hit number
}
}
@@ -0,0 +1,35 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLEJOBCACHE__
#define __CRYSIMPLEJOBCACHE__
#include "CrySimpleJob.hpp"
#include <Core/STLHelper.hpp>
class CCrySimpleJobCache
: public CCrySimpleJob
{
tdHash m_HashID;
protected:
void CheckHashID(std::vector<uint8_t>& rVec, size_t Size);
public:
CCrySimpleJobCache(uint32_t requestIP);
tdHash HashID() const{return m_HashID; }
};
#endif
@@ -0,0 +1,976 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySimpleSock.hpp"
#include "CrySimpleJobCompile.hpp"
#include "CrySimpleFileGuard.hpp"
#include "CrySimpleServer.hpp"
#include "CrySimpleCache.hpp"
#include "ShaderList.hpp"
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <tinyxml/tinyxml.h>
#include <Core/StdTypes.hpp>
#include <Core/WindowsAPIImplementation.h>
#include <AzCore/IO/SystemFile.h>
#include <AzCore/std/algorithm.h>
#include <AzCore/std/sort.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/containers/unordered_map.h>
#include <zlib.h>
#include <iostream>
#include <fstream>
#include <sstream>
#if defined(AZ_TOOLS_EXPAND_FOR_RESTRICTED_PLATFORMS)
#undef AZ_RESTRICTED_SECTION
#define CRYSIMPLEJOBCOMPILE_CPP_SECTION_1 1
#endif
#define MAX_COMPILER_WAIT_TIME (60 * 1000)
AZStd::atomic_long CCrySimpleJobCompile::m_GlobalCompileTasks = {0};
AZStd::atomic_long CCrySimpleJobCompile::m_GlobalCompileTasksMax = {0};
volatile int32_t CCrySimpleJobCompile::m_RemoteServerID = 0;
volatile int64_t CCrySimpleJobCompile::m_GlobalCompileTime = 0;
struct STimer
{
int64_t m_freq;
STimer()
{
QueryPerformanceFrequency((LARGE_INTEGER*)&m_freq);
}
int64_t GetTime() const
{
int64_t t;
QueryPerformanceCounter((LARGE_INTEGER*)&t);
return t;
}
double TimeToSeconds(int64_t t)
{
return ((double)t) / m_freq;
}
};
STimer g_Timer;
// This function validates executables up to version 21
// because it's received within the compilation flags.
bool ValidateExecutableStringLegacy(const AZStd::string& executableString)
{
AZStd::string::size_type endOfCommand = executableString.find(" ");
// Game always sends some type of options after the command. If we don't
// have a space then that implies that there are no options. Reject the
// command as someone being malicious
if (endOfCommand == AZStd::string::npos)
{
return false;
}
AZStd::string commandString = executableString.substr(0, endOfCommand);
// The game never sends a parent directory in the compiler flags so lets
// reject any commands that have .. in it
if (commandString.find("..") != AZStd::string::npos)
{
return false;
}
// Though the code later down would fail gracefully reject any absolute paths here
if (commandString.find("\\\\") != AZStd::string::npos ||
commandString.find(":") != AZStd::string::npos)
{
return false;
}
// Only allow a subset of executables to be accepted...
if (commandString.find("fxc.exe") == AZStd::string::npos &&
commandString.find("FXC.exe") == AZStd::string::npos &&
commandString.find("HLSLcc.exe") == AZStd::string::npos &&
commandString.find("HLSLcc_dedicated.exe") == AZStd::string::npos &&
commandString.find("DXProvoShaderCompiler.exe") == AZStd::string::npos &&
commandString.find("dxcGL") == AZStd::string::npos &&
commandString.find("dxcMetal") == AZStd::string::npos)
{
return false;
}
return true;
}
CCrySimpleJobCompile::CCrySimpleJobCompile(uint32_t requestIP, EProtocolVersion Version, std::vector<uint8_t>* pRVec)
: CCrySimpleJobCache(requestIP)
, m_Version(Version)
, m_pRVec(pRVec)
{
++m_GlobalCompileTasks;
if (m_GlobalCompileTasksMax < m_GlobalCompileTasks)
{
//Need this cast as the copy assignment operator is implicitly deleted
m_GlobalCompileTasksMax = static_cast<long>(m_GlobalCompileTasks);
}
}
CCrySimpleJobCompile::~CCrySimpleJobCompile()
{
--m_GlobalCompileTasks;
}
bool CCrySimpleJobCompile::Execute(const TiXmlElement* pElement)
{
std::vector<uint8_t>& rVec = *m_pRVec;
size_t Size = SizeOf(rVec);
CheckHashID(rVec, Size);
if (State() == ECSJS_CACHEHIT)
{
State(ECSJS_DONE);
return true;
}
if (!SEnviropment::Instance().m_FallbackServer.empty() && m_GlobalCompileTasks > SEnviropment::Instance().m_FallbackTreshold)
{
tdEntryVec ServerVec;
CSTLHelper::Tokenize(ServerVec, SEnviropment::Instance().m_FallbackServer, ";");
uint32_t Idx = m_RemoteServerID++;
uint32_t Count = (uint32_t)ServerVec.size();
std::string Server = ServerVec[Idx % Count];
printf(" Remote Compile on %s ...\n", Server.c_str());
CCrySimpleSock Sock(Server, SEnviropment::Instance().m_port);
if (Sock.Valid())
{
Sock.Forward(rVec);
std::vector<uint8_t> Tmp;
if (Sock.Backward(Tmp))
{
rVec = Tmp;
if (Tmp.size() <= 4 || (m_Version == EPV_V002 && Tmp[4] != ECSJS_DONE))
{
State(ECSJS_ERROR_COMPILE);
CrySimple_ERROR("failed to compile request");
return false;
}
State(ECSJS_DONE);
//printf("done\n");
}
else
{
printf("failed, fallback to local\n");
}
}
else
{
printf("failed, fallback to local\n");
}
}
if (State() == ECSJS_NONE)
{
if (!Compile(pElement, rVec) || rVec.size() == 0)
{
State(ECSJS_ERROR_COMPILE);
CrySimple_ERROR("failed to compile request");
return false;
}
tdDataVector rDataRaw;
rDataRaw.swap(rVec);
if (!CSTLHelper::Compress(rDataRaw, rVec))
{
State(ECSJS_ERROR_COMPRESS);
CrySimple_ERROR("failed to compress request");
return false;
}
State(ECSJS_DONE);
}
// Cache compiled data
const char* pCaching = pElement->Attribute("Caching");
if (State() != ECSJS_ERROR && (!pCaching || std::string(pCaching) == "1"))
{
CCrySimpleCache::Instance().Add(HashID(), rVec);
}
return true;
}
bool CCrySimpleJobCompile::Compile(const TiXmlElement* pElement, std::vector<uint8_t>& rVec)
{
AZStd::string platform;
AZStd::string compiler;
AZStd::string language;
AZStd::string shaderPath;
if (m_Version >= EPV_V0023)
{
// NOTE: These attributes were alredy validated.
platform = pElement->Attribute("Platform");
compiler = pElement->Attribute("Compiler");
language = pElement->Attribute("Language");
shaderPath = AZStd::string::format("%s%s-%s-%s/", SEnviropment::Instance().m_ShaderPath.c_str(), platform.c_str(), compiler.c_str(), language.c_str());
}
else
{
// In previous versions Platform attribute is the language
platform = "N/A";
language = pElement->Attribute("Platform");
// Map shader language to shader compiler key
const AZStd::unordered_map<AZStd::string, AZStd::string> languageToCompilerMap
{
{
"GL4", SEnviropment::m_GLSL_HLSLcc
},{
"GLES3_0", SEnviropment::m_GLSL_HLSLcc
},{
"GLES3_1", SEnviropment::m_GLSL_HLSLcc
},{
"DX11", SEnviropment::m_D3D11_FXC
},{
"METAL", SEnviropment::m_METAL_HLSLcc
},{
"ORBIS", SEnviropment::m_Orbis_DXC
},{
"DURANGO", SEnviropment::m_Durango_FXC
},{
"JASPER", SEnviropment::m_Jasper_FXC
}
};
auto foundShaderLanguage = languageToCompilerMap.find(language);
if (foundShaderLanguage == languageToCompilerMap.end())
{
State(ECSJS_ERROR_INVALID_LANGUAGE);
CrySimple_ERROR("Trying to compile with invalid shader language");
return false;
}
if (m_Version < EPV_V0022)
{
compiler = "N/A"; // Compiler exe will be specified inside 'compile flags', this variable won't be used
}
else
{
compiler = foundShaderLanguage->second;
if (!SEnviropment::Instance().IsShaderCompilerValid(compiler))
{
State(ECSJS_ERROR_INVALID_COMPILER);
CrySimple_ERROR("Trying to compile with invalid shader compiler");
return false;
}
}
shaderPath = AZStd::string::format("%s%s/", SEnviropment::Instance().m_ShaderPath.c_str(), language.c_str());
}
NormalizePath(shaderPath);
if (!IsPathValid(shaderPath))
{
State(ECSJS_ERROR);
CrySimple_ERROR("Shaders output path is invalid");
return false;
}
// Create shaders directory
AZ::IO::SystemFile::CreateDir( shaderPath.c_str() );
const char* pProfile = pElement->Attribute("Profile");
const char* pProgram = pElement->Attribute("Program");
const char* pEntry = pElement->Attribute("Entry");
const char* pCompileFlags = pElement->Attribute("CompileFlags");
const char* pShaderRequestLine = pElement->Attribute("ShaderRequest");
if (!pProfile)
{
State(ECSJS_ERROR_INVALID_PROFILE);
CrySimple_ERROR("failed to extract Profile of the request");
return false;
}
if (!pProgram)
{
State(ECSJS_ERROR_INVALID_PROGRAM);
CrySimple_ERROR("failed to extract Program of the request");
return false;
}
if (!pEntry)
{
State(ECSJS_ERROR_INVALID_ENTRY);
CrySimple_ERROR("failed to extract Entry of the request");
return false;
}
if (!pShaderRequestLine)
{
State(ECSJS_ERROR_INVALID_SHADERREQUESTLINE);
CrySimple_ERROR("failed to extract ShaderRequest of the request");
return false;
}
if (!pCompileFlags)
{
State(ECSJS_ERROR_INVALID_COMPILEFLAGS);
CrySimple_ERROR("failed to extract CompileFlags of the request");
return false;
}
// Validate that the shader request line has a set of open/close parens as
// the code below this expects at least the open paren to be in the string.
// Without the open paren the code below will crash the compiler
std::string strippedShaderRequestLine(pShaderRequestLine);
const size_t locationOfOpenParen = strippedShaderRequestLine.find("(");
const size_t locationOfCloseParen = strippedShaderRequestLine.find(")");
if (locationOfOpenParen == std::string::npos ||
locationOfCloseParen == std::string::npos || locationOfCloseParen < locationOfOpenParen)
{
State(ECSJS_ERROR_INVALID_SHADERREQUESTLINE);
CrySimple_ERROR("invalid ShaderRequest attribute");
return false;
}
static AZStd::atomic_long nTmpCounter = { 0 };
++nTmpCounter;
char tmpstr[64];
azsprintf(tmpstr, "%ld", static_cast<long>(nTmpCounter));
const std::string TmpIn = SEnviropment::Instance().m_TempPath + tmpstr + ".In";
const std::string TmpOut = SEnviropment::Instance().m_TempPath + tmpstr + ".Out";
CCrySimpleFileGuard FGTmpIn(TmpIn);
CCrySimpleFileGuard FGTmpOut(TmpOut);
CSTLHelper::ToFile(TmpIn, std::vector<uint8_t>(pProgram, &pProgram[strlen(pProgram)]));
const AZStd::string compilerPath = SEnviropment::Instance().m_CompilerPath.c_str();
AZStd::string command;
if (m_Version >= EPV_V0022)
{
AZStd::string compilerExecutable;
bool validCompiler = SEnviropment::Instance().GetShaderCompilerExecutable(compiler, compilerExecutable);
if (!validCompiler)
{
State(ECSJS_ERROR_INVALID_COMPILER);
CrySimple_ERROR("Trying to compile with unknown compiler");
return false;
}
AZStd::string commandStringToFormat = compilerPath + compilerExecutable;
#if defined(AZ_PLATFORM_LINUX) || defined(AZ_PLATFORM_MAC)
// Surrounding compiler path+executable with quotes to support spaces in the path.
// NOTE: Executable has a space at the end on purpose, inserting quote before.
commandStringToFormat.insert(0, "\"");
commandStringToFormat.insert(commandStringToFormat.length()-1, "\"");
#endif
commandStringToFormat.append(pCompileFlags);
if (strstr(pCompileFlags, "-fxc") != nullptr)
{
AZStd::string fxcCompilerExecutable;
bool validFXCCompiler = SEnviropment::Instance().GetShaderCompilerExecutable(SEnviropment::m_D3D11_FXC, fxcCompilerExecutable);
if (!validFXCCompiler)
{
State(ECSJS_ERROR_INVALID_COMPILER);
CrySimple_ERROR("FXC compiler executable cannot be found");
return false;
}
AZStd::string fxcLocation = compilerPath + fxcCompilerExecutable;
// Handle an extra string parameter to specify the base directory where the fxc compiler is located
command = AZStd::move(AZStd::string::format(commandStringToFormat.c_str(), fxcLocation.c_str(), pEntry, pProfile, TmpOut.c_str(), TmpIn.c_str()));
}
else
{
command = AZStd::move(AZStd::string::format(commandStringToFormat.c_str(), pEntry, pProfile, TmpOut.c_str(), TmpIn.c_str()));
}
}
else
{
if (!ValidateExecutableStringLegacy(pCompileFlags))
{
State(ECSJS_ERROR_INVALID_COMPILEFLAGS);
CrySimple_ERROR("CompileFlags failed validation");
return false;
}
if (strstr(pCompileFlags, "-fxc=\"%s") != nullptr)
{
// Check that the string after the %s is a valid shader compiler
// executable
AZStd::string tempString(pCompileFlags);
const AZStd::string::size_type fxcOffset = tempString.find("%s") + 2;
const AZStd::string::size_type endOfFxcString = tempString.find(" ", fxcOffset);
tempString = tempString.substr(fxcOffset, endOfFxcString);
if (!ValidateExecutableStringLegacy(tempString))
{
State(ECSJS_ERROR_INVALID_COMPILEFLAGS);
CrySimple_ERROR("CompileFlags failed validation");
return false;
}
// Handle an extra string parameter to specify the base directory where the fxc compiler is located
command = AZStd::move(AZStd::string::format(pCompileFlags, compilerPath.c_str(), pEntry, pProfile, TmpOut.c_str(), TmpIn.c_str()));
// Need to add the string for escaped quotes around the path to the compiler. This is in case the path has spaces.
// Adding just quotes (escaped) doesn't work because this cmd line is used to execute another process.
AZStd::string insertPattern = "\\\"";
// Search for the next space until that path exists. Then we assume that's the path to the executable.
size_t startPos = command.find(compilerPath);
for (size_t pos = command.find(" ", startPos); pos != AZStd::string::npos; pos = command.find(" ", pos + 1))
{
if (AZ::IO::SystemFile::Exists(command.substr(startPos, pos - startPos).c_str()))
{
command.insert(pos, insertPattern);
command.insert(startPos, insertPattern);
}
}
}
else
{
command = AZStd::move(AZStd::string::format(pCompileFlags, pEntry, pProfile, TmpOut.c_str(), TmpIn.c_str()));
}
command = compilerPath + command;
}
AZStd::string hardwareTarget;
#if defined(AZ_TOOLS_EXPAND_FOR_RESTRICTED_PLATFORMS)
#if defined(TOOLS_SUPPORT_XENIA)
#define AZ_RESTRICTED_SECTION CRYSIMPLEJOBCOMPILE_CPP_SECTION_1
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleJobCompile_cpp, xenia)
#endif
#if defined(TOOLS_SUPPORT_JASPER)
#define AZ_RESTRICTED_SECTION CRYSIMPLEJOBCOMPILE_CPP_SECTION_1
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleJobCompile_cpp, jasper)
#endif
#if defined(TOOLS_SUPPORT_PROVO)
#define AZ_RESTRICTED_SECTION CRYSIMPLEJOBCOMPILE_CPP_SECTION_1
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleJobCompile_cpp, provo)
#endif
#if defined(TOOLS_SUPPORT_SALEM)
#define AZ_RESTRICTED_SECTION CRYSIMPLEJOBCOMPILE_CPP_SECTION_1
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleJobCompile_cpp, salem)
#endif
#endif
int64_t t0 = g_Timer.GetTime();
std::string outError;
std::string shaderName;
std::stringstream crcStringStream;
// Dump source shader
if (SEnviropment::Instance().m_DumpShaders)
{
unsigned long crc = crc32(0l, Z_NULL, 0);
// shader permutations start with '('
size_t position = strippedShaderRequestLine.find('(');
// split the string into shader name
shaderName = strippedShaderRequestLine.substr(0, position);
// split the string into permutation
std::string permutation = strippedShaderRequestLine.substr(position, strippedShaderRequestLine.length() - position);
// replace illegal filename characters with valid ones
AZStd::replace(shaderName.begin(), shaderName.end(), '<', '(');
AZStd::replace(shaderName.begin(), shaderName.end(), '>', ')');
AZStd::replace(shaderName.begin(), shaderName.end(), '/', '_');
AZStd::replace(shaderName.begin(), shaderName.end(), '|', '+');
AZStd::replace(shaderName.begin(), shaderName.end(), '*', '^');
AZStd::replace(shaderName.begin(), shaderName.end(), ':', ';');
AZStd::replace(shaderName.begin(), shaderName.end(), '?', '!');
AZStd::replace(shaderName.begin(), shaderName.end(), '%', '$');
crc = crc32(crc, reinterpret_cast<const unsigned char*>(permutation.c_str()), static_cast<unsigned int>(permutation.length()));
crcStringStream << crc;
const std::string HlslDump = shaderPath.c_str() + shaderName + "_" + crcStringStream.str() + ".hlsl";
CSTLHelper::ToFile(HlslDump, std::vector<uint8_t>(pProgram, &pProgram[strlen(pProgram)]));
std::ofstream crcFile;
std::string crcFileName = shaderPath.c_str() + shaderName + "_" + crcStringStream.str() + ".txt";
crcFile.open(crcFileName, std::ios_base::trunc);
if (!crcFile.fail())
{
// store permutation
crcFile << permutation;
}
else
{
std::cout << "Error opening file " + crcFileName << std::endl;
}
crcFile.close();
}
if (SEnviropment::Instance().m_PrintCommands)
{
AZ_Printf(0, "Compiler Command:\n%s\n\n", command.c_str());
}
if (!ExecuteCommand(command.c_str(), outError))
{
unsigned char* nIP = (unsigned char*) &RequestIP();
char sIP[128];
azsprintf(sIP, "%d.%d.%d.%d", nIP[0], nIP[1], nIP[2], nIP[3]);
const char* pProject = pElement->Attribute("Project");
const char* pTags = pElement->Attribute("Tags");
const char* pEmailCCs = pElement->Attribute("EmailCCs");
std::string project = pProject ? pProject : "Unk/";
std::string ccs = pEmailCCs ? pEmailCCs : "";
std::string tags = pTags ? pTags : "";
std::string filteredError;
CSTLHelper::Replace(filteredError, outError, TmpIn + ".patched", "%filename%"); // DXPS does its own patching
CSTLHelper::Replace(filteredError, filteredError, TmpIn, "%filename%");
// replace any that don't have the full path
CSTLHelper::Replace(filteredError, filteredError, std::string(tmpstr) + ".In.patched", "%filename%"); // DXPS does its own patching
CSTLHelper::Replace(filteredError, filteredError, std::string(tmpstr) + ".In", "%filename%");
CSTLHelper::Replace(filteredError, filteredError, "\r\n", "\n");
State(ECSJS_ERROR_COMPILE);
throw new CCompilerError(pEntry, filteredError, ccs, sIP, pShaderRequestLine, pProgram, project, platform.c_str(), compiler.c_str(), language.c_str(), tags, pProfile);
}
if (!CSTLHelper::FromFile(TmpOut, rVec))
{
State(ECSJS_ERROR_FILEIO);
std::string errorString("Could not read: ");
errorString += TmpOut;
CrySimple_ERROR(errorString.c_str());
return false;
}
// Dump cross-compiled shader
if (SEnviropment::Instance().m_DumpShaders)
{
AZStd::string fileExtension = language;
AZStd::transform(fileExtension.begin(), fileExtension.end(), fileExtension.begin(), tolower);
std::string shaderDump = shaderPath.c_str() + shaderName + "_" + crcStringStream.str() + "." + fileExtension.c_str();
CSTLHelper::ToFile(shaderDump, rVec);
}
int64_t t1 = g_Timer.GetTime();
int64_t dt = t1 - t0;
m_GlobalCompileTime += dt;
int millis = (int)(g_Timer.TimeToSeconds(dt) * 1000.0);
int secondsTotal = (int)g_Timer.TimeToSeconds(m_GlobalCompileTime);
logmessage("Compiled [%5dms|%8ds] (%s - %s - %s - %s) %s\n", millis, secondsTotal, platform.c_str(), compiler.c_str(), language.c_str(), pProfile, pEntry);
if (hardwareTarget.empty())
{
logmessage("Compiled [%5dms|%8ds] (% 5s %s) %s\n", millis, secondsTotal, platform.c_str(), pProfile, pEntry);
}
else
{
logmessage("Compiled [%5dms|%8ds] (% 5s %s) %s %s\n", millis, secondsTotal, platform.c_str(), pProfile, pEntry, hardwareTarget.c_str());
}
return true;
}
//////////////////////////////////////////////////////////////////////////
inline bool SortByLinenum(const std::pair<int, std::string>& f1, const std::pair<int, std::string>& f2)
{
return f1.first < f2.first;
}
CCompilerError::CCompilerError(const std::string& entry, const std::string& errortext, const std::string& ccs, const std::string& IP,
const std::string& requestLine, const std::string& program, const std::string& project,
const std::string& platform, const std::string& compiler, const std::string& language, const std::string& tags, const std::string& profile)
: ICryError(COMPILE_ERROR)
, m_entry(entry)
, m_errortext(errortext)
, m_IP(IP)
, m_program(program)
, m_project(project)
, m_platform(platform)
, m_compiler(compiler)
, m_language(language)
, m_tags(tags)
, m_profile(profile)
, m_uniqueID(0)
{
m_requests.push_back(requestLine);
Init();
CSTLHelper::Tokenize(m_CCs, ccs, ";");
}
void CCompilerError::Init()
{
while (!m_errortext.empty() && (m_errortext.back() == '\r' || m_errortext.back() == '\n'))
{
m_errortext.pop_back();
}
if (m_requests[0].size())
{
m_shader = m_requests[0];
size_t offs = m_shader.find('>');
if (offs != std::string::npos)
{
m_shader.erase(0, m_shader.find('>') + 1); // remove <2> version
}
offs = m_shader.find('@');
if (offs != std::string::npos)
{
m_shader.erase(m_shader.find('@')); // remove everything after @
}
offs = m_shader.find('/');
if (offs != std::string::npos)
{
m_shader.erase(m_shader.find('/')); // remove everything after / (used on xenon)
}
}
else
{
// default to entry function
m_shader = m_entry;
size_t len = m_shader.length();
// if it ends in ?S then trim those two characters
if (m_shader[len - 1] == 'S')
{
m_shader.pop_back();
m_shader.pop_back();
}
}
std::vector<std::string> lines;
CSTLHelper::Tokenize(lines, m_errortext, "\n");
for (uint32_t i = 0; i < lines.size(); i++)
{
std::string& line = lines[i];
if (line.substr(0, 5) == "error")
{
m_errors.push_back(std::pair<int, std::string>(-1, line));
m_hasherrors += line;
continue;
}
if (line.find(": error") == std::string::npos)
{
continue;
}
if (line.substr(0, 10) != "%filename%")
{
continue;
}
if (line[10] != '(')
{
continue;
}
uint32_t c = 11;
int linenum = 0;
{
bool ln = true;
while (c < line.length() &&
((line[c] >= '0' && line[c] <= '9') || line[c] == ',' || line[c] == '-')
)
{
if (line[c] == ',')
{
ln = false; // reached column, don't save the value - just keep reading to the end
}
if (ln)
{
linenum *= 10;
linenum += line[c] - '0';
}
c++;
}
if (c >= line.length())
{
continue;
}
if (line[c] != ')')
{
continue;
}
c++;
}
while (c < line.length() && (line[c] == ' ' || line[c] == ':'))
{
c++;
}
if (line.substr(c, 5) != "error")
{
continue;
}
m_errors.push_back(std::pair<int, std::string>(linenum, line));
m_hasherrors += line.substr(c);
}
AZStd::sort(m_errors.begin(), m_errors.end(), SortByLinenum);
}
std::string CCompilerError::GetErrorLines() const
{
std::string ret = "";
for (uint32_t i = 0; i < m_errors.size(); i++)
{
if (m_errors[i].first < 0)
{
ret += m_errors[i].second + "\n";
}
else if (i > 0 && m_errors[i - 1].first < 0)
{
ret += "\n" + GetContext(m_errors[i].first) + "\n" + m_errors[i].second + "\n\n";
}
else if (i > 0 && m_errors[i - 1].first == m_errors[i].first)
{
ret.pop_back(); // pop extra newline
ret += m_errors[i].second + "\n\n";
}
else
{
ret += GetContext(m_errors[i].first) + "\n" + m_errors[i].second + "\n\n";
}
}
return ret;
}
std::string CCompilerError::GetContext(int linenum, int context, std::string prefix) const
{
std::vector<std::string> lines;
CSTLHelper::Tokenize(lines, m_program, "\n");
std::string ret = "";
linenum--; // line numbers start at one
char sLineNum[16];
for (uint32_t i = AZStd::GetMax(0U, (uint32_t)(linenum - context)); i <= AZStd::GetMin((uint32_t)lines.size() - 1U, (uint32_t)(linenum + context)); i++)
{
azsprintf(sLineNum, "% 3d", i + 1);
ret += sLineNum;
ret += " ";
if (prefix.size())
{
if (i == linenum)
{
ret += "*";
}
else
{
ret += " ";
}
ret += prefix;
ret += " ";
}
ret += lines[i] + "\n";
}
return ret;
}
void CCompilerError::AddDuplicate(ICryError* err)
{
ICryError::AddDuplicate(err);
if (err->GetType() == COMPILE_ERROR)
{
CCompilerError* comperr = (CCompilerError*)err;
m_requests.insert(m_requests.end(), comperr->m_requests.begin(), comperr->m_requests.end());
}
}
bool CCompilerError::Compare(const ICryError* err) const
{
if (GetType() != err->GetType())
{
return GetType() < err->GetType();
}
CCompilerError* e = (CCompilerError*)err;
if (m_platform != e->m_platform)
{
return m_platform < e->m_platform;
}
if (m_compiler != e->m_compiler)
{
return m_compiler < e->m_compiler;
}
if (m_language != e->m_language)
{
return m_language < e->m_language;
}
if (m_shader != e->m_shader)
{
return m_shader < e->m_shader;
}
if (m_entry != e->m_entry)
{
return m_entry < e->m_entry;
}
return Hash() < err->Hash();
}
bool CCompilerError::CanMerge(const ICryError* err) const
{
if (GetType() != err->GetType()) // don't merge with non compile errors
{
return false;
}
CCompilerError* e = (CCompilerError*)err;
if (m_platform != e->m_platform || m_compiler != e->m_compiler || m_language != e->m_language || m_shader != e->m_shader)
{
return false;
}
if (m_CCs.size() != e->m_CCs.size())
{
return false;
}
for (size_t a = 0, S = m_CCs.size(); a < S; a++)
{
if (m_CCs[a] != e->m_CCs[a])
{
return false;
}
}
return true;
}
void CCompilerError::AddCCs(std::set<std::string>& ccs) const
{
for (size_t a = 0, S = m_CCs.size(); a < S; a++)
{
ccs.insert(m_CCs[a]);
}
}
std::string CCompilerError::GetErrorName() const
{
return std::string("[") + m_tags + "] Shader Compile Errors in " + m_shader + " on " + m_language + " for " + m_platform + " " + m_compiler;
}
std::string CCompilerError::GetErrorDetails(EOutputFormatType outputType) const
{
std::string errorString("");
char sUniqueID[16], sNumDuplicates[16];
azsprintf(sUniqueID, "%d", m_uniqueID);
azsprintf(sNumDuplicates, "%d", NumDuplicates());
std::string errorOutput;
CSTLHelper::Replace(errorOutput, GetErrorLines(), "%filename%", std::string(sUniqueID) + "-" + GetFilename());
std::string fullOutput;
CSTLHelper::Replace(fullOutput, m_errortext, "%filename%", std::string(sUniqueID) + "-" + GetFilename());
if (outputType == OUTPUT_HASH)
{
errorString = GetFilename() + m_IP + m_platform + m_compiler + m_language + m_project + m_entry + m_tags + m_profile + m_hasherrors /*+ m_requestline*/;
}
else if (outputType == OUTPUT_EMAIL)
{
errorString = std::string("=== Shader compile error in ") + m_entry + " (" + sNumDuplicates + " duplicates)\n\n";
/////
errorString += std::string("* From: ") + m_IP + " on " + m_language + " for " + m_platform + " " + m_compiler + " " + m_project;
if (m_tags != "")
{
errorString += std::string(" (Tags: ") + m_tags + ")";
}
errorString += "\n";
/////
errorString += std::string("* Target profile: ") + m_profile + "\n";
/////
bool hasrequests = false;
for (uint32_t i = 0; i < m_requests.size(); i++)
{
if (m_requests[i].size())
{
errorString += std::string("* Shader request line: ") + m_requests[i] + "\n";
hasrequests = true;
}
}
errorString += "\n";
if (hasrequests)
{
errorString += "* Shader source from first listed request\n";
}
errorString += std::string("* Reported error(s) from ") + sUniqueID + "-" + GetFilename() + "\n\n";
errorString += errorOutput + "\n\n";
errorString += std::string("* Full compiler output:\n\n");
errorString += fullOutput + "\n";
}
else if (outputType == OUTPUT_TTY)
{
errorString = std::string("=== Shader compile error in ") + m_entry + " { " + m_requests[0] + " }\n";
// errors only
errorString += std::string("* Reported error(s):\n\n");
errorString += errorOutput;
errorString += m_errortext;
}
return errorString;
}
@@ -0,0 +1,92 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLEJOBCOMPILE__
#define __CRYSIMPLEJOBCOMPILE__
#include "CrySimpleJobCache.hpp"
#include <Core/Common.h>
#include <Core/Error.hpp>
class CCrySimpleJobCompile
: public CCrySimpleJobCache
{
public:
CCrySimpleJobCompile(uint32_t requestIP, EProtocolVersion Version, std::vector<uint8_t>* pRVec);
virtual ~CCrySimpleJobCompile();
virtual bool Execute(const TiXmlElement* pElement);
static long GlobalCompileTasks(){return m_GlobalCompileTasks; }
static long GlobalCompileTasksMax(){return m_GlobalCompileTasksMax; }
private:
static AZStd::atomic_long m_GlobalCompileTasks;
static AZStd::atomic_long m_GlobalCompileTasksMax;
static volatile int32_t m_RemoteServerID;
static volatile int64_t m_GlobalCompileTime;
EProtocolVersion m_Version;
std::vector<uint8_t>* m_pRVec;
virtual size_t SizeOf(std::vector<uint8_t>& rVec) = 0;
bool Compile(const TiXmlElement* pElement, std::vector<uint8_t>& rVec);
};
class CCompilerError
: public ICryError
{
public:
CCompilerError(const std::string& entry, const std::string& errortext, const std::string& ccs, const std::string& IP,
const std::string& requestLine, const std::string& program, const std::string& project,
const std::string& platform, const std::string& compiler, const std::string& language, const std::string& tags, const std::string& profile);
virtual ~CCompilerError() {}
virtual void AddDuplicate(ICryError* err);
virtual void SetUniqueID(int uniqueID) { m_uniqueID = uniqueID; }
virtual bool Compare(const ICryError* err) const;
virtual bool CanMerge(const ICryError* err) const;
virtual bool HasFile() const { return true; }
virtual void AddCCs(std::set<std::string>& ccs) const;
virtual std::string GetErrorName() const;
virtual std::string GetErrorDetails(EOutputFormatType outputType) const;
virtual std::string GetFilename() const { return m_entry + ".txt"; }
virtual std::string GetFileContents() const { return m_program; }
std::vector<std::string> m_requests;
private:
void Init();
std::string GetErrorLines() const;
std::string GetContext(int linenum, int context = 2, std::string prefix = ">") const;
std::vector< std::pair<int, std::string> > m_errors;
tdEntryVec m_CCs;
std::string m_entry, m_errortext, m_hasherrors, m_IP,
m_program, m_project, m_shader,
m_platform, m_compiler, m_language, m_tags, m_profile;
int m_uniqueID;
friend CCompilerError;
};
#endif
@@ -0,0 +1,31 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <tinyxml/tinyxml.h>
#include "CrySimpleSock.hpp"
#include "CrySimpleJobCompile1.hpp"
CCrySimpleJobCompile1::CCrySimpleJobCompile1(uint32_t requestIP, std::vector<uint8_t>* pRVec)
: CCrySimpleJobCompile(requestIP, EPV_V001, pRVec)
{
}
size_t CCrySimpleJobCompile1::SizeOf(std::vector<uint8_t>& rVec)
{
return rVec.size();
}
@@ -0,0 +1,29 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLEJOBCOMPILE1__
#define __CRYSIMPLEJOBCOMPILE1__
#include "CrySimpleJobCompile.hpp"
class CCrySimpleJobCompile1
: public CCrySimpleJobCompile
{
virtual size_t SizeOf(std::vector<uint8_t>& rVec);
public:
CCrySimpleJobCompile1(uint32_t requestIP, std::vector<uint8_t>* pRVec);
};
#endif
@@ -0,0 +1,34 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySimpleSock.hpp"
#include "CrySimpleJobCompile2.hpp"
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <tinyxml/tinyxml.h>
CCrySimpleJobCompile2::CCrySimpleJobCompile2(EProtocolVersion version, uint32_t requestIP, std::vector<uint8_t>* pRVec)
: CCrySimpleJobCompile(requestIP, version, pRVec)
{
}
size_t CCrySimpleJobCompile2::SizeOf(std::vector<uint8_t>& rVec)
{
const char* pXML = reinterpret_cast<const char*>(&rVec[0]);
const char* pFirst = strstr(pXML, "HashStop");
return pFirst ? pFirst - pXML : rVec.size();
}
@@ -0,0 +1,29 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLEJOBCOMPILE2__
#define __CRYSIMPLEJOBCOMPILE2__
#include "CrySimpleJobCompile.hpp"
class CCrySimpleJobCompile2
: public CCrySimpleJobCompile
{
virtual size_t SizeOf(std::vector<uint8_t>& rVec);
public:
CCrySimpleJobCompile2(EProtocolVersion version, uint32_t requestIP, std::vector<uint8_t>* pRVec);
};
#endif
@@ -0,0 +1,82 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include "CrySimpleJobGetShaderList.hpp"
#include "ShaderList.hpp"
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <Core/Common.h>
#include <tinyxml/tinyxml.h>
#include <AzCore/Debug/Trace.h>
#include <AzCore/std/string/string.h>
CCrySimpleJobGetShaderList::CCrySimpleJobGetShaderList(uint32_t requestIP, std::vector<uint8_t>* pRVec)
: CCrySimpleJob(requestIP)
, m_pRVec(pRVec)
{
}
bool CCrySimpleJobGetShaderList::Execute(const TiXmlElement* pElement)
{
AZStd::string shaderListFilename;
const char* project = pElement->Attribute("Project");
const char* shaderList = pElement->Attribute("ShaderList");
const char* platform = pElement->Attribute("Platform");
const char* compiler = pElement->Attribute("Compiler");
const char* language = pElement->Attribute("Language");
shaderListFilename = AZStd::string::format("./Cache/%s%s-%s-%s/%s", project, platform, compiler, language, shaderList);
//open the file and read into the rVec
FILE* pFile = nullptr;
azfopen(&pFile, shaderListFilename.c_str(), "rb");
if (!pFile)
{
// Fake a good result. We can't be sure if this file name is bad or if it doesn't exist *yet*, so we'll just assume the latter.
m_pRVec->resize(4, '\0');
State(ECSJS_DONE);
return true;
}
fseek(pFile, 0, SEEK_END);
size_t fileSize = ftell(pFile);
m_pRVec->resize(fileSize);
fseek(pFile, 0, SEEK_SET);
size_t remaining = fileSize;
size_t read = 0;
while (remaining)
{
read += fread(m_pRVec->data() + read, 1, remaining, pFile);
remaining -= read;
}
fclose(pFile);
//compress before sending
tdDataVector rDataRaw;
rDataRaw.swap(*m_pRVec);
if (!CSTLHelper::Compress(rDataRaw, *m_pRVec))
{
State(ECSJS_ERROR_COMPRESS);
CrySimple_ERROR("failed to compress request");
return false;
}
State(ECSJS_DONE);
return true;
}
@@ -0,0 +1,29 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#ifndef __CRYSIMPLEJOBGETSHADERLIST__
#define __CRYSIMPLEJOBGETSHADERLIST__
#include "CrySimpleJob.hpp"
class CCrySimpleJobGetShaderList
: public CCrySimpleJob
{
public:
CCrySimpleJobGetShaderList(uint32_t requestIP, std::vector<uint8_t>* pRVec);
virtual bool Execute(const TiXmlElement* pElement);
std::vector<uint8_t>* m_pRVec = nullptr;
};
#endif
@@ -0,0 +1,93 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySimpleJobRequest.hpp"
#include "CrySimpleServer.hpp"
#include "ShaderList.hpp"
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <Core/Common.h>
#include <tinyxml/tinyxml.h>
#include <Core/WindowsAPIImplementation.h>
#include <AzCore/Debug/Trace.h>
#include <AzCore/IO/SystemFile.h>
CCrySimpleJobRequest::CCrySimpleJobRequest(EProtocolVersion Version, uint32_t requestIP)
: CCrySimpleJob(requestIP)
, m_Version(Version)
{
}
bool CCrySimpleJobRequest::Execute(const TiXmlElement* pElement)
{
const char* shaderRequest = pElement->Attribute("ShaderRequest");
if (!shaderRequest)
{
State(ECSJS_ERROR_INVALID_SHADERREQUESTLINE);
CrySimple_ERROR("Missing shader request line");
return false;
}
AZStd::string shaderListFilename;
if (m_Version >= EPV_V0023)
{
const char* project = pElement->Attribute("Project");
const char* shaderList = pElement->Attribute("ShaderList");
if (!project)
{
State(ECSJS_ERROR_INVALID_PROJECT);
CrySimple_ERROR("Missing Project for shader request");
return false;
}
if (!shaderList)
{
State(ECSJS_ERROR_INVALID_SHADERLIST);
CrySimple_ERROR("Missing Shader List for shader request");
return false;
}
// NOTE: These attributes were alredy validated.
AZStd::string platform = pElement->Attribute("Platform");
AZStd::string compiler = pElement->Attribute("Compiler");
AZStd::string language = pElement->Attribute("Language");
shaderListFilename = AZStd::string::format("%s%s-%s-%s/%s", project, platform.c_str(), compiler.c_str(), language.c_str(), shaderList);
}
else
{
// In previous versions Platform attribute is the shader list filename directly
shaderListFilename = pElement->Attribute("Platform");
}
if (shaderListFilename.length() >= AZ_MAX_PATH_LEN)
{
State(ECSJS_ERROR);
CrySimple_ERROR("Shader list filename is too long");
return false;
}
std::string shaderRequestLine(shaderRequest);
tdEntryVec toks;
CSTLHelper::Tokenize(toks, shaderRequestLine, ";");
for (size_t a = 0, s = toks.size(); a < s; a++)
{
CShaderList::Instance().Add(shaderListFilename.c_str(), toks[a].c_str());
}
State(ECSJS_DONE);
return true;
}
@@ -0,0 +1,33 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLEJOBREQUEST__
#define __CRYSIMPLEJOBREQUEST__
#include "CrySimpleJob.hpp"
#include "CrySimpleSock.hpp"
class CCrySimpleJobRequest
: public CCrySimpleJob
{
public:
CCrySimpleJobRequest(EProtocolVersion Version, uint32_t requestIP);
virtual bool Execute(const TiXmlElement* pElement);
private:
EProtocolVersion m_Version;
};
#endif
@@ -0,0 +1,57 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySimpleMutex.hpp"
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
CCrySimpleMutex::CCrySimpleMutex()
{
#if defined(AZ_PLATFORM_WINDOWS)
InitializeCriticalSectionAndSpinCount(&cs, 10000);
#else
pthread_mutex_init(&m_Mutex, nullptr);
#endif
}
CCrySimpleMutex::~CCrySimpleMutex()
{
#if defined(AZ_PLATFORM_WINDOWS)
DeleteCriticalSection(&cs);
#else
pthread_mutex_destroy(&m_Mutex);
#endif
}
void CCrySimpleMutex::Lock()
{
#if defined(AZ_PLATFORM_WINDOWS)
EnterCriticalSection(&cs);
#else
pthread_mutex_lock(&m_Mutex);
#endif
}
void CCrySimpleMutex::Unlock()
{
#if defined(AZ_PLATFORM_WINDOWS)
LeaveCriticalSection(&cs);
#else
pthread_mutex_unlock(&m_Mutex);
#endif
}
@@ -0,0 +1,53 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLEMUTEX__
#define __CRYSIMPLEMUTEX__
#include <Core/Common.h>
#if !defined(AZ_PLATFORM_WINDOWS)
#include "pthread.h"
#endif
class CCrySimpleMutex
{
#if defined(AZ_PLATFORM_WINDOWS)
CRITICAL_SECTION cs;
#else
// Use posix thread support
pthread_mutex_t m_Mutex;
#endif
public:
CCrySimpleMutex();
~CCrySimpleMutex();
void Lock();
void Unlock();
};
class CCrySimpleMutexAutoLock
{
CCrySimpleMutex& m_rMutex;
public:
CCrySimpleMutexAutoLock(CCrySimpleMutex& rMutex)
: m_rMutex(rMutex)
{
rMutex.Lock();
}
~CCrySimpleMutexAutoLock()
{
m_rMutex.Unlock();
}
};
#endif
@@ -0,0 +1,795 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include <AzCore/base.h>
#include <AzCore/PlatformDef.h>
#include <AzCore/std/parallel/thread.h>
#include "CrySimpleServer.hpp"
#include "CrySimpleSock.hpp"
#include "CrySimpleJob.hpp"
#include "CrySimpleJobCompile1.hpp"
#include "CrySimpleJobCompile2.hpp"
#include "CrySimpleJobRequest.hpp"
#include "CrySimpleJobGetShaderList.hpp"
#include "CrySimpleCache.hpp"
#include "CrySimpleErrorLog.hpp"
#include "ShaderList.hpp"
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <tinyxml/tinyxml.h>
#include <Core/WindowsAPIImplementation.h>
#include <Core/Mailer.h>
#include <AzFramework/StringFunc/StringFunc.h>
#include <AzCore/Debug/Trace.h>
#include <AzCore/IO/Path/Path.h>
#include <AzCore/IO/SystemFile.h>
#include <AzCore/Jobs/Job.h>
#include <AzCore/Jobs/JobFunction.h>
#include <AzCore/std/functional.h>
#include <AzCore/std/time.h>
#include <AzCore/std/bind/bind.h>
#include <AzCore/std/string/string.h>
#include <AzCore/Utils/Utils.h>
#if defined(AZ_TOOLS_EXPAND_FOR_RESTRICTED_PLATFORMS)
#undef AZ_RESTRICTED_SECTION
#define CRYSIMPLESERVER_CPP_SECTION_1 1
#define CRYSIMPLESERVER_CPP_SECTION_2 2
#endif
#if defined(AZ_PLATFORM_MAC)
#include <libproc.h>
#include <sys/stat.h>
#endif
#include <assert.h>
#include <algorithm>
#include <memory>
#include <unordered_map>
#ifdef WIN32
#define EXTENSION ".exe"
#else
#define EXTENSION ""
#endif
AZStd::atomic_long CCrySimpleServer::ms_ExceptionCount = {0};
const static std::string SHADER_PROFILER = "NVShaderPerf" EXTENSION;
const static std::string SHADER_PATH_SOURCE = "Source";
const static std::string SHADER_PATH_BINARY = "Binary";
const static std::string SHADER_PATH_HALFSTRIPPED = "HalfStripped";
const static std::string SHADER_PATH_DISASSEMBLED = "DisAsm";
const static std::string SHADER_PATH_STRIPPPED = "Stripped";
const static std::string SHADER_PATH_CACHE = "Cache";
static const bool autoDeleteJobWhenDone = true;
static const int sleepTimeWhenWaiting = 10;
static AZStd::atomic_long g_ConnectionCount = {0};
SEnviropment* SEnviropment::m_instance=nullptr;
void SEnviropment::Create()
{
if (!m_instance)
{
m_instance = new SEnviropment;
}
}
void SEnviropment::Destroy()
{
if (m_instance)
{
delete m_instance;
m_instance = nullptr;
}
}
SEnviropment& SEnviropment::Instance()
{
AZ_Assert(m_instance, "Using SEnviropment::Instance() before calling SEnviropment::Create()");
return *m_instance;
}
// Shader Compilers ID
// NOTE: Values must be in sync with CShaderSrv::GetShaderCompilerName() function in the engine side.
const char* SEnviropment::m_Orbis_DXC = "Orbis_DXC";
const char* SEnviropment::m_Durango_FXC = "Durango_FXC";
const char* SEnviropment::m_Jasper_FXC = "Jasper_FXC";
const char* SEnviropment::m_D3D11_FXC = "D3D11_FXC";
const char* SEnviropment::m_GLSL_HLSLcc = "GLSL_HLSLcc";
const char* SEnviropment::m_METAL_HLSLcc = "METAL_HLSLcc";
const char* SEnviropment::m_GLSL_LLVM_DXC = "GLSL_LLVM_DXC";
const char* SEnviropment::m_METAL_LLVM_DXC = "METAL_LLVM_DXC";
void SEnviropment::InitializePlatformAttributes()
{
// Initialize valid Plaforms
// NOTE: Values must be in sync with CShaderSrv::GetPlatformName() function in the engine side.
m_Platforms.insert("Orbis");
m_Platforms.insert("Durango");
m_Platforms.insert("Nx");
m_Platforms.insert("PC");
m_Platforms.insert("Mac");
m_Platforms.insert("iOS");
m_Platforms.insert("Android");
m_Platforms.insert("Linux");
m_Platforms.insert("Jasper");
// Initialize valid Shader Languages
// NOTE: Values must be in sync with GetShaderLanguageName() function in the engine side.
m_ShaderLanguages.insert("Orbis");
m_ShaderLanguages.insert("Durango");
m_ShaderLanguages.insert("D3D11");
m_ShaderLanguages.insert("METAL");
m_ShaderLanguages.insert("GL4");
m_ShaderLanguages.insert("GLES3");
m_ShaderLanguages.insert("Jasper");
// These are added for legacy support (GLES3_0 and GLES3_1 are combined into just GLES3)
m_ShaderLanguages.insert("GL4_1");
m_ShaderLanguages.insert("GL4_4");
m_ShaderLanguages.insert("GLES3_0");
m_ShaderLanguages.insert("GLES3_1");
// Initialize valid Shader Compilers ID and Executables.
// Intentionally put a space after the executable name so that attackers can't try to change the executable name that we are going to run.
#if defined(AZ_TOOLS_EXPAND_FOR_RESTRICTED_PLATFORMS)
#if defined(TOOLS_SUPPORT_XENIA)
#define AZ_RESTRICTED_SECTION CRYSIMPLESERVER_CPP_SECTION_2
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleServer_cpp, xenia)
#endif
#if defined(TOOLS_SUPPORT_JASPER)
#define AZ_RESTRICTED_SECTION CRYSIMPLESERVER_CPP_SECTION_2
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleServer_cpp, jasper)
#endif
#if defined(TOOLS_SUPPORT_PROVO)
#define AZ_RESTRICTED_SECTION CRYSIMPLESERVER_CPP_SECTION_2
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleServer_cpp, provo)
#endif
#if defined(TOOLS_SUPPORT_SALEM)
#define AZ_RESTRICTED_SECTION CRYSIMPLESERVER_CPP_SECTION_2
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleServer_cpp, salem)
#endif
#endif
m_ShaderCompilersMap[m_D3D11_FXC] = "PCD3D11/v006/fxc.exe ";
m_ShaderCompilersMap[m_GLSL_HLSLcc] = "PCGL/V006/HLSLcc ";
m_ShaderCompilersMap[m_METAL_HLSLcc] = "PCGMETAL/HLSLcc/HLSLcc ";
#if defined(_DEBUG)
m_ShaderCompilersMap[m_GLSL_LLVM_DXC] = "LLVMGL/debug/dxcGL ";
m_ShaderCompilersMap[m_METAL_LLVM_DXC] = "LLVMMETAL/debug/dxcMetal ";
#else
m_ShaderCompilersMap[m_GLSL_LLVM_DXC] = "LLVMGL/release/dxcGL ";
m_ShaderCompilersMap[m_METAL_LLVM_DXC] = "LLVMMETAL/release/dxcMetal ";
#endif
}
bool SEnviropment::IsPlatformValid( const AZStd::string& platform ) const
{
return m_Platforms.find(platform) != m_Platforms.end();
}
bool SEnviropment::IsShaderLanguageValid( const AZStd::string& shaderLanguage ) const
{
return m_ShaderLanguages.find(shaderLanguage) != m_ShaderLanguages.end();
}
bool SEnviropment::IsShaderCompilerValid( const AZStd::string& shaderCompilerID ) const
{
bool validCompiler = (m_ShaderCompilersMap.find(shaderCompilerID) != m_ShaderCompilersMap.end());
// Extra check for Mac: Only GL_LLVM_DXC and METAL_LLVM_DXC compilers are supported.
#if defined(AZ_PLATFORM_MAC)
if (validCompiler &&
shaderCompilerID != m_GLSL_LLVM_DXC &&
shaderCompilerID != m_METAL_LLVM_DXC)
{
printf("error: trying to use an unsupported compiler on Mac.\n");
return false;
}
#endif
return validCompiler;
}
bool SEnviropment::GetShaderCompilerExecutable( const AZStd::string& shaderCompilerID, AZStd::string& shaderCompilerExecutable ) const
{
auto it = m_ShaderCompilersMap.find(shaderCompilerID);
if (it != m_ShaderCompilersMap.end())
{
shaderCompilerExecutable = it->second;
return true;
}
else
{
return false;
}
}
class CThreadData
{
uint32_t m_Counter;
CCrySimpleSock* m_pSock;
public:
CThreadData(uint32_t Counter, CCrySimpleSock* pSock)
: m_Counter(Counter)
, m_pSock(pSock){}
~CThreadData(){delete m_pSock; }
CCrySimpleSock* Socket(){return m_pSock; }
uint32_t ID() const{return m_Counter; }
};
//////////////////////////////////////////////////////////////////////////
bool CopyFileOnPlatform(const char* nameOfFileToCopy, const char* copiedFileName, bool failIfFileExists)
{
if (AZ::IO::SystemFile::Exists(copiedFileName) && failIfFileExists)
{
AZ_Warning("CrySimpleServer", false, ("File to copy to, %s, already exists."), copiedFileName);
return false;
}
AZ::IO::SystemFile fileToCopy;
if (!fileToCopy.Open(nameOfFileToCopy, AZ::IO::SystemFile::SF_OPEN_READ_ONLY))
{
AZ_Warning("CrySimpleServer", false, ("Unable to open file: %s for copying."), nameOfFileToCopy);
return false;
}
AZ::IO::SystemFile::SizeType fileLength = fileToCopy.Length();
AZ::IO::SystemFile newFile;
if (!newFile.Open(copiedFileName, AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY | AZ::IO::SystemFile::SF_OPEN_CREATE))
{
AZ_Warning("CrySimpleServer", false, ("Unable to open new file: %s for copying."), copiedFileName);
return false;
}
char* fileContents = new char[fileLength];
fileToCopy.Read(fileLength, fileContents);
newFile.Write(fileContents, fileLength);
delete[] fileContents;
return true;
}
void MakeErrorVec(const std::string& errorText, tdDataVector& Vec)
{
Vec.resize(errorText.size() + 1);
for (size_t i = 0; i < errorText.size(); i++)
{
Vec[i] = errorText[i];
}
Vec[errorText.size()] = 0;
// Compress output data
tdDataVector rDataRaw;
rDataRaw.swap(Vec);
if (!CSTLHelper::Compress(rDataRaw, Vec))
{
Vec.resize(0);
}
}
//////////////////////////////////////////////////////////////////////////
class CompileJob
: public AZ::Job
{
public:
CompileJob()
: Job(autoDeleteJobWhenDone, nullptr) { }
void SetThreadData(CThreadData* threadData) { m_pThreadData.reset(threadData); }
protected:
void Process() override;
bool ValidatePlatformAttributes(EProtocolVersion Version, const TiXmlElement* pElement);
private:
std::unique_ptr<CThreadData> m_pThreadData;
};
void CompileJob::Process()
{
std::vector<uint8_t> Vec;
std::unique_ptr<CCrySimpleJob> Job;
EProtocolVersion Version = EPV_V001;
ECrySimpleJobState State = ECSJS_JOBNOTFOUND;
try
{
if (m_pThreadData->Socket()->Recv(Vec))
{
std::string Request(reinterpret_cast<const char*>(&Vec[0]), Vec.size());
TiXmlDocument ReqParsed("Request.xml");
ReqParsed.Parse(Request.c_str());
if (ReqParsed.Error())
{
CrySimple_ERROR("failed to parse request XML");
return;
}
const TiXmlElement* pElement = ReqParsed.FirstChildElement();
if (!pElement)
{
CrySimple_ERROR("failed to extract First Element of the request");
return;
}
const char* pPing = pElement->Attribute("Identify");
if (pPing)
{
const std::string& rData("ShaderCompilerServer");
m_pThreadData->Socket()->Send(rData);
return;
}
const char* pVersion = pElement->Attribute("Version");
const char* pPlatform = pElement->Attribute("Platform");
const char* pHardwareTarget = nullptr;
//new request type?
if (pVersion)
{
if (std::string(pVersion) == "2.3")
{
Version = EPV_V0023;
}
else if (std::string(pVersion) == "2.2")
{
Version = EPV_V0022;
}
else if (std::string(pVersion) == "2.1")
{
Version = EPV_V0021;
}
else if (std::string(pVersion) == "2.0")
{
Version = EPV_V002;
}
}
// If the job type is 'GetShaderList', then we dont need to perform a validation on the platform
// attributes, since the command doesnt use them, and the incoming request will not have 'compiler' or 'language'
// attributes.
const char* pJobType = pElement->Attribute("JobType");
if ((!pJobType) || (azstricmp(pJobType,"GetShaderList")!=0))
{
if (!ValidatePlatformAttributes(Version, pElement))
{
return;
}
}
if (Version >= EPV_V002)
{
const std::string JobType(pJobType);
if (Version >= EPV_V0023)
{
pHardwareTarget = pElement->Attribute("HardwareTarget");
}
if (Version >= EPV_V0021)
{
m_pThreadData->Socket()->WaitForShutDownEvent(true);
}
#if defined(AZ_TOOLS_EXPAND_FOR_RESTRICTED_PLATFORMS)
#if defined(TOOLS_SUPPORT_XENIA)
#define AZ_RESTRICTED_SECTION CRYSIMPLESERVER_CPP_SECTION_1
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleServer_cpp, xenia)
#endif
#if defined(TOOLS_SUPPORT_JASPER)
#define AZ_RESTRICTED_SECTION CRYSIMPLESERVER_CPP_SECTION_1
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleServer_cpp, jasper)
#endif
#if defined(TOOLS_SUPPORT_PROVO)
#define AZ_RESTRICTED_SECTION CRYSIMPLESERVER_CPP_SECTION_1
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleServer_cpp, provo)
#endif
#if defined(TOOLS_SUPPORT_SALEM)
#define AZ_RESTRICTED_SECTION CRYSIMPLESERVER_CPP_SECTION_1
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySimpleServer_cpp, salem)
#endif
#endif
if (pJobType)
{
if (JobType == "RequestLine")
{
Job = std::make_unique<CCrySimpleJobRequest>(Version, m_pThreadData->Socket()->PeerIP());
Job->Execute(pElement);
State = Job->State();
Vec.resize(0);
}
else
if (JobType == "Compile")
{
Job = std::make_unique<CCrySimpleJobCompile2>(Version, m_pThreadData->Socket()->PeerIP(), &Vec);
Job->Execute(pElement);
State = Job->State();
}
else
if (JobType == "GetShaderList")
{
Job = std::make_unique<CCrySimpleJobGetShaderList>(m_pThreadData->Socket()->PeerIP(), &Vec);
Job->Execute(pElement);
State = Job->State();
}
else
{
printf("\nRequested unkown job %s\n", pJobType);
}
}
else
{
printf("\nVersion 2.0 or higher but has no JobType tag\n");
}
}
else
{
//legacy request
Version = EPV_V001;
Job = std::make_unique<CCrySimpleJobCompile1>(m_pThreadData->Socket()->PeerIP(), &Vec);
Job->Execute(pElement);
}
m_pThreadData->Socket()->Send(Vec, State, Version);
if (Version >= EPV_V0021)
{
/*
// wait until message has been succesfully delived before shutting down the connection
if(!m_pThreadData->Socket()->RecvResult())
{
printf("\nInvalid result from client\n");
}
*/
}
}
}
catch (const ICryError* err)
{
CCrySimpleServer::IncrementExceptionCount();
CRYSIMPLE_LOG("<Error> " + err->GetErrorName());
std::string returnStr = err->GetErrorDetails(ICryError::OUTPUT_TTY);
// Send error back
MakeErrorVec(returnStr, Vec);
if (Job.get())
{
State = Job->State();
if (State == ECSJS_ERROR_COMPILE && SEnviropment::Instance().m_PrintErrors)
{
printf("\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n");
printf("%s\n", err->GetErrorName().c_str());
printf("%s\n", returnStr.c_str());
printf("\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n\n");
}
}
bool added = CCrySimpleErrorLog::Instance().Add((ICryError*)err);
// error log hasn't taken ownership, delete this error.
if (!added)
{
delete err;
}
m_pThreadData->Socket()->Send(Vec, State, Version);
}
--g_ConnectionCount;
}
bool CompileJob::ValidatePlatformAttributes(EProtocolVersion Version, const TiXmlElement* pElement)
{
if (Version >= EPV_V0023)
{
const char* platform = pElement->Attribute("Platform"); // eg. PC, Mac...
const char* compiler = pElement->Attribute("Compiler"); // key to shader compiler executable
const char* language = pElement->Attribute("Language"); // eg. D3D11, GL4_1, GL3_1, METAL...
if (!platform || !SEnviropment::Instance().IsPlatformValid(platform))
{
CrySimple_ERROR("invalid Platform attribute from request.");
return false;
}
if (!compiler || !SEnviropment::Instance().IsShaderCompilerValid(compiler))
{
CrySimple_ERROR("invalid Compiler attribute from request.");
return false;
}
if (!language || !SEnviropment::Instance().IsShaderLanguageValid(language))
{
CrySimple_ERROR("invalid Language attribute from request.");
return false;
}
}
else
{
// In older versions the attribute Platform was used differently depending on the JobType
// - JobType Compile: Platform is the shader language
// - JobType RequestLine: Platform is the shader list filename
const char* platformLegacy = pElement->Attribute("Platform");
// The only check we can do here is if the attribute exists. Each JobType will check it has a valid value.
if (!platformLegacy)
{
CrySimple_ERROR("failed to extract required platform attribute from request.");
return false;
}
}
return true;
}
//////////////////////////////////////////////////////////////////////////
void TickThread()
{
AZ::u64 t0 = AZStd::GetTimeUTCMilliSecond();
while (true)
{
CrySimple_SECURE_START
AZ::u64 t1 = AZStd::GetTimeUTCMilliSecond();
if ((t1 < t0) || (t1 - t0 > 100))
{
t0 = t1;
const int maxStringSize = 512;
char str[maxStringSize] = { 0 };
azsnprintf(str, maxStringSize, "Amazon Shader Compiler Server (%ld compile tasks | %ld open sockets | %ld exceptions)",
CCrySimpleJobCompile::GlobalCompileTasks(), CCrySimpleSock::GetOpenSockets() + CSMTPMailer::GetOpenSockets(),
CCrySimpleServer::GetExceptionCount());
#if defined(AZ_PLATFORM_WINDOWS)
SetConsoleTitle(str);
#endif
}
const AZ::u64 T1 = AZStd::GetTimeUTCMilliSecond();
CCrySimpleErrorLog::Instance().Tick();
CShaderList::Instance().Tick();
CCrySimpleCache::Instance().ThreadFunc_SavePendingCacheEntries();
const AZ::u64 T2 = AZStd::GetTimeUTCMilliSecond();
if (T2 - T1 < 100)
{
Sleep(static_cast<DWORD>(100 - T2 + T1));
}
CrySimple_SECURE_END
}
}
//////////////////////////////////////////////////////////////////////////
void LoadCache()
{
const std::string& cachePath = SEnviropment::Instance().m_CachePath;
if (CCrySimpleCache::Instance().LoadCacheFile(cachePath + "Cache.dat"))
{
printf("Creating cache backup...\n");
AZ::IO::SystemFile::Delete((cachePath + "Cache.bak2").c_str());
printf("Move %s to %s\n", (cachePath + "Cache.bak").c_str(), (cachePath + "Cache.bak2").c_str());
AZ::IO::SystemFile::Rename((cachePath + "Cache.bak").c_str(), (cachePath + "Cache.bak2").c_str());
printf("Copy %s to %s\n", (cachePath + "Cache.dat").c_str(), (cachePath + "Cache.bak").c_str());
CopyFileOnPlatform((cachePath + "Cache.dat").c_str(), (cachePath + "Cache.bak").c_str(), FALSE);
printf("Cache backup done.\n");
}
else
{
// Restoring backup cache!
printf("Cache file corrupted!!!\n");
printf("Restoring backup cache...\n");
AZ::IO::SystemFile::Delete((cachePath + "Cache.dat").c_str());
printf("Copy %s to %s\n", (cachePath + "Cache.bak").c_str(), (cachePath + "Cache.dat").c_str());
CopyFileOnPlatform((cachePath + "Cache.bak").c_str(), (cachePath + "Cache.dat").c_str(), FALSE);
if (!CCrySimpleCache::Instance().LoadCacheFile(cachePath + "Cache.dat"))
{
// Backup file corrupted too!
printf("Backup file corrupted too!!!\n");
printf("Deleting cache completely\n");
AZ::IO::SystemFile::Delete((cachePath + "Cache.dat").c_str());
}
}
CCrySimpleCache::Instance().Finalize();
printf("Ready\n");
}
//////////////////////////////////////////////////////////////////////////
CCrySimpleServer::CCrySimpleServer([[maybe_unused]] const char* pShaderModel, [[maybe_unused]] const char* pDst, [[maybe_unused]] const char* pSrc, [[maybe_unused]] const char* pEntryFunction)
: m_pServerSocket(nullptr)
{
Init();
}
CCrySimpleServer::CCrySimpleServer()
: m_pServerSocket(nullptr)
{
CrySimple_SECURE_START
uint32_t Port = SEnviropment::Instance().m_port;
m_pServerSocket = new CCrySimpleSock(Port, SEnviropment::Instance().m_WhitelistAddresses);
Init();
m_pServerSocket->Listen();
AZ::Job* tickThreadJob = AZ::CreateJobFunction(&TickThread, autoDeleteJobWhenDone);
tickThreadJob->Start();
uint32_t JobCounter = 0;
while (1)
{
// New client message, receive new client socket connection.
CCrySimpleSock* newClientSocket = m_pServerSocket->Accept();
if(!newClientSocket)
{
continue;
}
// Thread Data for new job
CThreadData* pData = new CThreadData(JobCounter++, newClientSocket);
// Increase connection count and start new job.
// NOTE: CompileJob will be auto deleted when done, deleting thread data and client socket as well.
++g_ConnectionCount;
CompileJob* compileJob = new CompileJob();
compileJob->SetThreadData(pData);
compileJob->Start();
bool printedMessage = false;
while (g_ConnectionCount >= SEnviropment::Instance().m_MaxConnections)
{
if (!printedMessage)
{
logmessage("Waiting for a request to finish before accepting another connection...\n");
printedMessage = true;
}
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(sleepTimeWhenWaiting));
};
}
CrySimple_SECURE_END
}
bool GetBaseDirectory(AZStd::string& baseDir)
{
char executableDir[AZ_MAX_PATH_LEN];
if (AZ::Utils::GetExecutableDirectory(executableDir, AZ_MAX_PATH_LEN) == AZ::Utils::ExecutablePathResult::Success)
{
AZStd::string_view executableDirView(executableDir);
if (executableDirView.size() > 1 && !executableDirView.ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR) && executableDirView.size() < AZStd::size(executableDir) - 1)
{
executableDir[executableDirView.size()] = AZ_CORRECT_FILESYSTEM_SEPARATOR;
executableDir[executableDirView.size() + 1] = '\0';
executableDirView = { executableDir, executableDirView.size() + 1 };
}
baseDir = AZStd::string(executableDir);
return true;
}
else
{
return false;
}
}
void NormalizePath(AZStd::string& pathToNormalize)
{
AzFramework::StringFunc::Root::Normalize(pathToNormalize);
}
void NormalizePath(std::string& pathToNormalize)
{
AZStd::string tempString = pathToNormalize.c_str();
NormalizePath(tempString);
pathToNormalize = tempString.c_str();
}
bool IsPathValid(const AZStd::string& path)
{
// Calculating base directory every time.
// It's slower than using a cached value, but safer.
AZStd::string baseDir;
if (GetBaseDirectory(baseDir))
{
AZStd::string basePath(AZ::IO::Path(baseDir).LexicallyNormal().Native());
AZStd::string subPath(AZ::IO::Path(path).LexicallyNormal().Native());
return strncmp(basePath.c_str(), subPath.c_str(), basePath.size()) == 0;
}
else
{
return false;
}
}
bool IsPathValid(const std::string& path)
{
const AZStd::string tempString = path.c_str();
return IsPathValid(tempString);
}
void CCrySimpleServer::Init()
{
char executableDir[AZ_MAX_PATH_LEN];
AZ::Utils::GetExecutableDirectory(executableDir, AZ_MAX_PATH_LEN);
AZStd::string_view executableDirView(executableDir);
if (executableDirView.size() > 1 && !executableDirView.ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR) && executableDirView.size() < AZStd::size(executableDir) - 1)
{
executableDir[executableDirView.size()] = AZ_CORRECT_FILESYSTEM_SEPARATOR;
executableDir[executableDirView.size() + 1] = '\0';
executableDirView = { executableDir, executableDirView.size() + 1 };
}
SEnviropment::Instance().m_Root = std::string(executableDir);
AZStd::string baseDir;
GetBaseDirectory(baseDir);
SEnviropment::Instance().m_CompilerPath = baseDir.c_str();
SEnviropment::Instance().m_CompilerPath += "/Compiler/";
SEnviropment::Instance().m_CachePath = SEnviropment::Instance().m_Root + "Cache/";
if (SEnviropment::Instance().m_TempPath.empty())
{
SEnviropment::Instance().m_TempPath = SEnviropment::Instance().m_Root + "Temp/";
}
if (SEnviropment::Instance().m_ErrorPath.empty())
{
SEnviropment::Instance().m_ErrorPath = SEnviropment::Instance().m_Root + "Error/";
}
if (SEnviropment::Instance().m_ShaderPath.empty())
{
SEnviropment::Instance().m_ShaderPath = SEnviropment::Instance().m_Root + "Shaders/";
}
NormalizePath(SEnviropment::Instance().m_Root);
NormalizePath(SEnviropment::Instance().m_CompilerPath);
NormalizePath(SEnviropment::Instance().m_CachePath);
NormalizePath(SEnviropment::Instance().m_ErrorPath);
NormalizePath(SEnviropment::Instance().m_TempPath);
NormalizePath(SEnviropment::Instance().m_ShaderPath);
AZ::IO::SystemFile::CreateDir(SEnviropment::Instance().m_ErrorPath.c_str());
AZ::IO::SystemFile::CreateDir(SEnviropment::Instance().m_TempPath.c_str());
AZ::IO::SystemFile::CreateDir(SEnviropment::Instance().m_CachePath.c_str());
AZ::IO::SystemFile::CreateDir(SEnviropment::Instance().m_ShaderPath.c_str());
if (SEnviropment::Instance().m_Caching)
{
AZ::Job* loadCacheJob = AZ::CreateJobFunction(&LoadCache, autoDeleteJobWhenDone);
loadCacheJob->Start();
}
else
{
printf("\nNO CACHING, disabled by config\n");
}
}
void CCrySimpleServer::IncrementExceptionCount()
{
++ms_ExceptionCount;
}
@@ -0,0 +1,124 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLESERVER__
#define __CRYSIMPLESERVER__
#include <Core/Common.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/containers/unordered_set.h>
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/parallel/atomic.h>
#include <string>
#include <vector>
extern bool g_Success;
bool GetExecutableDirectory(AZStd::string& executableDir);
bool GetBaseDirectory(AZStd::string& baseDir);
void NormalizePath(AZStd::string& pathToNormalize);
void NormalizePath(std::string& pathToNormalize);
bool IsPathValid(const AZStd::string& path);
bool IsPathValid(const std::string& path);
namespace AZ {
class JobManager;
}
class CCrySimpleSock;
class SEnviropment
{
public:
std::string m_Root;
std::string m_CompilerPath;
std::string m_CachePath;
std::string m_TempPath;
std::string m_ErrorPath;
std::string m_ShaderPath;
std::string m_FailEMail;
std::string m_MailServer;
uint32_t m_port;
uint32_t m_MailInterval; // seconds since last error to flush error mails
bool m_Caching;
bool m_PrintErrors = 1;
bool m_PrintWarnings;
bool m_PrintCommands;
bool m_PrintListUpdates;
bool m_DedupeErrors;
bool m_DumpShaders = false;
bool m_RunAsRoot = false;
std::string m_FallbackServer;
int32_t m_FallbackTreshold;
int32_t m_MaxConnections;
std::vector<AZStd::string> m_WhitelistAddresses;
// Shader Compilers ID
static const char* m_Orbis_DXC;
static const char* m_Durango_FXC;
static const char* m_Jasper_FXC;
static const char* m_D3D11_FXC;
static const char* m_GLSL_HLSLcc;
static const char* m_METAL_HLSLcc;
static const char* m_GLSL_LLVM_DXC;
static const char* m_METAL_LLVM_DXC;
int m_hardwareTarget = -1;
static void Create();
static void Destroy();
static SEnviropment& Instance();
void InitializePlatformAttributes();
bool IsPlatformValid( const AZStd::string& platform ) const;
bool IsShaderLanguageValid( const AZStd::string& shaderLanguage ) const;
bool IsShaderCompilerValid( const AZStd::string& shaderCompilerID ) const;
bool GetShaderCompilerExecutable( const AZStd::string& shaderCompilerID, AZStd::string& shaderCompilerExecutable ) const;
private:
SEnviropment() = default;
// The single instance of the environment
static SEnviropment* m_instance;
// Platforms
AZStd::unordered_set<AZStd::string> m_Platforms;
// Shader Languages
AZStd::unordered_set<AZStd::string> m_ShaderLanguages;
// Shader Compilers ID to Executable map
AZStd::unordered_map<AZStd::string, AZStd::string> m_ShaderCompilersMap;
};
class CCrySimpleServer
{
static AZStd::atomic_long ms_ExceptionCount;
CCrySimpleSock* m_pServerSocket;
void Init();
public:
CCrySimpleServer(const char* pShaderModel, const char* pDst, const char* pSrc, const char* pEntryFunction);
CCrySimpleServer();
static long GetExceptionCount() { return ms_ExceptionCount; }
static void IncrementExceptionCount();
};
#endif
@@ -0,0 +1,761 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "CrySimpleSock.hpp"
#include <Core/StdTypes.hpp>
#include <Core/WindowsAPIImplementation.h>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
#include <AzCore/Debug/Trace.h>
#include <AzCore/std/parallel/atomic.h>
#include <algorithm>
#if defined(AZ_PLATFORM_LINUX) || defined(AZ_PLATFORM_MAC)
#include <libkern/OSAtomic.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <errno.h>
#include <unistd.h>
#else
#include <ws2tcpip.h>
typedef int socklen_t;
#endif
namespace
{
enum ECrySimpleS_TYPE
{
ECrySimpleST_ROOT,
ECrySimpleST_SERVER,
ECrySimpleST_CLIENT,
ECrySimpleST_INVALID,
};
static AZStd::atomic_long numberOfOpenSockets = {0};
const int MAX_DATA_SIZE = 1024 * 1024; // Only allow 1 MB of data to come through. Lumberyard Game Engine has the same size constraint
const size_t BLOCKSIZE = 4 * 1024;
const size_t MAX_ERROR_MESSAGE_SIZE = 1024;
const size_t MAX_HOSTNAME_BUFFER_SIZE = 1024;
struct Ip4WhitelistAddress
{
Ip4WhitelistAddress() : m_address(0), m_mask(-1) { }
// IP Address in network order to whitelist
uint32_t m_address;
// Mask in network order to apply to connecting IP addresses
uint32_t m_mask;
};
}
struct CCrySimpleSock::Implementation
{
Implementation(ECrySimpleS_TYPE type)
: m_Type(type) { }
void SetWhitelist(const std::vector<AZStd::string>& whiteList)
{
// Add in our local address so that we always allow connections from the local machine
char hostNameBuffer[MAX_HOSTNAME_BUFFER_SIZE] = { 0 };
gethostname(hostNameBuffer, MAX_HOSTNAME_BUFFER_SIZE);
struct addrinfo* addressInfos{};
struct addrinfo hints{};
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
int addressInfoResultCode = getaddrinfo(hostNameBuffer, nullptr, &hints, &addressInfos);
if (addressInfoResultCode == 0)
{
int i = 0;
for (auto addressInfoIter = addressInfos; addressInfoIter != nullptr; addressInfoIter = addressInfoIter->ai_next)
{
Ip4WhitelistAddress whitelistAddress;
whitelistAddress.m_address = static_cast<uint32_t>(reinterpret_cast<sockaddr_in*>(addressInfoIter->ai_addr)->sin_addr.s_addr);
m_ipWhiteList.push_back(whitelistAddress);
++i;
}
}
else
{
printf("Network error trying to get host computer local address. The host computer's local IP addresses will not be automatically whitelisted.");
}
for (const auto& address : whiteList)
{
Ip4WhitelistAddress whitelistAddress;
AZStd::string::size_type maskLocation = address.rfind("/");
if (maskLocation != AZStd::string::npos)
{
//x.x.x.x/0 is all addresses
// For CIDR that specify the network mask, mask out the address that is
// supplied here once instead of everytime we check the address during
// accept calls.
int mask = atoi(address.substr(maskLocation+1).c_str());
if (mask == 0)
{
whitelistAddress.m_mask = 0;
whitelistAddress.m_address = 0;
static bool warnOnce = true;
if (warnOnce)
{
warnOnce = false;
printf("\nWARNING: Attempting to run the CrySCompileServer authorizing every IP. This is a security risk and not recommended.\nPlease use a more restrictive whitelist in the config.ini file by not using netmask 0.\n\n");
}
}
else
{
whitelistAddress.m_mask ^= (1 << (32 - mask)) - 1;
whitelistAddress.m_mask = htonl(whitelistAddress.m_mask);
struct in_addr ipv4Address{};
if (inet_pton(AF_INET, address.substr(0, maskLocation).c_str(), &ipv4Address) == 1)
{
whitelistAddress.m_address = static_cast<uint32_t>(ipv4Address.s_addr);
}
}
}
else
{
struct in_addr ipv4Address{};
if (inet_pton(AF_INET, address.c_str(), &ipv4Address) == 1)
{
whitelistAddress.m_address = static_cast<uint32_t>(ipv4Address.s_addr);
}
}
m_ipWhiteList.push_back(whitelistAddress);
}
}
CCrySimpleSock* m_pInstance;
const ECrySimpleS_TYPE m_Type;
SOCKET m_Socket;
uint16_t m_Port;
#ifdef USE_WSAEVENTS
WSAEVENT m_Event;
#endif
bool m_WaitForShutdownEvent;
bool m_SwapEndian;
bool m_bHasReceivedData;
bool m_bHasSendData;
tdDataVector m_tempSendBuffer;
std::vector<Ip4WhitelistAddress> m_ipWhiteList;
};
#if defined(AZ_PLATFORM_WINDOWS)
typedef BOOL (WINAPI * LPFN_DISCONNECTEX)(SOCKET, LPOVERLAPPED, DWORD, DWORD);
#define WSAID_DISCONNECTEX {0x7fda2e11, 0x8630, 0x436f, {0xa0, 0x31, 0xf5, 0x36, 0xa6, 0xee, 0xc1, 0x57} \
}
#endif
#ifdef USE_WSAEVENTS
CCrySimpleSock::CCrySimpleSock(SOCKET Sock, CCrySimpleSock* pInstance, WSAEVENT wsaEvent)
#else
CCrySimpleSock::CCrySimpleSock(SOCKET Sock, CCrySimpleSock * pInstance)
#endif
: m_pImpl(new Implementation(ECrySimpleST_SERVER))
{
#ifdef USE_WSAEVENTS
m_pImpl->m_Event = wsaEvent;
#endif
m_pImpl->m_pInstance = pInstance;
m_pImpl->m_Socket = Sock;
m_pImpl->m_WaitForShutdownEvent = false;
m_pImpl->m_bHasReceivedData = false;
m_pImpl->m_bHasSendData = false;
m_pImpl->m_Port = ~0;
++numberOfOpenSockets;
InitClient();
}
CCrySimpleSock::CCrySimpleSock(const std::string& rServerName, uint16_t Port)
: m_pImpl(new Implementation(ECrySimpleST_CLIENT))
{
m_pImpl->m_pInstance = nullptr;
m_pImpl->m_Socket = INVALID_SOCKET;
m_pImpl->m_WaitForShutdownEvent = false;
m_pImpl->m_bHasReceivedData = false;
m_pImpl->m_bHasSendData = false;
m_pImpl->m_Port = Port;
struct sockaddr_in addr;
memset(&addr, 0, sizeof addr);
addr.sin_family = AF_INET;
addr.sin_port = htons(Port);
const char* pHostName = rServerName.c_str();
bool IP = true;
for (size_t a = 0, size = strlen(pHostName); a < size; a++)
{
IP &= (pHostName[a] >= '0' && pHostName[a] <= '9') || pHostName[a] == '.';
}
if (IP)
{
struct in_addr ipv4Address{};
if (inet_pton(AF_INET, pHostName, &ipv4Address) == 1)
{
addr.sin_addr = ipv4Address;
}
}
else
{
struct addrinfo* addressInfo{};
struct addrinfo hints{};
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
int addressInfoResultCode = getaddrinfo(pHostName, nullptr, &hints, &addressInfo);
if (addressInfoResultCode != 9)
{
return;
}
addr = *reinterpret_cast<sockaddr_in*>(addressInfo->ai_addr);
}
m_pImpl->m_Socket = socket(AF_INET, SOCK_STREAM, 0);
++numberOfOpenSockets;
int Err = connect(m_pImpl->m_Socket, (struct sockaddr*)&addr, sizeof addr);
if (Err < 0)
{
m_pImpl->m_Socket = INVALID_SOCKET;
}
}
CCrySimpleSock::~CCrySimpleSock()
{
Release();
}
CCrySimpleSock::CCrySimpleSock(uint16_t Port, const std::vector<AZStd::string>& ipWhiteList)
: m_pImpl(new Implementation(ECrySimpleST_ROOT))
{
m_pImpl->m_pInstance = nullptr;
m_pImpl->m_WaitForShutdownEvent = false;
m_pImpl->m_bHasReceivedData = false;
m_pImpl->m_bHasSendData = false;
m_pImpl->m_Port = Port;
#ifdef _MSC_VER
WSADATA Data;
m_pImpl->m_Socket = INVALID_SOCKET;
if (WSAStartup(MAKEWORD(2, 0), &Data))
{
CrySimple_ERROR("Could not init root socket");
return;
}
#endif
m_pImpl->SetWhitelist(ipWhiteList);
m_pImpl->m_Socket = socket(AF_INET, SOCK_STREAM, 0);
if (INVALID_SOCKET == m_pImpl->m_Socket)
{
CrySimple_ERROR("Could not initialize basic server due to invalid socket");
return;
}
#if defined(AZ_PLATFORM_LINUX) || defined(AZ_PLATFORM_MAC)
int arg = 1;
setsockopt(m_pImpl->m_Socket, SOL_SOCKET, SO_KEEPALIVE, &arg, sizeof arg);
arg = 1;
setsockopt(m_pImpl->m_Socket, SOL_SOCKET, SO_REUSEADDR, &arg, sizeof arg);
#endif
sockaddr_in SockAddr;
memset(&SockAddr, 0, sizeof(sockaddr_in));
SockAddr.sin_family = PF_INET;
SockAddr.sin_port = htons(Port);
if (bind(m_pImpl->m_Socket, (sockaddr*)&SockAddr, sizeof(sockaddr_in)) == SOCKET_ERROR)
{
#if defined(AZ_PLATFORM_WINDOWS)
AZ_Warning(0, false, "bind failed with error = %d", WSAGetLastError());
#else
shutdown(m_pImpl->m_Socket, SHUT_RDWR);
#endif
closesocket(m_pImpl->m_Socket);
CrySimple_ERROR("Could not bind server socket. This can happen if there is another process running already that is using this port or antivirus software/firewall is blocking the port.\n");
return;
}
++numberOfOpenSockets;
}
void CCrySimpleSock::Listen()
{
listen(m_pImpl->m_Socket, SOMAXCONN);
}
void CCrySimpleSock::InitClient()
{
}
void CCrySimpleSock::Release()
{
if (m_pImpl->m_Socket != INVALID_SOCKET)
{
// check if we have received and sended data but ignore that for the HTTP server
if ((!m_pImpl->m_bHasSendData || !m_pImpl->m_bHasReceivedData) && (!m_pImpl->m_pInstance || m_pImpl->m_pInstance->m_pImpl->m_Port != 80))
{
char acTmp[MAX_ERROR_MESSAGE_SIZE];
azsprintf(acTmp, "ERROR : closing socket without both receiving and sending data: receive: %d send: %d",
m_pImpl->m_bHasReceivedData, m_pImpl->m_bHasSendData);
CRYSIMPLE_LOG(acTmp);
}
#ifdef USE_WSAEVENTS
if (m_pImpl->m_WaitForShutdownEvent)
{
// wait until client has shutdown its socket
DWORD nReturnCode = WSAWaitForMultipleEvents(1, &m_pImpl->m_Event,
FALSE, INFINITE, FALSE);
if ((nReturnCode != WSA_WAIT_FAILED) && (nReturnCode != WSA_WAIT_TIMEOUT))
{
WSANETWORKEVENTS NetworkEvents;
WSAEnumNetworkEvents(m_pImpl->m_Socket, m_pImpl->m_Event, &NetworkEvents);
if (NetworkEvents.lNetworkEvents & FD_CLOSE)
{
int iErrorCode = NetworkEvents.iErrorCode[FD_CLOSE_BIT];
if (iErrorCode != 0)
{
// error shutting down
}
}
}
}
// shutdown the server side of the connection since no more data will be sent
shutdown(m_pImpl->m_Socket, SHUT_RDWR);
closesocket(m_pImpl->m_Socket);
#endif
#if defined(AZ_PLATFORM_WINDOWS)
LPFN_DISCONNECTEX pDisconnectEx = NULL;
DWORD Bytes;
GUID guidDisconnectEx = WSAID_DISCONNECTEX;
WSAIoctl(m_pImpl->m_Socket, SIO_GET_EXTENSION_FUNCTION_POINTER, &guidDisconnectEx,
sizeof(GUID), &pDisconnectEx, sizeof(pDisconnectEx), &Bytes, NULL, NULL);
pDisconnectEx(m_pImpl->m_Socket, NULL, 0, 0); // retrieve this function pointer with WSAIoctl(WSAID_DISCONNECTEX).
#else
shutdown(m_pImpl->m_Socket, SHUT_RDWR);
#endif
closesocket(m_pImpl->m_Socket);
m_pImpl->m_Socket = INVALID_SOCKET;
--numberOfOpenSockets;
}
#if defined(AZ_PLATFORM_WINDOWS)
switch (m_pImpl->m_Type)
{
case ECrySimpleST_ROOT:
WSACleanup();
break;
case ECrySimpleST_SERVER: // Intentionally fall through
case ECrySimpleST_CLIENT:
break;
default:
CrySimple_ERROR("unknown SocketType Released");
}
#endif
}
CCrySimpleSock* CCrySimpleSock::Accept()
{
if (m_pImpl->m_Type != ECrySimpleST_ROOT)
{
CrySimple_ERROR("called Accept on non root socket");
return nullptr;
}
while (true)
{
sockaddr_in connectingAddress;
int addressSize = sizeof(connectingAddress);
SOCKET Sock = accept(m_pImpl->m_Socket, reinterpret_cast<sockaddr*>(&connectingAddress), reinterpret_cast<socklen_t*>(&addressSize));
if (Sock == INVALID_SOCKET)
{
#if defined(AZ_PLATFORM_MAC)
switch (errno)
{
case EINTR:
// OS X tends to get interupt calls on every other accept call
// so just ignore this particular error and try the accept call
// again.
continue;
default:
// Do nothing - all other errors are "real" and we should exit
break;
}
#endif
AZ_Warning(0, false, "Errno = %d", WSAGetLastError());
CrySimple_ERROR("Accept recived invalid socket");
return nullptr;
}
bool allowConnection = false;
for (const auto& ip4WhitelistAddress : m_pImpl->m_ipWhiteList)
{
if ((connectingAddress.sin_addr.s_addr & ip4WhitelistAddress.m_mask) == (ip4WhitelistAddress.m_address))
{
allowConnection = true;
break;
}
}
if (!allowConnection)
{
constexpr size_t ipAddressBufferSize = 17;
char ipAddressBuffer[ipAddressBufferSize]{};
inet_ntop(AF_INET, &connectingAddress.sin_addr, ipAddressBuffer, ipAddressBufferSize);
printf("Warning: unauthorized IP %s trying to connect. If this IP is authorized please add it to the whitelist in the config.ini file\n", ipAddressBuffer);
closesocket(Sock);
continue;
}
int arg = 1;
setsockopt(Sock, SOL_SOCKET, SO_REUSEADDR, (char*)&arg, sizeof arg);
/*
// keep socket open for another 2 seconds until data has been fully send
LINGER linger;
int len = sizeof(LINGER);
linger.l_onoff = 1;
linger.l_linger = 2;
setsockopt(Sock, SOL_SOCKET, SO_LINGER, (char*)&linger, sizeof linger);
*/
#ifdef USE_WSAEVENTS
WSAEVENT wsaEvent = WSACreateEvent();
if (wsaEvent == WSA_INVALID_EVENT)
{
closesocket(Sock);
int Error = WSAGetLastError();
CrySimple_ERROR("Couldn't create wsa event");
return nullptr;
}
int Status = WSAEventSelect(Sock, wsaEvent, FD_CLOSE);
if (Status == SOCKET_ERROR)
{
closesocket(Sock);
int Error = WSAGetLastError();
CrySimple_ERROR("Couldn't create wsa event");
return nullptr;
}
return new CCrySimpleSock(Sock, this, wsaEvent);
#else
return new CCrySimpleSock(Sock, this);
#endif
}
return nullptr;
}
union CrySimpleRecvSize
{
uint8_t m_Data8[8];
uint64_t m_Data64;
};
static const int MAX_TIME_TO_WAIT = 10000;
int CCrySimpleSock::Recv(char* acData, int len, int flags)
{
int recived = SOCKET_ERROR;
int waitingtime = 0;
while (recived < 0)
{
recived = recv(m_pImpl->m_Socket, acData, len, flags);
if (recived == SOCKET_ERROR)
{
int WSAError = WSAGetLastError();
#if defined(AZ_PLATFORM_WINDOWS)
if (WSAError == WSAEWOULDBLOCK)
{
// are we out of time
if (waitingtime > MAX_TIME_TO_WAIT)
{
char acTmp[MAX_ERROR_MESSAGE_SIZE];
azsprintf(acTmp, "Error while receiving size of data - Timeout on blocking. (Error Code: %i)", WSAError);
CrySimple_ERROR(acTmp);
return recived;
}
waitingtime += 5;
// sleep a bit and try again
Sleep(5);
}
else
#endif
{
char acTmp[MAX_ERROR_MESSAGE_SIZE];
azsprintf(acTmp, "Error while receiving size of data - Network error. (Error Code: %i)", WSAError);
CrySimple_ERROR(acTmp);
return recived;
}
}
}
return recived;
}
bool CCrySimpleSock::Recv(std::vector<uint8_t>& rVec)
{
CrySimpleRecvSize size;
int received = Recv(reinterpret_cast<char*>(&size.m_Data8[0]), 8, 0);
if (received != 8)
{
#if defined(AZ_PLATFORM_WINDOWS)
int WSAError = WSAGetLastError();
#else
int WSAError = errno;
#endif
char acTmp[MAX_ERROR_MESSAGE_SIZE];
azsprintf(acTmp, "Error while receiving size of data - Invalid size (Error Code: %i)", WSAError);
CrySimple_ERROR(acTmp);
return false;
}
if (size.m_Data64 == 0)
{
int WSAError = WSAGetLastError();
char acTmp[MAX_ERROR_MESSAGE_SIZE];
azsprintf(acTmp, "Error while receiving size of data - Size of zero (Error Code: %i)", WSAError);
CrySimple_ERROR(acTmp);
return false;
}
if (size.m_Data64 > MAX_DATA_SIZE)
{
int WSAError = WSAGetLastError();
char acTmp[MAX_ERROR_MESSAGE_SIZE];
azsprintf(acTmp, "Error while receiving size of data - Size is greater than max support data size.");
CrySimple_ERROR(acTmp);
return false;
}
m_pImpl->m_SwapEndian = (size.m_Data64 >> 32) != 0;
if (m_pImpl->m_SwapEndian)
{
CSTLHelper::EndianSwizzleU64(size.m_Data64);
}
rVec.clear();
rVec.resize(static_cast<size_t>(size.m_Data64));
for (uint32_t a = 0; a < size.m_Data64; )
{
int read = Recv(reinterpret_cast<char*>(&rVec[a]), static_cast<int>(size.m_Data64) - a, 0);
if (read <= 0)
{
int WSAError = WSAGetLastError();
char acTmp[MAX_ERROR_MESSAGE_SIZE];
azsprintf(acTmp, "Error while receiving tcp-data (size: %d - Error Code: %i)", static_cast<int>(size.m_Data64), WSAError);
CrySimple_ERROR(acTmp);
return false;
}
a += read;
}
m_pImpl->m_bHasReceivedData = true;
return true;
}
bool CCrySimpleSock::RecvResult()
{
CrySimpleRecvSize size;
if (recv(m_pImpl->m_Socket, reinterpret_cast<char*>(&size.m_Data8[0]), 8, 0) != 8)
{
CrySimple_ERROR("Error while receiving result");
return false;
}
return size.m_Data64 > 0;
}
void CCrySimpleSock::Forward(const std::vector<uint8_t>& rVecIn)
{
tdDataVector& rVec = m_pImpl->m_tempSendBuffer;
rVec.resize(rVecIn.size() + 8);
CrySimpleRecvSize& rHeader = *(CrySimpleRecvSize*)(&rVec[0]);
rHeader.m_Data64 = (uint32_t)rVecIn.size();
memcpy(&rVec[8], &rVecIn[0], rVecIn.size());
CrySimpleRecvSize size;
size.m_Data64 = static_cast<int>(rVec.size());
for (uint64_t a = 0; a < size.m_Data64; a += BLOCKSIZE)
{
char* pData = reinterpret_cast<char*>(&rVec[(size_t)a]);
int nSendRes = send(m_pImpl->m_Socket, pData, std::min<int>(static_cast<int>(size.m_Data64 - a), BLOCKSIZE), 0);
if (nSendRes == SOCKET_ERROR)
{
int nLastSendError = WSAGetLastError();
logmessage("Socket send(forward) error: %d", nLastSendError);
}
}
}
bool CCrySimpleSock::Backward(std::vector<uint8_t>& rVec)
{
uint32_t size;
if (recv(m_pImpl->m_Socket, reinterpret_cast<char*>(&size), 4, 0) != 4)
{
CrySimple_ERROR("Error while receiving size of data");
return false;
}
rVec.clear();
rVec.resize(static_cast<size_t>(size));
for (uint32_t a = 0; a < size; )
{
int read = recv(m_pImpl->m_Socket, reinterpret_cast<char*>(&rVec[a]), size - a, 0);
if (read <= 0)
{
CrySimple_ERROR("Error while receiving tcp-data");
return false;
}
a += read;
}
return true;
}
void CCrySimpleSock::Send(const std::vector<uint8_t>& rVecIn, size_t state, EProtocolVersion version)
{
const size_t offset = version == EPV_V001 ? 4 : 5;
tdDataVector& rVec = m_pImpl->m_tempSendBuffer;
rVec.resize(rVecIn.size() + offset);
if (rVecIn.size())
{
*(uint32_t*)(&rVec[0]) = (uint32_t)rVecIn.size();
memcpy(&rVec[offset], &rVecIn[0], rVecIn.size());
}
if (version >= EPV_V002)
{
rVec[4] = static_cast<uint8_t>(state);
}
if (m_pImpl->m_SwapEndian)
{
CSTLHelper::EndianSwizzleU32(*(uint32_t*)&rVec[0]);
}
// send can fail, you must retry unsent parts.
size_t remainingBytes = rVec.size();
const char* pData = reinterpret_cast<char*>(rVec.data());
while (remainingBytes != 0)
{
size_t sendThisRound = remainingBytes;
if (sendThisRound > BLOCKSIZE)
{
sendThisRound = BLOCKSIZE;
}
int bytesActuallySent = send(m_pImpl->m_Socket, pData, static_cast<int>(sendThisRound), 0);
if (bytesActuallySent < 0)
{
int nLastSendError = WSAGetLastError();
logmessage("Socket send error: %d", nLastSendError);
m_pImpl->m_bHasSendData = true;
return;
}
size_t actuallySent = static_cast<size_t>(bytesActuallySent);
remainingBytes -= actuallySent;
pData += actuallySent;
}
m_pImpl->m_bHasSendData = true;
}
void CCrySimpleSock::Send(const std::string& rData)
{
const size_t S = rData.size();
for (uint64_t a = 0; a < S; a += BLOCKSIZE)
{
const char* pData = &rData.c_str()[a];
const int nSendRes = send(m_pImpl->m_Socket, pData, std::min<int>(static_cast<int>(S - a), BLOCKSIZE), 0);
if (nSendRes == SOCKET_ERROR)
{
int nLastSendError = WSAGetLastError();
logmessage("Socket send error: %d", nLastSendError);
}
else
{
m_pImpl->m_bHasSendData = true;
}
}
}
uint32_t CCrySimpleSock::PeerIP()
{
struct sockaddr_in addr;
#if defined(AZ_PLATFORM_WINDOWS)
int addr_size = sizeof(sockaddr_in);
#else
socklen_t addr_size = sizeof(sockaddr_in);
#endif
int nRes = getpeername(m_pImpl->m_Socket, (sockaddr*) &addr, &addr_size);
if (nRes == SOCKET_ERROR)
{
int nError = WSAGetLastError();
logmessage("Socket getpeername error: %d", nError);
return 0;
}
#if defined(AZ_PLATFORM_WINDOWS)
return addr.sin_addr.S_un.S_addr;
#else
return addr.sin_addr.s_addr;
#endif
}
bool CCrySimpleSock::Valid() const
{
return m_pImpl->m_Socket != INVALID_SOCKET;
}
void CCrySimpleSock::WaitForShutDownEvent(bool bValue)
{
m_pImpl->m_WaitForShutdownEvent = bValue;
}
long CCrySimpleSock::GetOpenSockets()
{
return numberOfOpenSockets;
}
@@ -0,0 +1,104 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __CRYSIMPLESOCK__
#define __CRYSIMPLESOCK__
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/string/string.h>
#include <Core/Common.h>
#include <Core/STLHelper.hpp>
#if defined(AZ_PLATFORM_LINUX) || defined(AZ_PLATFORM_MAC)
typedef int SOCKET;
#define INVALID_SOCKET (-1)
#define SOCKET_ERROR (-1)
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/ip.h>
#include <errno.h>
#define closesocket close
#else
#ifndef _WINSOCK_DEPRECATED_NO_WARNINGS
#define _WINSOCK_DEPRECATED_NO_WARNINGS // till we swtich to in inet_pton and getaddrinfo
#endif
#include <WinSock2.h>
#endif
#include <vector>
#include <memory>
//#define USE_WSAEVENTS
enum EProtocolVersion
{
EPV_V001,
EPV_V002,
EPV_V0021,
EPV_V0022,
EPV_V0023,
};
class CCrySimpleSock
{
public:
#ifdef USE_WSAEVENTS
CCrySimpleSock(SOCKET Sock, CCrySimpleSock* pInstance, WSAEVENT wsaEvent);
#else
CCrySimpleSock(SOCKET Sock, CCrySimpleSock* pInstance);
#endif
CCrySimpleSock(const CCrySimpleSock&);
CCrySimpleSock(const std::string& rServerName, uint16_t Port);
CCrySimpleSock(uint16_t Port, const std::vector<AZStd::string>& ipWhiteList);
~CCrySimpleSock();
void InitClient();
void Release();
int Recv(char* acData, int len, int flags);
void Listen();
CCrySimpleSock* Accept();
bool Recv(std::vector<uint8_t>& rVec);
bool RecvResult();
bool Backward(std::vector<uint8_t>& rVec);
void Send(const std::vector<uint8_t>& rVec, size_t State, EProtocolVersion Version);
void Forward(const std::vector<uint8_t>& rVec);
//used for HTML
void Send(const std::string& rData);
uint32_t PeerIP();
bool Valid() const;
void WaitForShutDownEvent(bool bValue);
static long GetOpenSockets();
private:
struct Implementation;
std::unique_ptr<Implementation> m_pImpl;
};
#endif
@@ -0,0 +1,521 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "ShaderList.hpp"
#include <AzCore/base.h>
#include <AzCore/PlatformDef.h>
#include "CrySimpleServer.hpp"
#include <Core/WindowsAPIImplementation.h>
#include <assert.h>
#ifdef _MSC_VER
#include <process.h>
#include <direct.h>
#endif
#if defined(AZ_PLATFORM_LINUX) || defined(AZ_PLATFORM_MAC)
#include <pthread.h>
#include <unistd.h>
#include <sys/stat.h>
#include <time.h>
#endif
static bool g_bSaveThread = false;
CShaderList& CShaderList::Instance()
{
static CShaderList g_Cache;
return g_Cache;
}
//////////////////////////////////////////////////////////////////////////
CShaderList::CShaderList()
{
m_lastTime = 0;
}
//////////////////////////////////////////////////////////////////////////
void CShaderList::Tick()
{
#if defined(AZ_PLATFORM_WINDOWS)
DWORD t = GetTickCount();
#else
unsigned long t = time(nullptr)*1000; // Current time in milliseconds
#endif
if (t < m_lastTime || (t - m_lastTime) > 1000) //check every second
{
m_lastTime = t;
Save();
}
}
//////////////////////////////////////////////////////////////////////////
void CShaderList::Add(const std::string& rShaderListName, const char* pLine)
{
tdShaderLists::iterator it;
{
CCrySimpleMutexAutoLock Lock(m_Mutex);
it = m_ShaderLists.find(rShaderListName);
//not existing yet?
if (it == m_ShaderLists.end())
{
CCrySimpleMutexAutoLock Lock2(m_Mutex2); //load/save mutex
m_ShaderLists[rShaderListName] = new CShaderListFile(rShaderListName);
it = m_ShaderLists.find(rShaderListName);
it->second->Load((SEnviropment::Instance().m_CachePath + rShaderListName).c_str());
}
}
it->second->InsertLine(pLine);
}
//////////////////////////////////////////////////////////////////////////
void CShaderList::Save()
{
CCrySimpleMutexAutoLock Lock(m_Mutex2); //load/save mutex
for (tdShaderLists::iterator it = m_ShaderLists.begin(); it != m_ShaderLists.end(); ++it)
{
it->second->MergeNewLinesAndSave();
}
}
//////////////////////////////////////////////////////////////////////////
CShaderListFile::CShaderListFile(std::string ListName)
{
m_bModified = false;
m_listname = ListName;
// some test cases
SMetaData MD;
assert(CheckSyntax("<1>watervolume@WaterVolumeOutofPS()()(0)(0)(0)(ps_2_0)", MD) == true);
assert(CheckSyntax("<1>Blurcloak@BlurCloakPS(%BUMP_MAP)(%_RT_FOG|%_RT_HDR_MODE|%_RT_BUMP)(0)(0)(1)(ps_2_0)", MD) == true);
assert(CheckSyntax("<1>Burninglayer@BurnPS()(%_RT_ADDBLEND|%_RT_)HDR_MODE|%_RT_BUMP|%_RT_3DC)(0)(0)(0)(ps_2_0)", MD) == false);
assert(CheckSyntax("<1>Illum@IlluminationVS(%DIFFUSE|%SPECULAR|%BUMP_MAP|%VERTCOLORS|%STAT_BRANCHING)(%_RT_RAE_GEOMTERM)(101)(0)(0)(vs_2_0)", MD) == true);
assert(CheckSyntax("<660><2>Cloth@Common_SG_VS()(%_RT_QUALITY|%_RT_SHAPEDEFORM|%_RT_SKELETON_SSD|%_RT_HW_PCF_COMPARE)(0)(0)(0)(VS)", MD) == true);
assert(CheckSyntax("<6452><2>ShadowMaskGen@FrustumClipVolumeVS()()(0)(0)(0)(VS)", MD) == true);
assert(CheckSyntax("<5604><2>ParticlesNoMat@ParticlePS()(%_RT_FOG|%_RT_AMBIENT|%_RT_ALPHABLEND|%_RT_QUALITY1)(0)(0)(0)(PS)", MD) == true);
}
//////////////////////////////////////////////////////////////////////////
bool CShaderListFile::Reload()
{
return Load(m_filename.c_str());
}
void CShaderListFile::CreatePath(const std::string& rPath)
{
std::string Path = rPath;
CSTLHelper::Replace(Path, rPath, "\\", "/");
tdEntryVec rToks;
CSTLHelper::Tokenize(rToks, Path, "/");
Path = "";
for (size_t a = 0; a + 1 < rToks.size(); a++)
{
Path += rToks[a] + "/";
#if defined(AZ_PLATFORM_WINDOWS)
_mkdir(Path.c_str());
#else
mkdir(Path.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
}
//////////////////////////////////////////////////////////////////////////
bool CShaderListFile::Load(const char* filename)
{
CreatePath(filename);
printf("Loading ShaderList file: %s\n", filename);
m_filename = filename;
m_filenametmp = filename;
m_filenametmp += ".tmp";
FILE* f = nullptr;
azfopen(&f, filename, "rt");
if (!f)
{
return false;
}
int nNumLines = 0;
m_entries.clear();
char str[65535];
while (fgets(str, sizeof(str), f) != NULL)
{
if (*str && InsertLineInternal(str))
{
++nNumLines;
}
}
fclose(f);
if (nNumLines == m_entries.size())
{
m_bModified = false;
}
else
{
m_bModified = true;
}
printf("Loaded %d combination for %s\n", nNumLines, filename);
return true;
}
//////////////////////////////////////////////////////////////////////////
bool CShaderListFile::Save()
{
//not needed regarding timur, m_entries is just accessed by one thread
//CCrySimpleMutexAutoLock Lock(m_Mutex);
CreatePath(m_filename);
if (m_filename.empty())
{
return false;
}
// write to tmp file
FILE* f = nullptr;
azfopen(&f, m_filenametmp.c_str(), "wt");
if (!f)
{
return false;
}
for (Entries::iterator it = m_entries.begin(); it != m_entries.end(); ++it)
{
const char* str = it->first.c_str();
if (it->second.m_Count == -1)
{
fprintf(f, "<%d>%s\n", it->second.m_Version, str);
}
else
{
fprintf(f, "<%d><%d>%s\n", it->second.m_Count, it->second.m_Version, str);
}
}
fclose(f);
// first check if original file excists
f = nullptr;
azfopen(&f, m_filename.c_str(), "rt");
if (f)
{
fclose(f);
// remove original file (keep on trying until success - shadercompiler could currently be copying it for example)
int sleeptime = 0;
while (remove(m_filename.c_str()))
{
Sleep(100);
sleeptime += 100;
if (sleeptime > 5000)
{
break;
}
}
}
{
int sleeptime = 0;
while (rename(m_filenametmp.c_str(), m_filename.c_str()))
{
Sleep(100);
sleeptime += 100;
if (sleeptime > 5000)
{
break;
}
}
}
m_bModified = false;
return true;
}
//////////////////////////////////////////////////////////////////////////
inline bool IsHexNumberCharacter(const char c)
{
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
}
//////////////////////////////////////////////////////////////////////////
inline bool IsDecNumberCharacter(const char c)
{
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
}
//////////////////////////////////////////////////////////////////////////
inline bool IsNameCharacter(const char c)
{
return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || c == '@' || c == '/' || c == '%' || c == '_';
}
int shGetHex(const char* buf)
{
if (!buf)
{
return 0;
}
int i = 0;
azsscanf(buf, "%x", &i);
return i;
}
//////////////////////////////////////////////////////////////////////////
bool CShaderListFile::CheckSyntax(const char* szLine, SMetaData& rMD, const char** sOutStr)
{
assert(szLine);
if (!szLine)
{
return false;
}
if (sOutStr)
{
*sOutStr = 0;
}
// e.g. Blurcloak@BlurCloakPS(%BUMP_MAP|%SPECULAR)(%_RT_FOG|%_RT_HDR_MODE|%_RT_BUMP)(0)(0)(0)(ps_2_0)
const char* p = szLine;
if (strlen(szLine) < 4)
{
return false;
}
int Value0 = 0;
int Value1 = 0;
if (*p != '<')
{
return false;
}
char Last = 0;
while (IsDecNumberCharacter(Last = *++p))
{
Value0 = Value0 * 10 + (Last - '0');
}
if (*p++ != '>')
{
return false;
}
if (*p == '<')
{
while (IsDecNumberCharacter(Last = *++p))
{
Value1 = Value1 * 10 + (Last - '0');
}
if (*p++ != '>')
{
return false;
}
rMD.m_Version = Value1;
rMD.m_Count = Value0;
}
else
{
rMD.m_Version = Value0;
rMD.m_Count = -1;
}
const char* pStart = p;
// e.g. "Blurcloak@BlurCloakPS"
while (IsNameCharacter(*p++))
{
;
}
p--;
// e.g. "(%BUMP_MAP|%SPECULAR)(%_RT_FOG|%_RT_HDR_MODE|%_RT_BUMP)"
for (int i = 0; i < 2; ++i)
{
if (*p++ != '(')
{
return false;
}
while (true)
{
while (IsNameCharacter(*p++))
{
;
}
p--;
if (*p != '|')
{
break;
}
p++;
}
if (*p++ != ')')
{
return false;
}
}
// e.g. "(0)(0)(0)"
for (int i = 0; i < 3; ++i)
{
if (*p++ != '(')
{
return false;
}
while (IsHexNumberCharacter(*p++))
{
;
}
p--;
if (*p++ != ')')
{
return false;
}
}
// e.g. "(ps_2_0)"
if (*p++ != '(')
{
return false;
}
while (IsNameCharacter(*p++))
{
;
}
p--;
if (*p++ != ')')
{
return false;
}
// Copy rest of the line.
if (sOutStr)
{
*sOutStr = pStart;
}
return true;
}
//////////////////////////////////////////////////////////////////////////
void CShaderListFile::InsertLine(const char* szLine)
{
if (*szLine != 0)
{
CCrySimpleMutexAutoLock Lock(m_Mutex);
m_newLines.push_back(szLine);
m_bModified = true;
}
}
//////////////////////////////////////////////////////////////////////////
bool CShaderListFile::InsertLineInternal(const char* szLine)
{
const char* szCorrectedLine = 0;
SMetaData MD;
if (CheckSyntax(szLine, MD, &szCorrectedLine))
{
// Trim \n\r
char* s = const_cast<char*>(szCorrectedLine);
for (size_t p = strlen(s) - 1; p > 0; p--)
{
if (s[p] == '\n' || s[p] == '\r')
{
s[p] = '\0';
}
else
{
break;
}
}
if (szCorrectedLine)
{
Entries::iterator it = m_entries.find(szCorrectedLine);
if (it == m_entries.end())
{
m_entries[szCorrectedLine] = MD;
m_bModified = true;
}
else
{
if (it->second.m_Version < MD.m_Version)
{
it->second = MD;
m_bModified = true;
}
else
if (it->second.m_Count < MD.m_Count)
{
it->second.m_Count = MD.m_Count;
m_bModified = true;
}
}
}
return true;
}
return false;
}
//////////////////////////////////////////////////////////////////////////
void CShaderListFile::MergeNewLines()
{
std::vector<std::string> newLines;
{
CCrySimpleMutexAutoLock Lock(m_Mutex);
newLines.swap(m_newLines);
}
m_bModified = false;
if (newLines.empty())
{
return;
}
for (std::vector<std::string>::iterator it = newLines.begin(); it != newLines.end(); ++it)
{
InsertLineInternal((*it).c_str());
}
}
//////////////////////////////////////////////////////////////////////////
void CShaderListFile::MergeNewLinesAndSave()
{
if (m_bModified)
{
MergeNewLines();
}
if (m_bModified)
{
if (SEnviropment::Instance().m_PrintListUpdates)
{
logmessage("Updating: %s\n", m_listname.c_str());
}
Save();
}
}
@@ -0,0 +1,95 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __SHADERLIST__
#define __SHADERLIST__
#include <map>
#include <set>
#include <vector>
#include "CrySimpleMutex.hpp"
#include <Core/StdTypes.hpp>
#include <Core/Error.hpp>
#include <Core/STLHelper.hpp>
class CShaderListFile
{
//////////////////////////////////////////////////////////////////////////
struct SMetaData
{
SMetaData()
: m_Version(0)
, m_Count(-1)
{}
int32_t m_Version;
int32_t m_Count;
};
bool m_bModified;
std::string m_listname;
std::string m_filename;
std::string m_filenametmp;
typedef std::map<std::string, SMetaData> Entries;
Entries m_entries;
std::vector<std::string> m_newLines;
CCrySimpleMutex m_Mutex;
//do not copy -> not safe
CShaderListFile(const CShaderListFile&);
CShaderListFile& operator=(const CShaderListFile&);
public:
CShaderListFile(std::string ListName);
bool Load(const char* filename);
bool Save();
bool Reload();
bool IsModified() const { return m_bModified; }
void InsertLine(const char* szLine);
void MergeNewLinesAndSave();
private:
void CreatePath(const std::string& rPath);
void MergeNewLines();
// Returns:
// true - line was instered, false otherwise
bool InsertLineInternal(const char* szLine);
// Returns
// true=syntax is ok, false=syntax is wrong
static bool CheckSyntax(const char* szLine, SMetaData& rMD, const char** sOutStr = NULL);
};
typedef std::map<std::string, CShaderListFile*> tdShaderLists;
class CShaderList
{
CCrySimpleMutex m_Mutex;
CCrySimpleMutex m_Mutex2;
unsigned long m_lastTime;
tdShaderLists m_ShaderLists;
void Save();
public:
static CShaderList& Instance();
CShaderList();
void Add(const std::string& rShaderListName, const char* pLine);
void Tick();
};
#endif
@@ -0,0 +1,41 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#ifndef __STDTYPES_DUMMY__
#define __STDTYPES_DUMMY__
#include <AzCore/PlatformDef.h>
#if defined(AZ_PLATFORM_WINDOWS)
typedef signed __int8 int8_t;
typedef signed __int16 int16_t;
typedef signed __int32 int32_t;
typedef signed __int64 int64_t;
typedef unsigned __int8 uint8_t;
typedef unsigned __int16 uint16_t;
typedef unsigned __int32 uint32_t;
typedef unsigned __int64 uint64_t;
#endif
#if defined(UNIX)
#include <stdint.h>
#include "Core/UnixCompat.h"
#endif
#if defined(AZ_PLATFORM_MAC)
#include <stdint.h>
#endif
#endif
@@ -0,0 +1,80 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include "WindowsAPIImplementation.h"
#if defined(AZ_PLATFORM_MAC) || defined(AZ_PLATFORM_LINUX)
#include <cstring>
#include <errno.h>
#import <mach/mach_time.h>
#include <libkern/OSAtomic.h>
bool QueryPerformanceCounter(LARGE_INTEGER* counter)
{
#if defined(LINUX)
// replaced gettimeofday
// http://fixunix.com/kernel/378888-gettimeofday-resolution-linux.html
timespec tv;
clock_gettime(CLOCK_MONOTONIC, &tv);
counter->QuadPart = (uint64_t)tv.tv_sec * 1000000 + tv.tv_nsec / 1000;
#elif defined(APPLE)
counter->QuadPart = mach_absolute_time();
#endif
return true;
}
bool QueryPerformanceFrequency(LARGE_INTEGER* frequency)
{
#if defined(LINUX)
// On Linux we'll use gettimeofday(). The API resolution is microseconds,
// so we'll report that to the caller.
frequency->u.LowPart = 1000000;
frequency->u.HighPart = 0;
#elif defined(APPLE)
static mach_timebase_info_data_t s_kTimeBaseInfoData;
if (s_kTimeBaseInfoData.denom == 0)
{
mach_timebase_info(&s_kTimeBaseInfoData);
}
// mach_timebase_info_data_t expresses the tick period in nanoseconds
frequency->QuadPart = 1e+9 * (uint64_t)s_kTimeBaseInfoData.denom / (uint64_t)s_kTimeBaseInfoData.numer;
#endif
return true;
}
int WSAGetLastError()
{
return errno;
}
DWORD Sleep(DWORD dwMilliseconds)
{
timespec req;
timespec rem;
memset(&req, 0, sizeof(req));
memset(&rem, 0, sizeof(rem));
time_t sec = (int)(dwMilliseconds / 1000);
req.tv_sec = sec;
req.tv_nsec = (dwMilliseconds - (sec * 1000)) * 1000000L;
if (nanosleep(&req, &rem) == -1)
{
nanosleep(&rem, 0);
}
return 0;
}
#endif
@@ -0,0 +1,78 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/base.h>
#include <AzCore/PlatformDef.h>
#if defined(AZ_PLATFORM_MAC) || defined(AZ_PLATFORM_LINUX)
#include <pthread.h>
#ifndef MAX_PATH
#define MAX_PATH PATH_MAX
#endif
typedef uint32_t DWORD;
typedef union _LARGE_INTEGER
{
struct
{
uint32_t LowPart;
int32_t HighPart;
};
struct
{
uint32_t LowPart;
int32_t HighPart;
} u;
int64_t QuadPart;
} LARGE_INTEGER;
bool QueryPerformanceCounter(LARGE_INTEGER* counter);
bool QueryPerformanceFrequency(LARGE_INTEGER* frequency);
int WSAGetLastError();
DWORD Sleep(DWORD dwMilliseconds);
#if defined(AZ_PLATFORM_LINUX)
namespace PthreadImplementation
{
static pthread_mutex_t g_interlockMutex;
}
template<typename T>
const volatile T InterlockedIncrement(volatile T* pT)
{
pthread_mutex_lock(&PthreadImplementation::g_interlockMutex);
++(*pT);
pthread_mutex_unlock(&PthreadImplementation::g_interlockMutex);
return *pT;
}
template<typename T>
const volatile T InterlockedDecrement(volatile T* pT)
{
pthread_mutex_lock(&PthreadImplementation::g_interlockMutex);
--(*pT);
pthread_mutex_unlock(&PthreadImplementation::g_interlockMutex);
return *pT;
}
#endif
#endif
@@ -0,0 +1,427 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
// Original file Copyright Crytek GMBH or its affiliates, used under license.
#include "Core/StdTypes.hpp"
#include "Core/Server/CrySimpleServer.hpp"
#include "Core/Server/CrySimpleHTTP.hpp"
#include <AzFramework/StringFunc/StringFunc.h>
#include <AzCore/PlatformDef.h>
#include <AzCore/base.h>
#include <AzCore/Jobs/JobContext.h>
#include <AzCore/Jobs/JobManager.h>
#include <AzCore/Memory/PoolAllocator.h>
#include <AzCore/Memory/SystemAllocator.h>
#include <AzCore/IO/Path/Path.h>
#include <AzCore/IO/SystemFile.h>
#include <AzCore/Utils/Utils.h>
#include <iostream>
#include <string>
#include <regex>
#if AZ_TRAIT_OS_PLATFORM_APPLE
// Needed for geteuid()
#include <sys/types.h>
#include <unistd.h>
#endif
namespace
{
const int STD_TCP_PORT = 61453;
const int DEFAULT_MAX_CONNECTIONS = 255;
}
//////////////////////////////////////////////////////////////////////////
class CConfigFile
{
public:
CConfigFile() {}
//////////////////////////////////////////////////////////////////////////
void OnLoadConfigurationEntry(const std::string& strKey, const std::string& strValue, [[maybe_unused]] const std::string& strGroup)
{
if (azstricmp(strKey.c_str(), "MailError") == 0)
{
SEnviropment::Instance().m_FailEMail = strValue;
}
if (azstricmp(strKey.c_str(), "port") == 0)
{
SEnviropment::Instance().m_port = atoi(strValue.c_str());
}
if (azstricmp(strKey.c_str(), "MailInterval") == 0)
{
SEnviropment::Instance().m_MailInterval = atoi(strValue.c_str());
}
if (azstricmp(strKey.c_str(), "TempDir") == 0)
{
SEnviropment::Instance().m_TempPath = AddSlash(strValue);
}
if (azstricmp(strKey.c_str(), "MailServer") == 0)
{
SEnviropment::Instance().m_MailServer = strValue;
}
if (azstricmp(strKey.c_str(), "Caching") == 0)
{
SEnviropment::Instance().m_Caching = atoi(strValue.c_str()) != 0;
}
if (azstricmp(strKey.c_str(), "PrintErrors") == 0)
{
SEnviropment::Instance().m_PrintErrors = atoi(strValue.c_str()) != 0;
}
if (azstricmp(strKey.c_str(), "PrintWarnings") == 0)
{
SEnviropment::Instance().m_PrintWarnings = atoi(strValue.c_str()) != 0;
}
if (azstricmp(strKey.c_str(), "PrintCommands") == 0)
{
SEnviropment::Instance().m_PrintCommands = atoi(strValue.c_str()) != 0;
}
if (azstricmp(strKey.c_str(), "PrintListUpdates") == 0)
{
SEnviropment::Instance().m_PrintListUpdates = atoi(strValue.c_str()) != 0;
}
if (azstricmp(strKey.c_str(), "DedupeErrors") == 0)
{
SEnviropment::Instance().m_DedupeErrors = atoi(strValue.c_str()) != 0;
}
if (azstricmp(strKey.c_str(), "FallbackServer") == 0)
{
SEnviropment::Instance().m_FallbackServer = strValue;
}
if (azstricmp(strKey.c_str(), "FallbackTreshold") == 0)
{
SEnviropment::Instance().m_FallbackTreshold = atoi(strValue.c_str());
}
if (azstricmp(strKey.c_str(), "DumpShaders") == 0)
{
SEnviropment::Instance().m_DumpShaders = atoi(strValue.c_str()) != 0;
}
if (azstricmp(strKey.c_str(), "MaxConnections") == 0)
{
int maxConnections = atoi(strValue.c_str());
if (maxConnections <= 0)
{
printf("Warning: MaxConnections value is invalid. Using default value of %d\n", DEFAULT_MAX_CONNECTIONS);
}
else
{
SEnviropment::Instance().m_MaxConnections = maxConnections;
}
}
if (azstricmp(strKey.c_str(), "whitelist") == 0 || azstricmp(strKey.c_str(), "white_list") == 0)
{
std::regex ip4_address_regex("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/([0-9]|[1-2][0-9]|3[0-2]))?$");
AZStd::vector<AZStd::string> addresses;
AzFramework::StringFunc::Tokenize(strValue.c_str(), addresses, ',');
for (const auto& address : addresses)
{
if (std::regex_match(address.c_str(), ip4_address_regex))
{
SEnviropment::Instance().m_WhitelistAddresses.push_back(address);
}
else
{
printf("Warning: invalid IP address in the whitelist field: %s", address.c_str());
}
}
}
if (azstricmp(strKey.c_str(), "AllowElevatedPermissions") == 0)
{
int runAsRoot = atoi(strValue.c_str());
SEnviropment::Instance().m_RunAsRoot = (runAsRoot == 1);
}
#if defined(TOOLS_SUPPORT_XENIA)
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySCompileServer_cpp, xenia)
#endif
#if defined(TOOLS_SUPPORT_JASPER)
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySCompileServer_cpp, jasper)
#endif
#if defined(TOOLS_SUPPORT_PROVO)
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySCompileServer_cpp, provo)
#endif
#if defined(TOOLS_SUPPORT_SALEM)
#include AZ_RESTRICTED_FILE_EXPLICIT(CrySCompileServer_cpp, salem)
#endif
}
//////////////////////////////////////////////////////////////////////////
bool ParseConfig(const char* filename)
{
FILE* file = nullptr;
azfopen(&file, filename, "rb");
if (!file)
{
std::cout << "Config file not found" << std::endl;
return false;
}
fseek(file, 0, SEEK_END);
int nLen = ftell(file);
fseek(file, 0, SEEK_SET);
char* sAllText = new char [nLen + 16];
fread(sAllText, 1, nLen, file);
sAllText[nLen] = '\0';
sAllText[nLen + 1] = '\0';
std::string strGroup; // current group e.g. "[General]"
char* strLast = sAllText + nLen;
char* str = sAllText;
while (str < strLast)
{
char* s = str;
while (str < strLast && *str != '\n' && *str != '\r')
{
str++;
}
*str = '\0';
str++;
while (str < strLast && (*str == '\n' || *str == '\r'))
{
str++;
}
std::string strLine = s;
// detect groups e.g. "[General]" should set strGroup="General"
{
std::string strTrimmedLine(RemoveWhiteSpaces(strLine));
size_t size = strTrimmedLine.size();
if (size >= 3)
{
if (strTrimmedLine[0] == '[' && strTrimmedLine[size - 1] == ']') // currently no comments are allowed to be behind groups
{
strGroup = &strTrimmedLine[1];
strGroup.resize(size - 2); // remove [ and ]
continue; // next line
}
}
}
// skip comments
if (0 < strLine.find("--"))
{
// extract key
std::string::size_type posEq(strLine.find("=", 0));
if (std::string::npos != posEq)
{
std::string stemp(strLine, 0, posEq);
std::string strKey(RemoveWhiteSpaces(stemp));
// if (!strKey.empty())
{
// extract value
std::string::size_type posValueStart(strLine.find("\"", posEq + 1) + 1);
// std::string::size_type posValueEnd( strLine.find( "\"", posValueStart ) );
std::string::size_type posValueEnd(strLine.rfind('\"'));
std::string strValue;
if (std::string::npos != posValueStart && std::string::npos != posValueEnd)
{
strValue = std::string(strLine, posValueStart, posValueEnd - posValueStart);
}
else
{
std::string strTmp(strLine, posEq + 1, strLine.size() - (posEq + 1));
strValue = RemoveWhiteSpaces(strTmp);
}
OnLoadConfigurationEntry(strKey, strValue, strGroup);
}
}
} //--
}
delete []sAllText;
fclose(file);
return true;
}
std::string RemoveWhiteSpaces(std::string& str)
{
std::string::size_type pos1 = str.find_first_not_of(' ');
std::string::size_type pos2 = str.find_last_not_of(' ');
str = str.substr(pos1 == std::string::npos ? 0 : pos1, pos2 == std::string::npos ? str.length() - 1 : pos2 - pos1 + 1);
return str;
}
std::string AddSlash(const std::string& str)
{
if (!str.empty() &&
(str[str.size() - 1] != '\\') &&
(str[str.size() - 1] != '/'))
{
return str + "/";
}
return str;
}
};
namespace
{
AZ::JobManager* jobManager;
AZ::JobContext* globalJobContext;
#if defined(AZ_PLATFORM_WINDOWS)
BOOL ControlHandler([[maybe_unused]] DWORD controlType)
{
AZ::AllocatorInstance<AZ::ThreadPoolAllocator>::Destroy();
AZ::AllocatorInstance<AZ::SystemAllocator>::Destroy();
return FALSE;
}
#endif
}
void InitDefaults()
{
SEnviropment::Instance().m_port = STD_TCP_PORT;
SEnviropment::Instance().m_MaxConnections = DEFAULT_MAX_CONNECTIONS;
SEnviropment::Instance().m_FailEMail = "";
SEnviropment::Instance().m_MailInterval = 10;
SEnviropment::Instance().m_MailServer = "example.com";
SEnviropment::Instance().m_Caching = true;
SEnviropment::Instance().m_PrintErrors = true;
SEnviropment::Instance().m_PrintWarnings = false;
SEnviropment::Instance().m_PrintCommands = false;
SEnviropment::Instance().m_DedupeErrors = true;
SEnviropment::Instance().m_PrintListUpdates = true;
SEnviropment::Instance().m_FallbackTreshold = 16;
SEnviropment::Instance().m_FallbackServer = "";
SEnviropment::Instance().m_WhitelistAddresses.push_back("127.0.0.1");
SEnviropment::Instance().m_RunAsRoot = false;
SEnviropment::Instance().InitializePlatformAttributes();
}
bool ReadConfigFile()
{
char executableDir[AZ_MAX_PATH_LEN];
if (AZ::Utils::GetExecutableDirectory(executableDir, AZ_MAX_PATH_LEN) == AZ::Utils::ExecutablePathResult::Success)
{
auto configFilename = AZ::IO::Path(executableDir).Append("config.ini");
CConfigFile config;
config.ParseConfig(configFilename.c_str());
return true;
}
else
{
printf("error: failed to get executable directory.\n");
return false;
}
}
void RunServer(bool isRunningAsRoot)
{
if (isRunningAsRoot)
{
printf("\nWARNING: Attempting to run the CrySCompileServer as a user that has admininstrator permissions. This is a security risk and not recommended. Please run the service with a user account that does not have administrator permissions.\n\n");
}
if (!isRunningAsRoot || SEnviropment::Instance().m_RunAsRoot)
{
CCrySimpleHTTP HTTP;
CCrySimpleServer();
}
else
{
printf("If you need to run CrySCompileServer with administrator permisions you can create/edit the config.ini file in the same directory as this executable and add the following line to it:\n\tAllowElevatedPermissions=1\n");
}
}
int main(int argc, [[maybe_unused]] char* argv[])
{
if (argc != 1)
{
printf("usage: run without arguments\n");
return 0;
}
bool isRunningAsRoot = false;
#if defined(AZ_PLATFORM_WINDOWS)
// Check to see if we are running as root...
SID_IDENTIFIER_AUTHORITY ntAuthority = { SECURITY_NT_AUTHORITY };
PSID administratorsGroup;
BOOL sidAllocated = AllocateAndInitializeSid(
&ntAuthority,
2,
SECURITY_BUILTIN_DOMAIN_RID,
DOMAIN_ALIAS_RID_ADMINS,
0, 0, 0, 0, 0, 0,
&administratorsGroup);
if(sidAllocated)
{
BOOL isRoot = FALSE;
if (!CheckTokenMembership( NULL, administratorsGroup, &isRoot))
{
isRoot = FALSE;
}
FreeSid(administratorsGroup);
isRunningAsRoot = (isRoot == TRUE);
}
#if defined(_DEBUG)
int tmpFlag = _CrtSetDbgFlag(_CRTDBG_REPORT_FLAG);
tmpFlag |= _CRTDBG_LEAK_CHECK_DF;
// tmpFlag &= ~_CRTDBG_CHECK_CRT_DF;
_CrtSetDbgFlag(tmpFlag);
#endif
AZ_Verify(SetConsoleCtrlHandler(ControlHandler, TRUE), "Unable to setup windows console control handler");
#else
// if either the effective user id or effective group id is root, then we
// are running as root
isRunningAsRoot = (geteuid() == 0 || getegid() == 0);
#endif
AZ::AllocatorInstance<AZ::SystemAllocator>::Create();
AZ::AllocatorInstance<AZ::ThreadPoolAllocator>::Create();
AZ::JobManagerDesc jobManagerDescription;
int workers = AZStd::GetMin(AZStd::thread::hardware_concurrency(), static_cast<unsigned int>(8));
for (int idx = 0; idx < workers; ++idx)
{
jobManagerDescription.m_workerThreads.push_back(AZ::JobManagerThreadDesc());
}
jobManager = aznew AZ::JobManager(jobManagerDescription);
globalJobContext = aznew AZ::JobContext(*jobManager);
AZ::JobContext::SetGlobalContext(globalJobContext);
SEnviropment::Create();
InitDefaults();
if (ReadConfigFile())
{
RunServer(isRunningAsRoot);
}
SEnviropment::Destroy();
AZ::JobContext::SetGlobalContext(nullptr);
delete globalJobContext;
delete jobManager;
AZ::AllocatorInstance<AZ::ThreadPoolAllocator>::Destroy();
AZ::AllocatorInstance<AZ::SystemAllocator>::Destroy();
return 0;
}
@@ -0,0 +1,530 @@
/** @mainpage
<h1> TinyXML </h1>
TinyXML is a simple, small, C++ XML parser that can be easily
integrated into other programs.
<h2> What it does. </h2>
In brief, TinyXML parses an XML document, and builds from that a
Document Object Model (DOM) that can be read, modified, and saved.
XML stands for "eXtensible Markup Language." It allows you to create
your own document markups. Where HTML does a very good job of marking
documents for browsers, XML allows you to define any kind of document
markup, for example a document that describes a "to do" list for an
organizer application. XML is a very structured and convenient format.
All those random file formats created to store application data can
all be replaced with XML. One parser for everything.
The best place for the complete, correct, and quite frankly hard to
read spec is at <a href="http://www.w3.org/TR/2004/REC-xml-20040204/">
http://www.w3.org/TR/2004/REC-xml-20040204/</a>. An intro to XML
(that I really like) can be found at
<a href="http://skew.org/xml/tutorial/">http://skew.org/xml/tutorial</a>.
There are different ways to access and interact with XML data.
TinyXML uses a Document Object Model (DOM), meaning the XML data is parsed
into a C++ objects that can be browsed and manipulated, and then
written to disk or another output stream. You can also construct an XML document
from scratch with C++ objects and write this to disk or another output
stream.
TinyXML is designed to be easy and fast to learn. It is two headers
and four cpp files. Simply add these to your project and off you go.
There is an example file - xmltest.cpp - to get you started.
TinyXML is released under the ZLib license,
so you can use it in open source or commercial code. The details
of the license are at the top of every source file.
TinyXML attempts to be a flexible parser, but with truly correct and
compliant XML output. TinyXML should compile on any reasonably C++
compliant system. It does not rely on exceptions or RTTI. It can be
compiled with or without STL support. TinyXML fully supports
the UTF-8 encoding, and the first 64k character entities.
<h2> What it doesn't do. </h2>
TinyXML doesn't parse or use DTDs (Document Type Definitions) or XSLs
(eXtensible Stylesheet Language.) There are other parsers out there
(check out www.sourceforge.org, search for XML) that are much more fully
featured. But they are also much bigger, take longer to set up in
your project, have a higher learning curve, and often have a more
restrictive license. If you are working with browsers or have more
complete XML needs, TinyXML is not the parser for you.
The following DTD syntax will not parse at this time in TinyXML:
@verbatim
<!DOCTYPE Archiv [
<!ELEMENT Comment (#PCDATA)>
]>
@endverbatim
because TinyXML sees this as a !DOCTYPE node with an illegally
embedded !ELEMENT node. This may be addressed in the future.
<h2> Tutorials. </h2>
For the impatient, here is a tutorial to get you going. A great way to get started,
but it is worth your time to read this (very short) manual completely.
- @subpage tutorial0
<h2> Code Status. </h2>
TinyXML is mature, tested code. It is very stable. If you find
bugs, please file a bug report on the sourceforge web site
(www.sourceforge.net/projects/tinyxml). We'll get them straightened
out as soon as possible.
There are some areas of improvement; please check sourceforge if you are
interested in working on TinyXML.
<h2> Related Projects </h2>
TinyXML projects you may find useful! (Descriptions provided by the projects.)
<ul>
<li> <b>TinyXPath</b> (http://tinyxpath.sourceforge.net). TinyXPath is a small footprint
XPath syntax decoder, written in C++.</li>
<li> <b>TinyXML++</b> (http://code.google.com/p/ticpp/). TinyXML++ is a completely new
interface to TinyXML that uses MANY of the C++ strengths. Templates,
exceptions, and much better error handling.</li>
</ul>
<h2> Features </h2>
<h3> Using STL </h3>
TinyXML can be compiled to use or not use STL. When using STL, TinyXML
uses the std::string class, and fully supports std::istream, std::ostream,
operator<<, and operator>>. Many API methods have both 'const char*' and
'const std::string&' forms.
When STL support is compiled out, no STL files are included whatsoever. All
the string classes are implemented by TinyXML itself. API methods
all use the 'const char*' form for input.
Use the compile time #define:
TIXML_USE_STL
to compile one version or the other. This can be passed by the compiler,
or set as the first line of "tinyxml.h".
Note: If compiling the test code in Linux, setting the environment
variable TINYXML_USE_STL=YES/NO will control STL compilation. In the
Windows project file, STL and non STL targets are provided. In your project,
It's probably easiest to add the line "#define TIXML_USE_STL" as the first
line of tinyxml.h.
<h3> UTF-8 </h3>
TinyXML supports UTF-8 allowing to manipulate XML files in any language. TinyXML
also supports "legacy mode" - the encoding used before UTF-8 support and
probably best described as "extended ascii".
Normally, TinyXML will try to detect the correct encoding and use it. However,
by setting the value of TIXML_DEFAULT_ENCODING in the header file, TinyXML
can be forced to always use one encoding.
TinyXML will assume Legacy Mode until one of the following occurs:
<ol>
<li> If the non-standard but common "UTF-8 lead bytes" (0xef 0xbb 0xbf)
begin the file or data stream, TinyXML will read it as UTF-8. </li>
<li> If the declaration tag is read, and it has an encoding="UTF-8", then
TinyXML will read it as UTF-8. </li>
<li> If the declaration tag is read, and it has no encoding specified, then TinyXML will
read it as UTF-8. </li>
<li> If the declaration tag is read, and it has an encoding="something else", then TinyXML
will read it as Legacy Mode. In legacy mode, TinyXML will work as it did before. It's
not clear what that mode does exactly, but old content should keep working.</li>
<li> Until one of the above criteria is met, TinyXML runs in Legacy Mode.</li>
</ol>
What happens if the encoding is incorrectly set or detected? TinyXML will try
to read and pass through text seen as improperly encoded. You may get some strange results or
mangled characters. You may want to force TinyXML to the correct mode.
You may force TinyXML to Legacy Mode by using LoadFile( TIXML_ENCODING_LEGACY ) or
LoadFile( filename, TIXML_ENCODING_LEGACY ). You may force it to use legacy mode all
the time by setting TIXML_DEFAULT_ENCODING = TIXML_ENCODING_LEGACY. Likewise, you may
force it to TIXML_ENCODING_UTF8 with the same technique.
For English users, using English XML, UTF-8 is the same as low-ASCII. You
don't need to be aware of UTF-8 or change your code in any way. You can think
of UTF-8 as a "superset" of ASCII.
UTF-8 is not a double byte format - but it is a standard encoding of Unicode!
TinyXML does not use or directly support wchar, TCHAR, or Microsoft's _UNICODE at this time.
It is common to see the term "Unicode" improperly refer to UTF-16, a wide byte encoding
of unicode. This is a source of confusion.
For "high-ascii" languages - everything not English, pretty much - TinyXML can
handle all languages, at the same time, as long as the XML is encoded
in UTF-8. That can be a little tricky, older programs and operating systems
tend to use the "default" or "traditional" code page. Many apps (and almost all
modern ones) can output UTF-8, but older or stubborn (or just broken) ones
still output text in the default code page.
For example, Japanese systems traditionally use SHIFT-JIS encoding.
Text encoded as SHIFT-JIS can not be read by TinyXML.
A good text editor can import SHIFT-JIS and then save as UTF-8.
The <a href="http://skew.org/xml/tutorial/">Skew.org link</a> does a great
job covering the encoding issue.
The test file "utf8test.xml" is an XML containing English, Spanish, Russian,
and Simplified Chinese. (Hopefully they are translated correctly). The file
"utf8test.gif" is a screen capture of the XML file, rendered in IE. Note that
if you don't have the correct fonts (Simplified Chinese or Russian) on your
system, you won't see output that matches the GIF file even if you can parse
it correctly. Also note that (at least on my Windows machine) console output
is in a Western code page, so that Print() or printf() cannot correctly display
the file. This is not a bug in TinyXML - just an OS issue. No data is lost or
destroyed by TinyXML. The console just doesn't render UTF-8.
<h3> Entities </h3>
TinyXML recognizes the pre-defined "character entities", meaning special
characters. Namely:
@verbatim
&amp; &
&lt; <
&gt; >
&quot; "
&apos; '
@endverbatim
These are recognized when the XML document is read, and translated to there
UTF-8 equivalents. For instance, text with the XML of:
@verbatim
Far &amp; Away
@endverbatim
will have the Value() of "Far & Away" when queried from the TiXmlText object,
and will be written back to the XML stream/file as an ampersand. Older versions
of TinyXML "preserved" character entities, but the newer versions will translate
them into characters.
Additionally, any character can be specified by its Unicode code point:
The syntax "&#xA0;" or "&#160;" are both to the non-breaking space characher.
<h3> Printing </h3>
TinyXML can print output in several different ways that all have strengths and limitations.
- Print( FILE* ). Output to a std-C stream, which includes all C files as well as stdout.
- "Pretty prints", but you don't have control over printing options.
- The output is streamed directly to the FILE object, so there is no memory overhead
in the TinyXML code.
- used by Print() and SaveFile()
- operator<<. Output to a c++ stream.
- Integrates with standart C++ iostreams.
- Outputs in "network printing" mode without line breaks. Good for network transmission
and moving XML between C++ objects, but hard for a human to read.
- TiXmlPrinter. Output to a std::string or memory buffer.
- API is less concise
- Future printing options will be put here.
- Printing may change slightly in future versions as it is refined and expanded.
<h3> Streams </h3>
With TIXML_USE_STL on TinyXML supports C++ streams (operator <<,>>) streams as well
as C (FILE*) streams. There are some differences that you may need to be aware of.
C style output:
- based on FILE*
- the Print() and SaveFile() methods
Generates formatted output, with plenty of white space, intended to be as
human-readable as possible. They are very fast, and tolerant of ill formed
XML documents. For example, an XML document that contains 2 root elements
and 2 declarations, will still print.
C style input:
- based on FILE*
- the Parse() and LoadFile() methods
A fast, tolerant read. Use whenever you don't need the C++ streams.
C++ style output:
- based on std::ostream
- operator<<
Generates condensed output, intended for network transmission rather than
readability. Depending on your system's implementation of the ostream class,
these may be somewhat slower. (Or may not.) Not tolerant of ill formed XML:
a document should contain the correct one root element. Additional root level
elements will not be streamed out.
C++ style input:
- based on std::istream
- operator>>
Reads XML from a stream, making it useful for network transmission. The tricky
part is knowing when the XML document is complete, since there will almost
certainly be other data in the stream. TinyXML will assume the XML data is
complete after it reads the root element. Put another way, documents that
are ill-constructed with more than one root element will not read correctly.
Also note that operator>> is somewhat slower than Parse, due to both
implementation of the STL and limitations of TinyXML.
<h3> White space </h3>
The world simply does not agree on whether white space should be kept, or condensed.
For example, pretend the '_' is a space, and look at "Hello____world". HTML, and
at least some XML parsers, will interpret this as "Hello_world". They condense white
space. Some XML parsers do not, and will leave it as "Hello____world". (Remember
to keep pretending the _ is a space.) Others suggest that __Hello___world__ should become
Hello___world.
It's an issue that hasn't been resolved to my satisfaction. TinyXML supports the
first 2 approaches. Call TiXmlBase::SetCondenseWhiteSpace( bool ) to set the desired behavior.
The default is to condense white space.
If you change the default, you should call TiXmlBase::SetCondenseWhiteSpace( bool )
before making any calls to Parse XML data, and I don't recommend changing it after
it has been set.
<h3> Handles </h3>
Where browsing an XML document in a robust way, it is important to check
for null returns from method calls. An error safe implementation can
generate a lot of code like:
@verbatim
TiXmlElement* root = document.FirstChildElement( "Document" );
if ( root )
{
TiXmlElement* element = root->FirstChildElement( "Element" );
if ( element )
{
TiXmlElement* child = element->FirstChildElement( "Child" );
if ( child )
{
TiXmlElement* child2 = child->NextSiblingElement( "Child" );
if ( child2 )
{
// Finally do something useful.
@endverbatim
Handles have been introduced to clean this up. Using the TiXmlHandle class,
the previous code reduces to:
@verbatim
TiXmlHandle docHandle( &document );
TiXmlElement* child2 = docHandle.FirstChild( "Document" ).FirstChild( "Element" ).Child( "Child", 1 ).ToElement();
if ( child2 )
{
// do something useful
@endverbatim
Which is much easier to deal with. See TiXmlHandle for more information.
<h3> Row and Column tracking </h3>
Being able to track nodes and attributes back to their origin location
in source files can be very important for some applications. Additionally,
knowing where parsing errors occured in the original source can be very
time saving.
TinyXML can tracks the row and column origin of all nodes and attributes
in a text file. The TiXmlBase::Row() and TiXmlBase::Column() methods return
the origin of the node in the source text. The correct tabs can be
configured in TiXmlDocument::SetTabSize().
<h2> Using and Installing </h2>
To Compile and Run xmltest:
A Linux Makefile and a Windows Visual C++ .dsw file is provided.
Simply compile and run. It will write the file demotest.xml to your
disk and generate output on the screen. It also tests walking the
DOM by printing out the number of nodes found using different
techniques.
The Linux makefile is very generic and runs on many systems - it
is currently tested on mingw and
MacOSX. You do not need to run 'make depend'. The dependecies have been
hard coded.
<h3>Windows project file for VC6</h3>
<ul>
<li>tinyxml: tinyxml library, non-STL </li>
<li>tinyxmlSTL: tinyxml library, STL </li>
<li>tinyXmlTest: test app, non-STL </li>
<li>tinyXmlTestSTL: test app, STL </li>
</ul>
<h3>Makefile</h3>
At the top of the makefile you can set:
PROFILE, DEBUG, and TINYXML_USE_STL. Details (such that they are) are in
the makefile.
In the tinyxml directory, type "make clean" then "make". The executable
file 'xmltest' will be created.
<h3>To Use in an Application:</h3>
Add tinyxml.cpp, tinyxml.h, tinyxmlerror.cpp, tinyxmlparser.cpp, tinystr.cpp, and tinystr.h to your
project or make file. That's it! It should compile on any reasonably
compliant C++ system. You do not need to enable exceptions or
RTTI for TinyXML.
<h2> How TinyXML works. </h2>
An example is probably the best way to go. Take:
@verbatim
<?xml version="1.0" standalone=no>
<!-- Our to do list data -->
<ToDo>
<Item priority="1"> Go to the <bold>Toy store!</bold></Item>
<Item priority="2"> Do bills</Item>
</ToDo>
@endverbatim
Its not much of a To Do list, but it will do. To read this file
(say "demo.xml") you would create a document, and parse it in:
@verbatim
TiXmlDocument doc( "demo.xml" );
doc.LoadFile();
@endverbatim
And its ready to go. Now lets look at some lines and how they
relate to the DOM.
@verbatim
<?xml version="1.0" standalone=no>
@endverbatim
The first line is a declaration, and gets turned into the
TiXmlDeclaration class. It will be the first child of the
document node.
This is the only directive/special tag parsed by TinyXML.
Generally directive tags are stored in TiXmlUnknown so the
commands wont be lost when it is saved back to disk.
@verbatim
<!-- Our to do list data -->
@endverbatim
A comment. Will become a TiXmlComment object.
@verbatim
<ToDo>
@endverbatim
The "ToDo" tag defines a TiXmlElement object. This one does not have
any attributes, but does contain 2 other elements.
@verbatim
<Item priority="1">
@endverbatim
Creates another TiXmlElement which is a child of the "ToDo" element.
This element has 1 attribute, with the name "priority" and the value
"1".
@verbatim
Go to the
@endverbatim
A TiXmlText. This is a leaf node and cannot contain other nodes.
It is a child of the "Item" TiXmlElement.
@verbatim
<bold>
@endverbatim
Another TiXmlElement, this one a child of the "Item" element.
Etc.
Looking at the entire object tree, you end up with:
@verbatim
TiXmlDocument "demo.xml"
TiXmlDeclaration "version='1.0'" "standalone=no"
TiXmlComment " Our to do list data"
TiXmlElement "ToDo"
TiXmlElement "Item" Attribtutes: priority = 1
TiXmlText "Go to the "
TiXmlElement "bold"
TiXmlText "Toy store!"
TiXmlElement "Item" Attributes: priority=2
TiXmlText "Do bills"
@endverbatim
<h2> Documentation </h2>
The documentation is build with Doxygen, using the 'dox'
configuration file.
<h2> License </h2>
TinyXML is released under the zlib license:
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any
damages arising from the use of this software.
Permission is granted to anyone to use this software for any
purpose, including commercial applications, and to alter it and
redistribute it freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must
not claim that you wrote the original software. If you use this
software in a product, an acknowledgment in the product documentation
would be appreciated but is not required.
2. Altered source versions must be plainly marked as such, and
must not be misrepresented as being the original software.
3. This notice may not be removed or altered from any source
distribution.
<h2> References </h2>
The World Wide Web Consortium is the definitive standard body for
XML, and their web pages contain huge amounts of information.
The definitive spec: <a href="http://www.w3.org/TR/2004/REC-xml-20040204/">
http://www.w3.org/TR/2004/REC-xml-20040204/</a>
I also recommend "XML Pocket Reference" by Robert Eckstein and published by
OReilly...the book that got the whole thing started.
<h2> Contributors, Contacts, and a Brief History </h2>
Thanks very much to everyone who sends suggestions, bugs, ideas, and
encouragement. It all helps, and makes this project fun. A special thanks
to the contributors on the web pages that keep it lively.
So many people have sent in bugs and ideas, that rather than list here
we try to give credit due in the "changes.txt" file.
TinyXML was originally written by Lee Thomason. (Often the "I" still
in the documentation.) Lee reviews changes and releases new versions,
with the help of Yves Berquin, Andrew Ellerton, and the tinyXml community.
We appreciate your suggestions, and would love to know if you
use TinyXML. Hopefully you will enjoy it and find it useful.
Please post questions, comments, file bugs, or contact us at:
www.sourceforge.net/projects/tinyxml
Lee Thomason, Yves Berquin, Andrew Ellerton
*/
@@ -0,0 +1,116 @@
/*
www.sourceforge.net/projects/tinyxml
Original file by Yves Berquin.
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any
damages arising from the use of this software.
Permission is granted to anyone to use this software for any
purpose, including commercial applications, and to alter it and
redistribute it freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must
not claim that you wrote the original software. If you use this
software in a product, an acknowledgment in the product documentation
would be appreciated but is not required.
2. Altered source versions must be plainly marked as such, and
must not be misrepresented as being the original software.
3. This notice may not be removed or altered from any source
distribution.
*/
/*
* THIS FILE WAS ALTERED BY Tyge Lovset, 7. April 2005.
*/
#ifndef TIXML_USE_STL
#include "tinystr.h"
// Error value for find primitive
const TiXmlString::size_type TiXmlString::npos = static_cast< TiXmlString::size_type >(-1);
// Null rep.
TiXmlString::Rep TiXmlString::nullrep_ = { 0, 0, { '\0' } };
void TiXmlString::reserve (size_type cap)
{
if (cap > capacity())
{
TiXmlString tmp;
tmp.init(length(), cap);
memcpy(tmp.start(), data(), length());
swap(tmp);
}
}
TiXmlString& TiXmlString::assign(const char* str, size_type len)
{
size_type cap = capacity();
if (len > cap || cap > 3*(len + 8))
{
TiXmlString tmp;
tmp.init(len);
memcpy(tmp.start(), str, len);
swap(tmp);
}
else
{
memmove(start(), str, len);
set_size(len);
}
return *this;
}
TiXmlString& TiXmlString::append(const char* str, size_type len)
{
size_type newsize = length() + len;
if (newsize > capacity())
{
reserve (newsize + capacity());
}
memmove(finish(), str, len);
set_size(newsize);
return *this;
}
TiXmlString operator + (const TiXmlString & a, const TiXmlString & b)
{
TiXmlString tmp;
tmp.reserve(a.length() + b.length());
tmp += a;
tmp += b;
return tmp;
}
TiXmlString operator + (const TiXmlString & a, const char* b)
{
TiXmlString tmp;
TiXmlString::size_type b_len = static_cast<TiXmlString::size_type>( strlen(b) );
tmp.reserve(a.length() + b_len);
tmp += a;
tmp.append(b, b_len);
return tmp;
}
TiXmlString operator + (const char* a, const TiXmlString & b)
{
TiXmlString tmp;
TiXmlString::size_type a_len = static_cast<TiXmlString::size_type>( strlen(a) );
tmp.reserve(a_len + b.length());
tmp.append(a, a_len);
tmp += b;
return tmp;
}
#endif // TIXML_USE_STL
@@ -0,0 +1,309 @@
// Modifications copyright Amazon.com, Inc. or its affiliates.
/*
www.sourceforge.net/projects/tinyxml
Original file by Yves Berquin.
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any
damages arising from the use of this software.
Permission is granted to anyone to use this software for any
purpose, including commercial applications, and to alter it and
redistribute it freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must
not claim that you wrote the original software. If you use this
software in a product, an acknowledgment in the product documentation
would be appreciated but is not required.
2. Altered source versions must be plainly marked as such, and
must not be misrepresented as being the original software.
3. This notice may not be removed or altered from any source
distribution.
*/
#ifndef TIXML_USE_STL
#ifndef TIXML_STRING_INCLUDED
#define TIXML_STRING_INCLUDED
#include <assert.h>
#include <string.h>
/* The support for explicit isn't that universal, and it isn't really
required - it is used to check that the TiXmlString class isn't incorrectly
used. Be nice to old compilers and macro it here:
*/
#if defined(_MSC_VER)
// Microsoft visual studio, version 6 and higher.
#define TIXML_EXPLICIT explicit
#elif defined(__GNUC__) && (__GNUC__ >= 3 )
// GCC version 3 and higher.s
#define TIXML_EXPLICIT explicit
#else
#define TIXML_EXPLICIT
#endif
/*
TiXmlString is an emulation of a subset of the std::string template.
Its purpose is to allow compiling TinyXML on compilers with no or poor STL support.
Only the member functions relevant to the TinyXML project have been implemented.
The buffer allocation is made by a simplistic power of 2 like mechanism : if we increase
a string and there's no more room, we allocate a buffer twice as big as we need.
*/
class TiXmlString
{
public :
// The size type used
typedef size_t size_type;
// Error value for find primitive
static const size_type npos; // = -1;
// TiXmlString empty constructor
TiXmlString () : rep_(&nullrep_)
{
}
// TiXmlString copy constructor
TiXmlString ( const TiXmlString & copy) : rep_(0)
{
init(copy.length());
memcpy(start(), copy.data(), length());
}
// TiXmlString constructor, based on a string
TIXML_EXPLICIT TiXmlString ( const char * copy) : rep_(0)
{
init( static_cast<size_type>( strlen(copy) ));
memcpy(start(), copy, length());
}
// TiXmlString constructor, based on a string
TIXML_EXPLICIT TiXmlString ( const char * str, size_type len) : rep_(0)
{
init(len);
memcpy(start(), str, len);
}
// TiXmlString destructor
~TiXmlString ()
{
quit();
}
// = operator
TiXmlString& operator = (const char * copy)
{
return assign( copy, (size_type)strlen(copy));
}
// = operator
TiXmlString& operator = (const TiXmlString & copy)
{
return assign(copy.start(), copy.length());
}
// += operator. Maps to append
TiXmlString& operator += (const char * suffix)
{
return append(suffix, static_cast<size_type>( strlen(suffix) ));
}
// += operator. Maps to append
TiXmlString& operator += (char single)
{
return append(&single, 1);
}
// += operator. Maps to append
TiXmlString& operator += (const TiXmlString & suffix)
{
return append(suffix.data(), suffix.length());
}
// Convert a TiXmlString into a null-terminated char *
const char * c_str () const { return rep_->str; }
// Convert a TiXmlString into a char * (need not be null terminated).
const char * data () const { return rep_->str; }
// Return the length of a TiXmlString
size_type length () const { return rep_->size; }
// Alias for length()
size_type size () const { return rep_->size; }
// Checks if a TiXmlString is empty
bool empty () const { return rep_->size == 0; }
// Return capacity of string
size_type capacity () const { return rep_->capacity; }
// single char extraction
const char& at (size_type index) const
{
assert( index < length() );
return rep_->str[ index ];
}
// [] operator
char& operator [] (size_type index) const
{
assert( index < length() );
return rep_->str[ index ];
}
// find a char in a string. Return TiXmlString::npos if not found
size_type find (char lookup) const
{
return find(lookup, 0);
}
// find a char in a string from an offset. Return TiXmlString::npos if not found
size_type find (char tofind, size_type offset) const
{
if (offset >= length()) return npos;
for (const char* p = c_str() + offset; *p != '\0'; ++p)
{
if (*p == tofind) return static_cast< size_type >( p - c_str() );
}
return npos;
}
void clear ()
{
//Lee:
//The original was just too strange, though correct:
// TiXmlString().swap(*this);
//Instead use the quit & re-init:
quit();
init(0,0);
}
/* Function to reserve a big amount of data when we know we'll need it. Be aware that this
function DOES NOT clear the content of the TiXmlString if any exists.
*/
void reserve (size_type cap);
TiXmlString& assign (const char* str, size_type len);
TiXmlString& append (const char* str, size_type len);
void swap (TiXmlString& other)
{
Rep* r = rep_;
rep_ = other.rep_;
other.rep_ = r;
}
private:
void init(size_type sz) { init(sz, sz); }
void set_size(size_type sz) { rep_->str[ rep_->size = sz ] = '\0'; }
char* start() const { return rep_->str; }
char* finish() const { return rep_->str + rep_->size; }
struct Rep
{
size_type size, capacity;
char str[1];
};
void init(size_type sz, size_type cap)
{
if (cap)
{
// Lee: the original form:
// rep_ = static_cast<Rep*>(operator new(sizeof(Rep) + cap));
// doesn't work in some cases of new being overloaded. Switching
// to the normal allocation, although use an 'int' for systems
// that are overly picky about structure alignment.
const size_type bytesNeeded = sizeof(Rep) + cap;
const size_type intsNeeded = ( bytesNeeded + sizeof(int) - 1 ) / sizeof( int );
rep_ = reinterpret_cast<Rep*>( new int[ intsNeeded ] );
rep_->str[ rep_->size = sz ] = '\0';
rep_->capacity = cap;
}
else
{
rep_ = &nullrep_;
}
}
void quit()
{
if (rep_ != &nullrep_)
{
// The rep_ is really an array of ints. (see the allocator, above).
// Cast it back before delete, so the compiler won't incorrectly call destructors.
delete [] ( reinterpret_cast<int*>( rep_ ) );
}
}
Rep * rep_;
static Rep nullrep_;
} ;
inline bool operator == (const TiXmlString & a, const TiXmlString & b)
{
return ( a.length() == b.length() ) // optimization on some platforms
&& ( strcmp(a.c_str(), b.c_str()) == 0 ); // actual compare
}
inline bool operator < (const TiXmlString & a, const TiXmlString & b)
{
return strcmp(a.c_str(), b.c_str()) < 0;
}
inline bool operator != (const TiXmlString & a, const TiXmlString & b) { return !(a == b); }
inline bool operator > (const TiXmlString & a, const TiXmlString & b) { return b < a; }
inline bool operator <= (const TiXmlString & a, const TiXmlString & b) { return !(b < a); }
inline bool operator >= (const TiXmlString & a, const TiXmlString & b) { return !(a < b); }
inline bool operator == (const TiXmlString & a, const char* b) { return strcmp(a.c_str(), b) == 0; }
inline bool operator == (const char* a, const TiXmlString & b) { return b == a; }
inline bool operator != (const TiXmlString & a, const char* b) { return !(a == b); }
inline bool operator != (const char* a, const TiXmlString & b) { return !(b == a); }
TiXmlString operator + (const TiXmlString & a, const TiXmlString & b);
TiXmlString operator + (const TiXmlString & a, const char* b);
TiXmlString operator + (const char* a, const TiXmlString & b);
/*
TiXmlOutStream is an emulation of std::ostream. It is based on TiXmlString.
Only the operators that we need for TinyXML have been developped.
*/
class TiXmlOutStream : public TiXmlString
{
public :
// TiXmlOutStream << operator.
TiXmlOutStream & operator << (const TiXmlString & in)
{
*this += in;
return *this;
}
// TiXmlOutStream << operator.
TiXmlOutStream & operator << (const char * in)
{
*this += in;
return *this;
}
} ;
#endif // TIXML_STRING_INCLUDED
#endif // TIXML_USE_STL
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,53 @@
/*
www.sourceforge.net/projects/tinyxml
Original code (2.0 and earlier )copyright (c) 2000-2006 Lee Thomason (www.grinninglizard.com)
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any
damages arising from the use of this software.
Permission is granted to anyone to use this software for any
purpose, including commercial applications, and to alter it and
redistribute it freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must
not claim that you wrote the original software. If you use this
software in a product, an acknowledgment in the product documentation
would be appreciated but is not required.
2. Altered source versions must be plainly marked as such, and
must not be misrepresented as being the original software.
3. This notice may not be removed or altered from any source
distribution.
*/
#include "tinyxml.h"
// The goal of the seperate error file is to make the first
// step towards localization. tinyxml (currently) only supports
// english error messages, but the could now be translated.
//
// It also cleans up the code a bit.
//
const char* TiXmlBase::errorString[ TIXML_ERROR_STRING_COUNT ] =
{
"No error",
"Error",
"Failed to open file",
"Memory allocation failed.",
"Error parsing Element.",
"Failed to read Element name",
"Error reading Element value.",
"Error reading Attributes.",
"Error: empty tag.",
"Error reading end tag.",
"Error parsing Unknown.",
"Error parsing Comment.",
"Error parsing Declaration.",
"Error document empty.",
"Error null (0) or unexpected EOF found in input stream.",
"Error parsing CDATA.",
"Error when TiXmlDocument added to document, because TiXmlDocument can only be at the root.",
};
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,16 @@
@echo off
REM
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
REM its licensors.
REM
REM REM For complete copyright and license terms please see the LICENSE at the root of this
REM distribution (the "License"). All use of this software is governed by the License,
REM or, if provided, by the license below or the license accompanying this file. Do not
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
REM
REM
REM Original file Copyright Crytek GMBH or its affiliates, used under license.
REM
dir Cache /a:-d /s /b >dir.txt
@@ -0,0 +1,12 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(PAL_TRAIT_BUILD_CRYSCOMPILESERVER_SUPPORTED FALSE)
@@ -0,0 +1,10 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
@@ -0,0 +1,15 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(LY_COMPILE_OPTIONS
PRIVATE
-fexceptions
)
@@ -0,0 +1,15 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(LY_COMPILE_OPTIONS
PRIVATE
/EHsc
)
@@ -0,0 +1,12 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(PAL_TRAIT_BUILD_CRYSCOMPILESERVER_SUPPORTED FALSE)
@@ -0,0 +1,10 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
@@ -0,0 +1,12 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(PAL_TRAIT_BUILD_CRYSCOMPILESERVER_SUPPORTED TRUE)
@@ -0,0 +1,15 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(LY_RUNTIME_DEPENDENCIES
3rdParty::DirectXShaderCompiler::dxcGL
3rdParty::DirectXShaderCompiler::dxcMetal
)
@@ -0,0 +1,12 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(PAL_TRAIT_BUILD_CRYSCOMPILESERVER_SUPPORTED TRUE)
@@ -0,0 +1,44 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(LY_RUNTIME_DEPENDENCIES
3rdParty::DirectXShaderCompiler::dxcGL
3rdParty::DirectXShaderCompiler::dxcMetal
)
file(TO_CMAKE_PATH "$ENV{ProgramFiles\(x86\)}" program_files_path)
ly_add_target_files(
TARGETS CrySCompileServer
OUTPUT_SUBDIRECTORY Compiler/PCD3D11/v006
FILES
"${program_files_path}/Windows Kits/10/bin/${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}/x64/fxc.exe"
"${program_files_path}/Windows Kits/10/bin/${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}/x64/d3dcompiler_47.dll"
"${program_files_path}/Windows Kits/10/bin/${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}/x64/d3dcsx_47.dll"
"${program_files_path}/Windows Kits/10/bin/${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}/x64/d3dcsxd_47.dll"
)
ly_add_target_files(
TARGETS CrySCompileServer
OUTPUT_SUBDIRECTORY Compiler/PCGL/V006
FILES
"${LY_ROOT_FOLDER}/Tools/CrySCompileServer/Compiler/PCGL/V006/D3DCompiler_47.dll"
"${LY_ROOT_FOLDER}/Tools/CrySCompileServer/Compiler/PCGL/V006/HLSLcc.exe"
)
ly_add_target_files(
TARGETS CrySCompileServer
OUTPUT_SUBDIRECTORY Compiler/PCGMETAL/HLSLcc
FILES
"${LY_ROOT_FOLDER}/Tools/CrySCompileServer/Compiler/PCGMETAL/HLSLcc/HLSLcc_d.exe"
"${LY_ROOT_FOLDER}/Tools/CrySCompileServer/Compiler/PCGMETAL/HLSLcc/HLSLcc.exe"
)
@@ -0,0 +1,12 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(PAL_TRAIT_BUILD_CRYSCOMPILESERVER_SUPPORTED FALSE)
@@ -0,0 +1,10 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
@@ -0,0 +1,60 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(FILES
CrySCompileServer.cpp
Core/Common.h
Core/Error.cpp
Core/Error.hpp
Core/WindowsAPIImplementation.h
Core/WindowsAPIImplementation.cpp
Core/Mailer.cpp
Core/Mailer.h
Core/MD5.hpp
Core/StdTypes.hpp
Core/STLHelper.cpp
Core/STLHelper.hpp
Core/Server/CrySimpleCache.cpp
Core/Server/CrySimpleCache.hpp
Core/Server/CrySimpleErrorLog.cpp
Core/Server/CrySimpleErrorLog.hpp
Core/Server/CrySimpleFileGuard.hpp
Core/Server/CrySimpleHTTP.cpp
Core/Server/CrySimpleHTTP.hpp
Core/Server/CrySimpleJob.cpp
Core/Server/CrySimpleJob.hpp
Core/Server/CrySimpleJobCache.cpp
Core/Server/CrySimpleJobCache.hpp
Core/Server/CrySimpleJobCompile.cpp
Core/Server/CrySimpleJobCompile.hpp
Core/Server/CrySimpleJobCompile1.cpp
Core/Server/CrySimpleJobCompile1.hpp
Core/Server/CrySimpleJobCompile2.cpp
Core/Server/CrySimpleJobCompile2.hpp
Core/Server/CrySimpleJobRequest.cpp
Core/Server/CrySimpleJobRequest.hpp
Core/Server/CrySimpleJobGetShaderList.cpp
Core/Server/CrySimpleJobGetShaderList.hpp
Core/Server/CrySimpleMutex.cpp
Core/Server/CrySimpleMutex.hpp
Core/Server/CrySimpleServer.cpp
Core/Server/CrySimpleServer.hpp
Core/Server/CrySimpleSock.cpp
Core/Server/CrySimpleSock.hpp
Core/Server/ShaderList.cpp
Core/Server/ShaderList.hpp
External/tinyxml/tinystr.cpp
External/tinyxml/tinystr.h
External/tinyxml/tinyxml.cpp
External/tinyxml/tinyxml.h
External/tinyxml/tinyxmlerror.cpp
External/tinyxml/tinyxmlparser.cpp
)