/* * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or * its licensors. * * For complete copyright and license terms please see the LICENSE at the root of this * distribution (the "License"). All use of this software is governed by the License, * or, if provided, by the license below or the license accompanying this file. Do not * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * */ #include #include #include #include #include #include #include #include #include #include #include #include #include namespace AZ { namespace RHI { static AZStd::mutex s_profilingMutex; static constexpr char ShaderPlatformInterfaceName[] = "ShaderPlatformInterface"; void ShaderCompilerProfiling::Entry::Reflect(ReflectContext* context) { if (auto* serializeContext = azrtti_cast(context)) { serializeContext->Class() ->Version(0) ->Field("ExecutablePath", &Entry::m_executablePath) ->Field("Parameters", &Entry::m_parameters) ->Field("ElapsedTimeSeconds", &Entry::m_elapsedTimeSeconds) ; } } void ShaderCompilerProfiling::Reflect(ReflectContext* context) { if (auto* serializeContext = azrtti_cast(context)) { serializeContext->Class() ->Version(0) ->Field("Entries", &ShaderCompilerProfiling::m_entries) ; } Entry::Reflect(context); } //! Helper function to write a new entry in the shader compilation profiling data (see the ShaderCompilerProfiling struct). //! The function tries to read the existing JSON file if it exists, and appends a new entry for a compilation request. //! The profiling data consists of the compiler executable, the parameters used to call it and the elapsed time for that call. bool WriteProfilingEntryToLog(const AZStd::string& shaderPath, const ShaderCompilerProfiling::Entry& profilingEntry) { AZStd::lock_guard lock(s_profilingMutex); AZStd::string folderPath; AzFramework::StringFunc::Path::GetFullPath(shaderPath.data(), folderPath); AZStd::string fileName; AzFramework::StringFunc::Path::GetFileName(shaderPath.data(), fileName); AzFramework::StringFunc::Path::ReplaceExtension(fileName, ""); const AZ::Platform::ProcessId processId = AZ::Platform::GetCurrentProcessId(); AZStd::string filePath = shaderPath; AZStd::string logName = AZStd::string::format("%s.%d.profiling", fileName.data(), processId); AzFramework::StringFunc::Path::Join(folderPath.data(), logName.data(), filePath); RHI::ShaderCompilerProfiling profiling; if (IO::FileIOBase::GetInstance()->Exists(filePath.data())) { auto loadResult = AZ::JsonSerializationUtils::LoadObjectFromFile(profiling, filePath); if (!loadResult.IsSuccess()) { AZ_Error(ShaderPlatformInterfaceName, false, "Failed to load shader compiler profiling from file [%s]", filePath.data()); AZ_Error(ShaderPlatformInterfaceName, false, "Loading issues: %s", loadResult.GetError().data()); return false; } } profiling.m_entries.push_back(profilingEntry); auto saveResult = AZ::JsonSerializationUtils::SaveObjectToFile(&profiling, filePath); if (!(saveResult.IsSuccess())) { AZ_Error(ShaderPlatformInterfaceName, false, "Failed to save shader compiler profiling to file %s", filePath.data()); AZ_Error(ShaderPlatformInterfaceName, false, "Saving issues: %s", saveResult.GetError().data()); return false; } return true; } AZStd::string PrependFile(PrependArguments& arguments) { static const char* executableFolder = nullptr; const auto AsAbsolute = [](const AZStd::string& localFile) -> AZStd::optional { if (!AzFramework::StringFunc::Path::IsRelative(localFile.c_str())) { return localFile; } if (!executableFolder) { AZ::ComponentApplicationBus::BroadcastResult(executableFolder, &AZ::ComponentApplicationBus::Events::GetExecutableFolder); if (!executableFolder) { AZ_Error(ShaderPlatformInterfaceName, false, "Unable to determine application root."); return AZStd::nullopt; } } AZStd::string fileAbsolutePath; AzFramework::StringFunc::Path::Join(executableFolder, localFile.c_str(), fileAbsolutePath); return fileAbsolutePath; }; // If either file path is empty there is nothing to prepend. Return the original source file path (even if empty). if (!arguments.m_prependFile || !arguments.m_sourceFile) { return arguments.m_sourceFile; } // Read the prepended file auto prependFileAbsolutePath = AsAbsolute(arguments.m_prependFile); if (!prependFileAbsolutePath) { return arguments.m_sourceFile; } if (!AZ::IO::SystemFile::Exists(prependFileAbsolutePath->c_str())) { AZ_Warning(ShaderPlatformInterfaceName, false, "Missing prepend file: '%s'. Will continue without prepending.", prependFileAbsolutePath->c_str()); return arguments.m_sourceFile; } auto prependFileLoadResult = LoadFileString(prependFileAbsolutePath->c_str()); if (!prependFileLoadResult) { AZ_Error(ShaderPlatformInterfaceName, false, "%s", prependFileLoadResult.GetError().c_str()); return arguments.m_sourceFile; } auto sourceFileAbsolutePath = AsAbsolute(arguments.m_sourceFile); if (!sourceFileAbsolutePath) { return arguments.m_sourceFile; } auto sourceFileLoadResult = LoadFileString(sourceFileAbsolutePath->c_str()); if (!sourceFileLoadResult) { AZ_Error(ShaderPlatformInterfaceName, false, "%s", sourceFileLoadResult.GetError().c_str()); return *sourceFileAbsolutePath; } AZStd::string combinedFile; if (arguments.m_destinationFolder) { AZStd::string filename; if(AzFramework::StringFunc::Path::GetFullFileName(sourceFileAbsolutePath->c_str(), filename)) { combinedFile = AZStd::string::format("%s/%s", arguments.m_destinationFolder, filename.c_str()); } else { AZ_Error(ShaderPlatformInterfaceName, false, "GetFullFileName('%s') failed", sourceFileAbsolutePath->c_str()); return *sourceFileAbsolutePath; } } else { combinedFile = *sourceFileAbsolutePath; } combinedFile += (arguments.m_addSuffixToFileName ? "." + AZStd::string{ arguments.m_addSuffixToFileName } : "") + ".prepend"; if (arguments.m_destinationStringOpt) { *arguments.m_destinationStringOpt = prependFileLoadResult.GetValue().c_str(); *arguments.m_destinationStringOpt += sourceFileLoadResult.GetValue().c_str(); } else { // Write the file back to disk (if requested) so the native shader compiler can read it. AZ::IO::FileIOStream combinedFileStream(combinedFile.c_str(), AZ::IO::OpenMode::ModeWrite); if (!combinedFileStream.IsOpen()) { AZ_Error(ShaderPlatformInterfaceName, false, "Failed to open output file %s", combinedFile.c_str()); return *sourceFileAbsolutePath; } combinedFileStream.Write(prependFileLoadResult.GetValue().size(), prependFileLoadResult.GetValue().data()); combinedFileStream.Write(sourceFileLoadResult.GetValue().size(), sourceFileLoadResult.GetValue().data()); combinedFileStream.Close(); } if (arguments.m_digest) // if the function's caller requested to compute a digest, let's hash the content and store it in the digest array. { // this is useful for lld/pdb file naming (shader debug symbols) because it's the automatically recognized scheme by PIX and alike. MD5Context md5; MD5Init(&md5); MD5Update(&md5, reinterpret_cast(prependFileLoadResult.GetValue().data()), aznumeric_cast(prependFileLoadResult.GetValue().size())); MD5Update(&md5, reinterpret_cast(sourceFileLoadResult.GetValue().data()), aznumeric_cast(sourceFileLoadResult.GetValue().size())); MD5Final(*arguments.m_digest, &md5); } return combinedFile; } bool ExecuteShaderCompiler(const AZStd::string& executablePath, const AZStd::string& parameters, const AZStd::string& shaderSourcePathForDebug, const char* toolNameForLog) { AZStd::string executableAbsolutePath; if (AzFramework::StringFunc::Path::IsRelative(executablePath.c_str())) { static const char* executableFolder = nullptr; if (!executableFolder) { AZ::ComponentApplicationBus::BroadcastResult(executableFolder, &AZ::ComponentApplicationBus::Events::GetExecutableFolder); if (!executableFolder) { AZ_Error(ShaderPlatformInterfaceName, false, "Unable to determine application root."); return false; } } AzFramework::StringFunc::Path::Join(executableFolder, executablePath.c_str(), executableAbsolutePath); } else { executableAbsolutePath = executablePath; } if (!AZ::IO::SystemFile::Exists(executableAbsolutePath.c_str())) { AZ_Error(ShaderPlatformInterfaceName, false, "Executable not found: '%s'", executableAbsolutePath.c_str()); return false; } AzFramework::ProcessLauncher::ProcessLaunchInfo processLaunchInfo; processLaunchInfo.m_commandlineParameters = AZStd::string::format("\"%s\" %s", executableAbsolutePath.c_str(), parameters.c_str()); processLaunchInfo.m_showWindow = true; processLaunchInfo.m_processPriority = AzFramework::ProcessPriority::PROCESSPRIORITY_NORMAL; { AZStd::string contextKey = toolNameForLog + AZStd::string(" Input File"); AZ_TraceContext(contextKey, shaderSourcePathForDebug); } { AZStd::string contextKey = toolNameForLog + AZStd::string(" Command Line"); AZ_TraceContext(contextKey, processLaunchInfo.m_commandlineParameters); } AZ_TracePrintf(ShaderPlatformInterfaceName, "Executing '%s' ...", processLaunchInfo.m_commandlineParameters.c_str()); AzFramework::ProcessWatcher* watcher = AzFramework::ProcessWatcher::LaunchProcess(processLaunchInfo, AzFramework::COMMUNICATOR_TYPE_STDINOUT); if (!watcher) { AZ_Error(ShaderPlatformInterfaceName, false, "Shader compiler could not be launched"); return false; } AZStd::unique_ptr watcherPtr = AZStd::unique_ptr(watcher); AZStd::string errorMessages; auto pumpOuputStreams = [&watcherPtr, &errorMessages]() { auto communicator = watcherPtr->GetCommunicator(); // Instead of collecting all the output in a giant string, it would be better to report // the chunks of messages as they arrive, but this should be good enough for now. if (auto byteCount = communicator->PeekError()) { AZStd::string chunk; chunk.resize(byteCount); communicator->ReadError(chunk.data(), byteCount); errorMessages += chunk; } // Even though we don't use the stdout stream, we have to read it or the process will hang if (auto byteCount = communicator->PeekOutput()) { AZStd::string chunk; chunk.resize(byteCount); communicator->ReadOutput(chunk.data(), byteCount); } }; uint32_t exitCode = 0; bool timedOut = false; const AZStd::sys_time_t maxWaitTimeSeconds = 120; const AZStd::sys_time_t startTimeSeconds = AZStd::GetTimeNowSecond(); const AZStd::sys_time_t startTime = AZStd::GetTimeNowTicks(); while (watcherPtr->IsProcessRunning(&exitCode)) { const AZStd::sys_time_t currentTimeSeconds = AZStd::GetTimeNowSecond(); if (currentTimeSeconds - startTimeSeconds > maxWaitTimeSeconds) { timedOut = true; static const uint32_t TimeOutExitCode = 121; exitCode = TimeOutExitCode; watcherPtr->TerminateProcess(TimeOutExitCode); break; } else { pumpOuputStreams(); } } AZ_Assert(!watcherPtr->IsProcessRunning(), "Shader compiler execution failed to terminate"); // Pump one last time to make sure the streams have been flushed pumpOuputStreams(); const bool reportedErrors = ReportErrorMessages(toolNameForLog, errorMessages); if (timedOut) { AZ_Error(ShaderPlatformInterfaceName, false, "%s execution timed out after %d seconds", toolNameForLog, maxWaitTimeSeconds); return false; } if (exitCode != 0) { AZ_Error(ShaderPlatformInterfaceName, false, "%s has exited with error code %d", toolNameForLog, exitCode); return false; } if (reportedErrors) { AZ_Error(ShaderPlatformInterfaceName, false, "%s returned successfully, but errors were detected.", toolNameForLog); return false; } // Write the shader compiler profiling data in distinct JSON files. // We cannot use a shared data structure because distinct builder processes are running in parallel. const AZStd::sys_time_t endTime = AZStd::GetTimeNowTicks(); const AZStd::sys_time_t deltaTime = endTime - startTime; const float elapsedTimeSeconds = (float)(deltaTime) / (float)AZStd::GetTimeTicksPerSecond(); AZ_TracePrintf(ShaderPlatformInterfaceName, "%s elapsedTimeMillis: %d", __FUNCTION__, aznumeric_cast(elapsedTimeSeconds * 1000.0f)); ShaderCompilerProfiling::Entry profilingEntry; profilingEntry.m_executablePath = executablePath; profilingEntry.m_parameters = parameters; profilingEntry.m_elapsedTimeSeconds = elapsedTimeSeconds; WriteProfilingEntryToLog(shaderSourcePathForDebug, profilingEntry); return true; } bool ReportErrorMessages([[maybe_unused]] AZStd::string_view window, AZStd::string_view errorMessages) { // There are more efficient ways to do this, but this approach is simple and gets us moving for now. AZStd::vector lines; AzFramework::StringFunc::Tokenize(errorMessages.data(), lines, "\n\r"); bool foundErrors = false; for (auto& line : lines) { if (AZStd::string::npos != AzFramework::StringFunc::Find(line, "error")) { AZ_Error(window.data(), false, "%s", line.data()); foundErrors = true; } else if (AZStd::string::npos != AzFramework::StringFunc::Find(line, "warning")) { AZ_Warning(window.data(), false, "%s", line.data()); } else { AZ_TracePrintf(window.data(), "%s", line.data()); } } return foundErrors; } ShaderStage ToRHIShaderStage(ShaderHardwareStage stageType) { switch (stageType) { case ShaderHardwareStage::Compute: return RHI::ShaderStage::Compute; case ShaderHardwareStage::Fragment: return RHI::ShaderStage::Fragment; case ShaderHardwareStage::Geometry: AZ_Assert(false, "RHI currently does not support geometry shaders"); return RHI::ShaderStage::Unknown; case ShaderHardwareStage::TessellationControl: case ShaderHardwareStage::TessellationEvaluation: return RHI::ShaderStage::Tessellation; case ShaderHardwareStage::Vertex: return RHI::ShaderStage::Vertex; case ShaderHardwareStage::RayTracing: return RHI::ShaderStage::RayTracing; } AZ_Assert(false, "Unable to find RHI Shader stage given RPI ShaderStageType %d", stageType); return RHI::ShaderStage::Unknown; } Outcome LoadFileString(const char* path) { AZ::IO::FileIOStream fileStream(path, AZ::IO::OpenMode::ModeRead); if (!fileStream.IsOpen()) { return AZ::Failure(AZStd::string::format("Could not open file '%s'.", path)); } AZStd::string text; text.resize(fileStream.GetLength()); auto bytesRead = fileStream.Read(text.size(), text.data()); if (bytesRead != fileStream.GetLength()) { return AZ::Failure(AZStd::string::format("Failed to load file '%s'.", path)); } return AZ::Success(AZStd::move(text)); } Outcome, AZStd::string> LoadFileBytes(const char* path) { AZ::IO::FileIOStream fileStream(path, AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeBinary); if (!fileStream.IsOpen()) { return AZ::Failure(AZStd::string::format("Could not open file '%s'.", path)); } AZStd::vector bytes; bytes.resize(fileStream.GetLength()); auto bytesRead = fileStream.Read(bytes.size(), bytes.data()); if (bytesRead != fileStream.GetLength()) { return AZ::Failure(AZStd::string::format("Failed to load file '%s'.", path)); } return AZ::Success(AZStd::move(bytes)); } size_t RegexCount(AZStd::string_view text, const char* regex) { size_t count = 0; AZStd::regex expression(regex); // Note this could be done much more simply using sregex_iterator, but AZStd::regex_iterator appears to be broken and doesn't compile. (LY-116293) AZStd::match_results results; AZStd::string::const_iterator searchFrom = text.begin(); do { if (AZStd::regex_search(searchFrom, text.end(), results, expression)) { count++; searchFrom = results[0].second; } else { searchFrom = text.end(); } } while (searchFrom != text.end()); return count; } AZStd::string BuildFileNameWithExtension(const AZStd::string& shaderSourceFile, const AZStd::string& tempFolder, const char* outputExtension) { AZStd::string outputFile; AzFramework::StringFunc::Path::GetFileName(shaderSourceFile.c_str(), outputFile); AzFramework::StringFunc::Path::Join(tempFolder.c_str(), outputFile.c_str(), outputFile); AzFramework::StringFunc::Path::ReplaceExtension(outputFile, outputExtension); return outputFile; } } // namespace RHI } // namespace AZ