Initial commit

Code/Framework/AzFramework/AzFramework/IO/FileOperations.cpp

@@ -0,0 +1,299 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#include <cstdarg>
+#include <stdio.h>
+#include <AzCore/IO/FileIO.h>
+#include "FileOperations.h"
+#include <AzFramework/StringFunc/StringFunc.h>
+
+namespace AZ
+{
+    namespace IO
+    {
+        int64_t Print(HandleType fileHandle, const char* format, ...)
+        {
+            va_list arglist;
+            va_start(arglist, format);
+            int64_t result = PrintV(fileHandle, format, arglist);
+            va_end(arglist);
+            return result;
+        }
+
+        int64_t PrintV(HandleType fileHandle, const char* format, va_list arglist)
+        {
+            const int bufferSize = 1024;
+            char buffer[bufferSize];
+
+            FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
+            AZ_Assert(fileIO, "AZ::IO::PrintV: No FileIO instance");
+
+            int printCount = azvsnprintf(buffer, bufferSize, format, arglist);
+
+            if (printCount > 0)
+            {
+                AZ_Assert(printCount < bufferSize, "AZ::IO::PrintV: required buffer size for vsnprintf is larger than working buffer");
+                if (!fileIO->Write(fileHandle, buffer, printCount))
+                {
+                    return -1;
+                }
+                return printCount;
+            }
+
+            return printCount;
+        }
+
+        Result Move(const char* sourceFilePath, const char* destinationFilePath)
+        {
+            FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
+            AZ_Assert(fileIO, "AZ::IO::Move: No FileIO instance");
+
+            if (!fileIO->Copy(sourceFilePath, destinationFilePath))
+            {
+                return ResultCode::Error;
+            }
+
+            if (!fileIO->Remove(sourceFilePath))
+            {
+                return ResultCode::Error;
+            }
+
+            return ResultCode::Success;
+        }
+
+
+
+        AZ::IO::Result SmartMove(const char* sourceFilePath, const char* destinationFilePath)
+        {
+            FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
+            AZ_Assert(fileIO, "AZ::IO::SmartMove: No FileIO instance");
+            if (!fileIO->Exists(sourceFilePath))
+            {
+                AZ_Warning("AZ::IO::SmartMove", false, "Source file does not exist (%s)", sourceFilePath);
+                return ResultCode::Error;
+            }
+            AZStd::string tmpDestFile;
+            bool destFileMoved = false;
+            if (fileIO->Exists(destinationFilePath))
+            {
+                if (CreateTempFileName(destinationFilePath, tmpDestFile))
+                {
+                    if (!fileIO->Rename(destinationFilePath, tmpDestFile.c_str()))
+                    {
+                        //if the rename fails try deleting the destination file
+                        if (!fileIO->Remove(destinationFilePath))
+                        {
+                            AZ_Warning("AZ::IO::SmartMove", false, "Unable to move/delete the destination file (%s)", destinationFilePath);
+                            return ResultCode::Error;
+                        }
+                    }
+                    else
+                    {
+                        destFileMoved = true;
+                    }
+                }
+                else
+                {
+                    if (!fileIO->Remove(destinationFilePath))
+                    {
+                        AZ_Warning("AZ::IO::SmartMove", false, "Unable to remove the destination file (%s)", destinationFilePath);
+                        return ResultCode::Error;
+                    }
+                }
+
+                //Now try renaming the source file with the destination file
+                if (!fileIO->Rename(sourceFilePath, destinationFilePath))
+                {
+                    // if the move fails, try copying instead
+                    if (!fileIO->Copy(sourceFilePath, destinationFilePath))
+                    {
+                        AZ_Warning("AZ::IO::SmartMove", false, "Unable to move/copy the source file (%s)", sourceFilePath);
+                        if (destFileMoved)
+                        {
+                            // if we were unable to move/copy the source file to the dest file, 
+                            // we will try to revert back the destination file from the temp file.
+                            if (!fileIO->Rename(tmpDestFile.c_str(), destinationFilePath))
+                            {
+                                AZ_Warning("AZ::IO::SmartMove", false, "Unable to rename back the destination file (%s)", destinationFilePath);
+                            }
+                        }
+
+                        return ResultCode::Error;
+                    }
+                    // removing the source file if copy succeeds 
+                    if (!fileIO->Remove(sourceFilePath))
+                    {
+                        AZ_Warning("AZ::IO::SmartMove", false, "Unable to delete the source file (%s)", sourceFilePath);
+                    }
+                }
+
+                if (destFileMoved)
+                {
+                    // Remove the temp file the destination file was moved to
+                    if (!fileIO->Remove(tmpDestFile.c_str()))
+                    {
+                        AZ_Warning("AZ::IO::SmartMove", false, "Unable to move/delete the destination file (%s)", destinationFilePath);
+                        return ResultCode::Error;
+                    }
+                }
+                
+                return ResultCode::Success;
+
+            }
+            else
+            {
+                return Move(sourceFilePath, destinationFilePath);
+            }
+
+        }
+
+        bool CreateTempFileName(const char* file, AZStd::string& tempFile)
+        {
+            FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
+            AZ_Assert(fileIO, "AZ::IO::CreateTempFileName: No FileIO instance");
+            const int s_MaxCreateTempFileTries = 16;
+            AZStd::string fullPath, fileName;
+            tempFile.clear();
+            
+            if (!AzFramework::StringFunc::Path::GetFullPath(file, fullPath))
+            {
+                AZ_Warning("AZ::IO::CreateTempFileName", false, " Filepath needs to be an absolute path: '%s'", file);
+                return false;
+            }
+
+            if (!AzFramework::StringFunc::Path::GetFullFileName(file, fileName))
+            {
+                AZ_Warning("AZ::IO::CreateTempFileName", false, " Filepath needs to be an absolute path: '%s'", file);
+                return false;
+            }
+           
+            for (int idx = 0; idx < s_MaxCreateTempFileTries; idx++)
+            {
+                AzFramework::StringFunc::Path::ConstructFull(fullPath.c_str(), AZStd::string::format("$tmp%d_%s", rand(), fileName.c_str()).c_str(), tempFile, true);
+                //Ensure that the file does not exist
+                if (!fileIO->Exists(tempFile.c_str()))
+                {
+                    return true;
+                }
+            }
+            AZ_Warning("AZ::IO::CreateTempFileName", false, "Unable to create temp file for (%s). Please check the folder and clear any temp files.", file);
+            tempFile.clear();
+            return false;
+        }
+
+        int GetC(HandleType fileHandle)
+        {
+            FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
+            AZ_Assert(fileIO, "AZ::IO::GetC: No FileIO instance");
+            char character;
+            if (!fileIO->Read(fileHandle, &character, 1, true))
+            {
+                return EOF;
+            }
+            return static_cast<int>(character);
+        }
+
+        int UnGetC(int character, HandleType fileHandle)
+        {
+            FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
+            AZ_Assert(fileIO, "AZ::IO::UnGetC: No FileIO instance");
+
+            // EOF does nothing
+            if (character == EOF)
+            {
+                return EOF;
+            }
+            char characterToRestore = static_cast<char>(character);
+            // Just move back a byte and write it in
+            // Still not identical behavior since ungetc normally gets lost if you seek, but it works for our case
+            if (!fileIO->Seek(fileHandle, -1, SeekType::SeekFromCurrent))
+            {
+                return EOF;
+            }
+
+            if (!fileIO->Write(fileHandle, &characterToRestore, sizeof(characterToRestore)))
+            {
+                return EOF;
+            }
+            return character;
+        }
+
+        char* FGetS(char* buffer, uint64_t bufferSize, HandleType fileHandle)
+        {
+            AZ_Assert(buffer && bufferSize, "AZ::IO::FGetS: Invalid buffer supplied");
+
+            FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
+            AZ_Assert(fileIO, "AZ::IO::FGetS: No FileIO instance");
+
+            memset(buffer, 0, bufferSize);
+
+            AZ::u64 bytesRead = 0;
+            fileIO->Read(fileHandle, buffer, bufferSize - 1, false, &bytesRead);
+            if (!bytesRead)
+            {
+                return 0;
+            }
+
+            char* currentPosition = buffer;
+            int64_t len = 0;
+
+            bool done = false;
+            int64_t slashRPosition = -1;
+            do
+            {
+                if (*currentPosition != '\r' && *currentPosition != '\n' && static_cast<uint64_t>(len) < bytesRead - 1)
+                {
+                    len++;
+                    currentPosition++;
+                }
+                else
+                {
+                    done = true;
+                    if (*currentPosition == '\r')
+                    {
+                        slashRPosition = len;
+                    }
+                }
+            } while (!done);
+
+            // null terminate string
+            buffer[len] = '\0';
+
+            //////////////////////////////////////////
+            //seek back to the end of the string
+            AZ::s64 seekback = bytesRead - len - 1;
+
+            // handle CR/LF for file coming from different platforms
+            if (slashRPosition > -1 && bytesRead > static_cast<uint64_t>(slashRPosition) && buffer[slashRPosition + 1] == '\n')
+            {
+                seekback--;
+            }
+            fileIO->Seek(fileHandle, -seekback, AZ::IO::SeekType::SeekFromCurrent);
+            ///////////////////////////////////////////
+
+            return buffer;
+        }
+
+        int FPutS(const char* buffer, HandleType fileHandle)
+        {
+            FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
+            AZ_Assert(fileIO, "AZ::IO::FPutS: No FileIO instance");
+
+            uint64_t writeAmount = strlen(buffer);
+            uint64_t bytesWritten = 0;
+            if (!fileIO->Write(fileHandle, buffer, writeAmount))
+            {
+                return EOF;
+            }
+            return static_cast<int>(bytesWritten);
+        }
+    } // namespace IO
+} // namespace AZ

Code/Framework/AzFramework/AzFramework/IO/FileOperations.h

@@ -0,0 +1,44 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#pragma once
+#ifndef CRYCOMMON_FILEOPERATIONS_H
+#define CRYCOMMON_FILEOPERATIONS_H
+
+#include <AzCore/IO/FileIO.h>
+
+namespace AZ
+{
+    namespace IO
+    {
+        int64_t Print(HandleType fileHandle, const char* format, ...);
+        int64_t PrintV(HandleType fileHandle, const char* format, va_list arglist);
+        AZ::IO::Result Move(const char* sourceFilePath, const char* destinationFilePath);
+        /**
+        * This function only tries to move/copy the sourceFile to the destination file
+        * This will only return success if it was able to move/copy the sourceFile to the destination file.
+        **/
+        AZ::IO::Result SmartMove(const char* sourceFile, const char* destinationFile);
+        /**
+        * Generate an unused filename from an input file name
+        * by prepending "$tmpN_" to it (N is a random integer).
+        * SourceFile must be an absolute path, it can have alias in the path. 
+        * Returns false if unable to find an unused name despite multiple attempts.
+        **/
+        bool CreateTempFileName(const char* file, AZStd::string& tempFile);
+        int GetC(HandleType fileHandle);
+        int UnGetC(int character, HandleType fileHandle);
+        char* FGetS(char* buffer, uint64_t bufferSize, HandleType fileHandle);
+        int FPutS(const char* buffer, HandleType fileHandle);
+    }
+}
+
+#endif // #ifndef CRYCOMMON_FILEOPERATIONS_H

Code/Framework/AzFramework/AzFramework/IO/LocalFileIO.cpp

@@ -0,0 +1,801 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#include <AzFramework/IO/LocalFileIO.h>
+#include <sys/stat.h>
+#include <AzCore/IO/SystemFile.h>
+#include <AzCore/IO/IOUtils.h>
+#include <AzCore/IO/Path/Path.h>
+#include <AzCore/Casting/numeric_cast.h>
+#include <AzCore/Casting/lossy_cast.h>
+#include <AzCore/std/functional.h>
+#include <AzCore/std/string/conversions.h>
+#include <cctype>
+
+namespace AZ
+{
+    namespace IO
+    {
+        const HandleType LocalHandleStartValue = 1000000; //start the local file io handles at 1 million
+
+        LocalFileIO::LocalFileIO()
+        {
+            m_nextHandle = LocalHandleStartValue;
+        }
+
+        LocalFileIO::~LocalFileIO()
+        {
+            AZStd::lock_guard<AZStd::recursive_mutex> lock(m_openFileGuard);
+            while (!m_openFiles.empty())
+            {
+                const auto& handlePair = *m_openFiles.begin();
+                Close(handlePair.first);
+            }
+
+            AZ_Assert(m_openFiles.empty(), "Trying to shutdown filing system with files still open");
+        }
+
+        Result LocalFileIO::Open(const char* filePath, OpenMode mode, HandleType& fileHandle)
+        {
+            char resolvedPath[AZ_MAX_PATH_LEN];
+            ResolvePath(filePath, resolvedPath, AZ_MAX_PATH_LEN);
+
+            AZ::IO::UpdateOpenModeForReading(mode);
+
+            // Generate open modes for SystemFile
+            int systemFileMode = TranslateOpenModeToSystemFileMode(resolvedPath, mode);
+            bool write = AnyFlag(mode & (OpenMode::ModeWrite | OpenMode::ModeUpdate | OpenMode::ModeAppend));
+            if (write)
+            {
+                CheckInvalidWrite(resolvedPath);
+            }
+
+            {
+                AZStd::lock_guard<AZStd::recursive_mutex> lock(m_openFileGuard);
+
+                fileHandle = GetNextHandle();
+
+                // Construct a new SystemFile in the map (SystemFiles don't copy/move very well).
+                auto newPair = m_openFiles.emplace(fileHandle);
+                // Check for successful insert
+                if (!newPair.second)
+                {
+                    fileHandle = InvalidHandle;
+                    return ResultCode::Error;
+                }
+                // Attempt to open the newly created file
+                if (newPair.first->second.Open(resolvedPath, systemFileMode, 0))
+                {
+                    return ResultCode::Success;
+                }
+                else
+                {
+                    // Remove file, it's not actually open
+                    m_openFiles.erase(fileHandle);
+                    // On failure, ensure the fileHandle returned is invalid
+                    //  some code does not check return but handle value (equivalent to checking for nullptr FILE*)
+                    fileHandle = InvalidHandle;
+                    return ResultCode::Error;
+                }
+            }
+        }
+
+        Result LocalFileIO::Close(HandleType fileHandle)
+        {
+            auto filePointer = GetFilePointerFromHandle(fileHandle);
+            if (!filePointer)
+            {
+                return ResultCode::Error_HandleInvalid;
+            }
+
+            filePointer->Close();
+
+            {
+                AZStd::lock_guard<AZStd::recursive_mutex> lock(m_openFileGuard);
+                m_openFiles.erase(fileHandle);
+            }
+            return ResultCode::Success;
+        }
+
+        Result LocalFileIO::Read(HandleType fileHandle, void* buffer, AZ::u64 size, bool failOnFewerThanSizeBytesRead, AZ::u64* bytesRead)
+        {
+            auto filePointer = GetFilePointerFromHandle(fileHandle);
+            if (!filePointer)
+            {
+                return ResultCode::Error_HandleInvalid;
+            }
+
+            SystemFile::SizeType readResult = filePointer->Read(size, buffer);
+
+            if (bytesRead)
+            {
+                *bytesRead = aznumeric_cast<AZ::u64>(readResult);
+            }
+
+            if (static_cast<AZ::u64>(readResult) != size)
+            {
+                if (failOnFewerThanSizeBytesRead)
+                {
+                    return ResultCode::Error;
+                }
+
+                // Reading less than desired is valid if ferror is not set
+                AZ_Assert(Eof(fileHandle), "End of file unexpectedly reached before all data was read");
+            }
+
+            return ResultCode::Success;
+        }
+
+        Result LocalFileIO::Write(HandleType fileHandle, const void* buffer, AZ::u64 size, AZ::u64* bytesWritten)
+        {
+            auto filePointer = GetFilePointerFromHandle(fileHandle);
+            if (!filePointer)
+            {
+                return ResultCode::Error_HandleInvalid;
+            }
+
+            SystemFile::SizeType writeResult = filePointer->Write(buffer, size);
+
+            if (bytesWritten)
+            {
+                *bytesWritten = writeResult;
+            }
+
+            if (static_cast<AZ::u64>(writeResult) != size)
+            {
+                return ResultCode::Error;
+            }
+
+            return ResultCode::Success;
+        }
+
+        Result LocalFileIO::Flush(HandleType fileHandle)
+        {
+            auto filePointer = GetFilePointerFromHandle(fileHandle);
+            if (!filePointer)
+            {
+                return ResultCode::Error_HandleInvalid;
+            }
+
+            filePointer->Flush();
+
+            return ResultCode::Success;
+        }
+
+        Result LocalFileIO::Tell(HandleType fileHandle, AZ::u64& offset)
+        {
+            auto filePointer = GetFilePointerFromHandle(fileHandle);
+            if (!filePointer)
+            {
+                return ResultCode::Error_HandleInvalid;
+            }
+
+            SystemFile::SizeType resultValue = filePointer->Tell();
+            if (resultValue == -1)
+            {
+                return ResultCode::Error;
+            }
+
+            offset = static_cast<AZ::u64>(resultValue);
+            return ResultCode::Success;
+        }
+
+        Result LocalFileIO::Seek(HandleType fileHandle, AZ::s64 offset, SeekType type)
+        {
+            auto filePointer = GetFilePointerFromHandle(fileHandle);
+            if (!filePointer)
+            {
+                return ResultCode::Error_HandleInvalid;
+            }
+
+            SystemFile::SeekMode mode = SystemFile::SF_SEEK_BEGIN;
+            if (type == SeekType::SeekFromCurrent)
+            {
+                mode = SystemFile::SF_SEEK_CURRENT;
+            }
+            else if (type == SeekType::SeekFromEnd)
+            {
+                mode = SystemFile::SF_SEEK_END;
+            }
+
+            filePointer->Seek(offset, mode);
+
+            return ResultCode::Success;
+        }
+
+        Result LocalFileIO::Size(HandleType fileHandle, AZ::u64& size)
+        {
+            auto filePointer = GetFilePointerFromHandle(fileHandle);
+            if (!filePointer)
+            {
+                return ResultCode::Error_HandleInvalid;
+            }
+
+            size = filePointer->Length();
+            return ResultCode::Success;
+        }
+
+        Result LocalFileIO::Size(const char* filePath, AZ::u64& size)
+        {
+            char resolvedPath[AZ_MAX_PATH_LEN];
+            ResolvePath(filePath, resolvedPath, AZ_MAX_PATH_LEN);
+
+            size = SystemFile::Length(resolvedPath);
+            if (!size)
+            {
+                return SystemFile::Exists(resolvedPath) ? ResultCode::Success : ResultCode::Error;
+            }
+
+            return ResultCode::Success;
+        }
+
+        bool LocalFileIO::IsReadOnly(const char* filePath)
+        {
+            char resolvedPath[AZ_MAX_PATH_LEN];
+            ResolvePath(filePath, resolvedPath, AZ_MAX_PATH_LEN);
+
+            return !SystemFile::IsWritable(resolvedPath);
+        }
+
+        bool LocalFileIO::Eof(HandleType fileHandle)
+        {
+            auto filePointer = GetFilePointerFromHandle(fileHandle);
+            if (!filePointer)
+            {
+                return false;
+            }
+
+            return filePointer->Eof();
+        }
+
+        AZ::u64 LocalFileIO::ModificationTime(const char* filePath)
+        {
+            char resolvedPath[AZ_MAX_PATH_LEN];
+            ResolvePath(filePath, resolvedPath, AZ_MAX_PATH_LEN);
+
+            return SystemFile::ModificationTime(resolvedPath);
+        }
+
+        AZ::u64 LocalFileIO::ModificationTime(HandleType fileHandle)
+        {
+            auto filePointer = GetFilePointerFromHandle(fileHandle);
+            if (!filePointer)
+            {
+                return 0;
+            }
+
+            return filePointer->ModificationTime();
+        }
+
+        bool LocalFileIO::Exists(const char* filePath)
+        {
+            char resolvedPath[AZ_MAX_PATH_LEN];
+            ResolvePath(filePath, resolvedPath, AZ_MAX_PATH_LEN);
+
+            return SystemFile::Exists(resolvedPath);
+        }
+
+        void LocalFileIO::CheckInvalidWrite(const char* path)
+        {
+            (void)path;
+
+#if defined(AZ_ENABLE_TRACING)
+            const char* assetsAlias = GetAlias("@assets@");
+
+            if (((path) && (assetsAlias) && (azstrnicmp(path, assetsAlias, strlen(assetsAlias)) == 0)))
+            {
+                AZ_Error("FileIO", false, "You may not alter data inside the asset cache.  Please check the call stack and consider writing into the source asset folder instead.\n"
+                    "Attempted write location: %s", path);
+            }
+#endif
+        }
+
+        Result LocalFileIO::Remove(const char* filePath)
+        {
+            char resolvedPath[AZ_MAX_PATH_LEN];
+
+            ResolvePath(filePath, resolvedPath, AZ_MAX_PATH_LEN);
+
+            CheckInvalidWrite(resolvedPath);
+
+            if (IsDirectory(resolvedPath))
+            {
+                return ResultCode::Error;
+            }
+
+            return SystemFile::Delete(resolvedPath) ? ResultCode::Success : ResultCode::Error;
+        }
+
+        Result LocalFileIO::Rename(const char* originalFilePath, const char* newFilePath)
+        {
+            char resolvedOldPath[AZ_MAX_PATH_LEN];
+            char resolvedNewPath[AZ_MAX_PATH_LEN];
+
+            ResolvePath(originalFilePath, resolvedOldPath, AZ_MAX_PATH_LEN);
+            ResolvePath(newFilePath, resolvedNewPath, AZ_MAX_PATH_LEN);
+
+            CheckInvalidWrite(resolvedNewPath);
+
+            if (!SystemFile::Rename(resolvedOldPath, resolvedNewPath))
+            {
+                return ResultCode::Error;
+            }
+
+            return ResultCode::Success;
+        }
+
+        SystemFile* LocalFileIO::GetFilePointerFromHandle(HandleType fileHandle)
+        {
+            AZStd::lock_guard<AZStd::recursive_mutex> lock(m_openFileGuard);
+            auto openFileIterator = m_openFiles.find(fileHandle);
+            if (openFileIterator != m_openFiles.end())
+            {
+                SystemFile& file = openFileIterator->second;
+                return &file;
+            }
+            return nullptr;
+        }
+
+        HandleType LocalFileIO::GetNextHandle()
+        {
+            return m_nextHandle++;
+        }
+
+        static ResultCode DestroyPath_Recurse(LocalFileIO* fileIO, const char* filePath)
+        {
+            // this is a deltree command.  It needs to eat everything.  Even files.
+            ResultCode res = ResultCode::Success;
+
+            fileIO->FindFiles(filePath, "*", [&](const char* iterPath) -> bool
+                {
+                    // depth first recurse into directories!
+
+                    // note:  findFiles returns full path names.
+
+                    if (fileIO->IsDirectory(iterPath))
+                    {
+                        // recurse.
+                        if (DestroyPath_Recurse(fileIO, iterPath) != ResultCode::Success)
+                        {
+                            res = ResultCode::Error;
+                            return false; // stop the find files.
+                        }
+                    }
+                    else
+                    {
+                        // if its a file, remove it
+                        if (fileIO->Remove(iterPath) != ResultCode::Success)
+                        {
+                            res = ResultCode::Error;
+                            return false; // stop the find files.
+                        }
+                    }
+                    return true; // continue the find files
+                });
+
+            if (res != ResultCode::Success)
+            {
+                return res;
+            }
+
+            // now that we've finished recursing, rmdir on the folder itself
+            return AZ::IO::SystemFile::DeleteDir(filePath) ? ResultCode::Success : ResultCode::Error;
+        }
+
+        Result LocalFileIO::DestroyPath(const char* filePath)
+        {
+            char resolvedPath[AZ_MAX_PATH_LEN];
+            ResolvePath(filePath, resolvedPath, AZ_MAX_PATH_LEN);
+
+            bool pathExists = Exists(resolvedPath);
+            if (!pathExists)
+            {
+                return ResultCode::Success;
+            }
+
+            if (pathExists && (!IsDirectory(resolvedPath)))
+            {
+                return ResultCode::Error;
+            }
+
+            return DestroyPath_Recurse(this, resolvedPath);
+        }
+
+        static void ToUnixSlashes(char* path, AZ::u64 size)
+        {
+            auto PrevAndCurrentCharIsPathSeparator = [](const char prev, const char next) -> bool
+            {
+                constexpr AZStd::string_view pathSeparator = "/";
+                const bool prevIsPathSeparator = pathSeparator.find_first_of(prev) != AZStd::string_view::npos;
+                const bool nextIsPathSeparator = pathSeparator.find_first_of(next) != AZStd::string_view::npos;
+                return prevIsPathSeparator && nextIsPathSeparator;
+            };
+
+            size_t copyOffset = 0;
+            for (size_t i = 0; i < size && path[i] != '\0'; i++)
+            {
+                if (path[i] == '\\')
+                {
+                    path[i] = '/';
+                }
+
+                // Replace runs of path separators with one path separator
+#if AZ_TRAIT_USE_WINDOWS_FILE_API
+                // Network file systems for Windows based APIs start with consecutive path separators
+                // so skip over the first character in this case
+                constexpr size_t duplicateSeparatorStartOffet = 1;
+#else
+                constexpr size_t duplicateSeparatorStartOffet = 0;
+#endif
+                if (i > duplicateSeparatorStartOffet)
+                {
+                    if (PrevAndCurrentCharIsPathSeparator(path[i - 1], path[i]))
+                    {
+                        continue;
+                    }
+                }
+
+                // If the copy offset is different than the iteration index of the path, then copy over it over
+                if (copyOffset != i)
+                {
+                    path[copyOffset] = path[i];
+                }
+
+                ++copyOffset;
+            }
+
+            // Null-terminate the path again in case duplicate slashes were collapsed
+            path[copyOffset] = '\0';
+
+        }
+
+        bool LocalFileIO::ResolvePath(const char* path, char* resolvedPath, AZ::u64 resolvedPathSize) const
+        {
+            if (resolvedPath == nullptr || resolvedPathSize == 0)
+            {
+                return false;
+            }
+
+            resolvedPath[0] = '\0';
+
+            if (path == nullptr)
+            {
+                return false;
+            }
+
+            if (IsAbsolutePath(path))
+            {
+                size_t pathLen = strlen(path);
+                if (pathLen + 1 < resolvedPathSize)
+                {
+                    azstrncpy(resolvedPath, resolvedPathSize, path, pathLen + 1);
+
+                    //see if the absolute path uses @assets@ or @root@, if it does lowercase the relative part
+                    if (!LowerIfBeginsWith(resolvedPath, resolvedPathSize, GetAlias("@assets@")))
+                    {
+                        LowerIfBeginsWith(resolvedPath, resolvedPathSize, GetAlias("@root@"));
+                    }
+
+                    ToUnixSlashes(resolvedPath, resolvedPathSize);
+                    return true;
+                }
+                else
+                {
+                    return false;
+                }
+            }
+
+            char rootedPathBuffer[AZ_MAX_PATH_LEN] = {0};
+            const char* rootedPath = path;
+            // if the path does not begin with an alias, then it is assumed to begin with @assets@
+            if (path[0] != '@')
+            {
+                if (GetAlias("@assets@"))
+                {
+                    const int rootLength = 9;// strlen("@assets@/")
+                    azstrncpy(rootedPathBuffer, AZ_MAX_PATH_LEN, "@assets@/", rootLength);
+                    size_t pathLen = strlen(path);
+                    size_t rootedPathBufferlength = rootLength + pathLen + 1;// +1 for null terminator
+                    if (rootedPathBufferlength > resolvedPathSize)
+                    {
+                        AZ_Assert(rootedPathBufferlength < resolvedPathSize, "Constructed path length is wrong:%s", rootedPathBuffer);//path constructed is wrong
+                        size_t remainingSize = resolvedPathSize - rootLength - 1;// - 1 for null terminator
+                        azstrncpy(rootedPathBuffer + rootLength, AZ_MAX_PATH_LEN, path, remainingSize);
+                        rootedPathBuffer[resolvedPathSize - 1] = '\0';
+                    }
+                    else
+                    {
+                        azstrncpy(rootedPathBuffer + rootLength, AZ_MAX_PATH_LEN - rootLength, path, pathLen + 1);
+                    }
+                }
+                else
+                {
+                    ConvertToAbsolutePath(path, rootedPathBuffer, AZ_MAX_PATH_LEN);
+                }
+                rootedPath = rootedPathBuffer;
+            }
+
+            if (ResolveAliases(rootedPath, resolvedPath, resolvedPathSize))
+            {
+                ToUnixSlashes(resolvedPath, resolvedPathSize);
+                return true;
+            }
+
+            return false;
+        }
+
+        bool LocalFileIO::ResolvePath(AZ::IO::FixedMaxPath& resolvedPath, const AZ::IO::PathView& path) const
+        {
+            if (AZ::IO::FixedMaxPathString fixedPath{ path.Native() };
+                ResolvePath(fixedPath.c_str(), resolvedPath.Native().data(), resolvedPath.Native().capacity()))
+            {
+                // Update the null-terminator offset
+                resolvedPath.Native().resize_no_construct(AZStd::char_traits<char>::length(resolvedPath.Native().data()));
+                return true;
+            }
+
+            return false;
+        }
+
+        void LocalFileIO::SetAlias(const char* key, const char* path)
+        {
+            char fullPath[AZ_MAX_PATH_LEN];
+            ConvertToAbsolutePath(path, fullPath, AZ_MAX_PATH_LEN);
+
+            const auto it = AZStd::find_if(m_aliases.begin(), m_aliases.end(), [key](const AliasType& alias)
+            {
+                return alias.first.compare(key) == 0;
+            });
+
+            if (it != m_aliases.end())
+            {
+                it->second = fullPath;
+            }
+            else
+            {
+                m_aliases.emplace_back(key, fullPath);
+            }
+        }
+
+        const char* LocalFileIO::GetAlias(const char* key) const
+        {
+            const auto it = AZStd::find_if(m_aliases.begin(), m_aliases.end(), [key](const AliasType& alias)
+                    {
+                        return alias.first.compare(key) == 0;
+                    });
+
+            if (it != m_aliases.end())
+            {
+                return it->second.c_str();
+            }
+            return nullptr;
+        }
+
+        void LocalFileIO::ClearAlias(const char* key)
+        {
+            m_aliases.erase(AZStd::remove_if(m_aliases.begin(), m_aliases.end(), [key](const AliasType& alias)
+                    {
+                        return alias.first.compare(key) == 0;
+                    }), m_aliases.end());
+        }
+
+        AZStd::optional<AZ::u64> LocalFileIO::ConvertToAliasBuffer(char* outBuffer, AZ::u64 outBufferLength, AZStd::string_view inBuffer) const
+        {
+            size_t longestMatch = 0;
+            size_t bufStringLength = inBuffer.size();
+            AZStd::string_view longestAlias;
+
+            for (const auto& [alias, resolvedAlias] : m_aliases)
+            {
+                // here we are making sure that the buffer being passed in has enough space to include the alias in it.
+                // we are trying to find the LONGEST match, meaning of the following two examples, the second should 'win'
+                // File:  g:/lumberyard/dev/files/morefiles/blah.xml
+                // Alias1 links to 'g:/lumberyard/dev/'
+                // Alias2 links to 'g:/lumberyard/dev/files/morefiles'
+                // so returning Alias2 is preferred as it is more specific, even though alias1 includes it.
+                // note that its not possible for this to be matched if the string is shorter than the length of the alias itself so we skip
+                // strings that are shorter than the alias's mapped path without checking.
+                if ((longestMatch == 0) || (resolvedAlias.size() > longestMatch) && (resolvedAlias.size() <= bufStringLength))
+                {
+                    // custom strcmp that ignores slash directions
+                    constexpr AZStd::string_view pathSeparators{ "/\\" };
+                    bool allMatch = AZStd::equal(resolvedAlias.begin(), resolvedAlias.end(), inBuffer.begin(),
+                        [&pathSeparators](const char lhs, const char rhs)
+                    {
+                        const bool lhsIsSeparator = pathSeparators.find_first_of(lhs) != AZStd::string_view::npos;
+                        const bool rhsIsSeparator = pathSeparators.find_first_of(lhs) != AZStd::string_view::npos;
+                        return (lhsIsSeparator && rhsIsSeparator) || tolower(lhs) == tolower(rhs);
+                    });
+
+                    if (allMatch)
+                    {
+                        // Either the resolvedAlias path must match the path exactly or the path must have a path separator character
+                        // right after the resolved alias
+                        if (const size_t matchLen = resolvedAlias.size();
+                            matchLen == bufStringLength || (pathSeparators.find_first_of(inBuffer[matchLen]) != AZStd::string_view::npos))
+                        {
+                            longestMatch = matchLen;
+                            longestAlias = alias;
+                        }
+                    }
+                }
+            }
+            if (!longestAlias.empty())
+            {
+                // rearrange the buffer to have
+                // [alias][old path]
+                size_t aliasSize = longestAlias.size();
+                size_t charsToAbsorb = longestMatch;
+                size_t remainingData = bufStringLength - charsToAbsorb;
+                size_t finalStringSize = aliasSize + remainingData;
+                if (finalStringSize >= outBufferLength)
+                {
+                    AZ_Error("FileIO", false, "Alias %.*s cannot fit in output buffer. The output buffer is too small (max len %lu, actual len %zu)",
+                        aznumeric_cast<int>(longestAlias.size()), longestAlias.data(), outBufferLength, finalStringSize);
+                    // avoid buffer overflow, return original untouched
+                    return AZStd::nullopt;
+                }
+
+                // move the remaining piece of the string:
+                memmove(outBuffer + aliasSize, inBuffer.data() + charsToAbsorb, remainingData);
+                // copy the alias
+                memcpy(outBuffer, longestAlias.data(), aliasSize);
+                /// add a null
+                outBuffer[finalStringSize] = 0;
+                return finalStringSize;
+            }
+
+            // If the input and output buffer are different, copy over the input buffer to the output buffer
+            if (outBuffer != inBuffer.data())
+            {
+                if (inBuffer.size() >= outBufferLength)
+                {
+                    AZ_Error("FileIO", false, R"(Path "%.*s" cannot fit in output buffer. The output buffer is too small (max len %llu, actual len %zu))",
+                        aznumeric_cast<int>(inBuffer.size()), inBuffer.data(), outBufferLength, inBuffer.size());
+                    return AZStd::nullopt;
+                }
+                size_t finalStringSize = inBuffer.copy(outBuffer, outBufferLength);
+                outBuffer[finalStringSize] = 0;
+            }
+            return bufStringLength;
+        }
+
+        AZStd::optional<AZ::u64> LocalFileIO::ConvertToAlias(char* inOutBuffer, AZ::u64 bufferLength) const
+        {
+            return ConvertToAliasBuffer(inOutBuffer, bufferLength, inOutBuffer);
+        }
+
+        bool LocalFileIO::ConvertToAlias(AZ::IO::FixedMaxPath& convertedPath, const AZ::IO::PathView& pathView) const
+        {
+            AZStd::optional<AZ::u64> convertedPathSize =
+                ConvertToAliasBuffer(convertedPath.Native().data(), convertedPath.Native().capacity(), pathView.Native());
+            if (convertedPathSize)
+            {
+                // Force update of AZStd::fixed_string m_size member to set correct null-terminator offset
+                convertedPath.Native().resize_no_construct(*convertedPathSize);
+                return true;
+            }
+
+            return false;
+        }
+
+        bool LocalFileIO::ResolveAliases(const char* path, char* resolvedPath, AZ::u64 resolvedPathSize) const
+        {
+            AZ_Assert(path != resolvedPath && resolvedPathSize > strlen(path), "Resolved path is incorrect");
+            AZ_Assert(path && path[0] != '%', "%% is deprecated, @ is the only valid alias token");
+
+            // we assert above, but we also need to properly handle the case when the resolvedPath buffer size
+            // is too small to copy the source into.
+            size_t pathLen = strlen(path) + 1; // account for null
+            if (path == resolvedPath || (resolvedPathSize < pathLen))
+            {
+                return false;
+            }
+
+            azstrncpy(resolvedPath, resolvedPathSize, path, pathLen);
+            for (const auto& alias : m_aliases)
+            {
+                const char* key = alias.first.c_str();
+                size_t keyLen = alias.first.length();
+                if (azstrnicmp(resolvedPath, key, keyLen) == 0) // we only support aliases at the front of the path
+                {
+                    if(azstrnicmp(key, "@assets@", 8) == 0 || azstrnicmp(key, "@root@", 6) == 0)
+                    {
+                        AZStd::to_lower(resolvedPath, resolvedPath + resolvedPathSize);
+                    }
+
+                    const char* dest = alias.second.c_str();
+                    size_t destLen = alias.second.length();
+                    char* afterKey = resolvedPath + keyLen;
+                    size_t afterKeyLen = pathLen - keyLen;
+                    // must ensure that we are replacing the entire folder name, not a partial (e.g. @GAME01@/ vs @GAME0@/)
+                    if (*afterKey == '/' || *afterKey == '\\' || *afterKey == 0)
+                    {
+                        if (afterKeyLen + destLen + 1 < resolvedPathSize)//if after replacing the alias the length is greater than the max path size than skip
+                        {
+                            // scoot the right hand side of the replacement over to make room
+                            memmove(resolvedPath + destLen, afterKey, afterKeyLen + 1); // make sure null is copied
+                            memcpy(resolvedPath, dest, destLen); // insert replacement
+                            pathLen -= keyLen;
+                            pathLen += destLen;
+
+                            AZStd::replace(resolvedPath, resolvedPath + resolvedPathSize, '\\', '/');
+                            return true;
+                        }
+                    }
+                }
+            }
+
+            // warn on failing to resolve an alias
+            AZ_Warning(
+                "LocalFileIO::ResolveAlias", path && path[0] != '@',
+                "Failed to resolve an alias: %s", path ? path : "(null)");
+
+            return false;
+        }
+
+        bool LocalFileIO::GetFilename(HandleType fileHandle, char* filename, AZ::u64 filenameSize) const
+        {
+            AZStd::lock_guard<AZStd::recursive_mutex> lock(m_openFileGuard);
+            auto fileIt = m_openFiles.find(fileHandle);
+            if (fileIt != m_openFiles.end())
+            {
+                azstrncpy(filename, filenameSize, fileIt->second.Name(), filenameSize);
+                return true;
+            }
+            return false;
+        }
+
+        bool LocalFileIO::LowerIfBeginsWith(char* inOutBuffer, AZ::u64 bufferLen, const char* alias) const
+        {
+            if (alias)
+            {
+                AZ::u64 aliasLen = azlossy_caster(strlen(alias));
+                if (azstrnicmp(inOutBuffer, alias, aliasLen) == 0)
+                {
+                    for (AZ::u64 i = aliasLen; i < bufferLen && inOutBuffer[i] != '\0'; ++i)
+                    {
+                        inOutBuffer[i] = static_cast<char>(std::tolower(static_cast<int>(inOutBuffer[i])));
+                    }
+
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        AZ::OSString LocalFileIO::RemoveTrailingSlash(const AZ::OSString& pathStr)
+        {
+            if (pathStr.empty() || (pathStr[pathStr.length() - 1] != '/' && pathStr[pathStr.length() - 1] != '\\'))
+            {
+                return pathStr;
+            }
+
+            return pathStr.substr(0, pathStr.length() - 1);
+        }
+
+        AZ::OSString LocalFileIO::CheckForTrailingSlash(const AZ::OSString& pathStr)
+        {
+            if (pathStr.empty() || pathStr[pathStr.length() - 1] == '/')
+            {
+                return pathStr;
+            }
+
+            if (pathStr[pathStr.length() - 1] == '\\')
+            {
+                return pathStr.substr(0, pathStr.length() - 1) + "/";
+            }
+
+            return pathStr + "/";
+        }
+    } // namespace IO
+} // namespace AZ

Code/Framework/AzFramework/AzFramework/IO/LocalFileIO.h

@@ -0,0 +1,108 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#pragma once
+
+#include <AzCore/base.h>
+#include <AzCore/std/containers/vector.h>
+#include <AzCore/std/containers/map.h>
+#include <AzCore/std/utils.h>
+#include <AzCore/std/string/osstring.h>
+#include <AzCore/std/parallel/mutex.h>
+#include <AzCore/std/parallel/atomic.h>
+#include <AzCore/std/parallel/lock.h>
+#include <AzCore/IO/FileIO.h>
+#include <AzCore/Memory/OSAllocator.h>
+#include <AzCore/RTTI/RTTI.h>
+
+// This header file and CPP handles the platform specific implementation of code as defined by the FileIOBase interface class.
+// In order to make your code portable and functional with both this and the RemoteFileIO class, use the interface to access
+// the methods.
+
+namespace AZ
+{
+    namespace IO
+    {
+        class SystemFile;
+        class LocalFileIO
+            : public FileIOBase
+        {
+        public:
+            AZ_RTTI(LocalFileIO, "{87A8D32B-F695-4105-9A4D-D99BE15DFD50}", FileIOBase);
+            AZ_CLASS_ALLOCATOR(LocalFileIO, OSAllocator, 0);
+
+            LocalFileIO();
+            ~LocalFileIO();
+
+            Result Open(const char* filePath, OpenMode mode, HandleType& fileHandle) override;
+            Result Close(HandleType fileHandle) override;
+            Result Tell(HandleType fileHandle, AZ::u64& offset) override;
+            Result Seek(HandleType fileHandle, AZ::s64 offset, SeekType type) override;
+            Result Size(HandleType fileHandle, AZ::u64& size) override;
+            Result Read(HandleType fileHandle, void* buffer, AZ::u64 size, bool failOnFewerThanSizeBytesRead = false, AZ::u64* bytesRead = nullptr) override;
+            Result Write(HandleType fileHandle, const void* buffer, AZ::u64 size, AZ::u64* bytesWritten = nullptr) override;
+            Result Flush(HandleType fileHandle) override;
+            bool Eof(HandleType fileHandle) override;
+            AZ::u64 ModificationTime(HandleType fileHandle) override;
+
+            bool Exists(const char* filePath) override;
+            Result Size(const char* filePath, AZ::u64& size) override;
+            AZ::u64 ModificationTime(const char* filePath) override;
+
+            bool IsDirectory(const char* filePath) override;
+            bool IsReadOnly(const char* filePath) override;
+            Result CreatePath(const char* filePath) override;
+            Result DestroyPath(const char* filePath) override;
+            Result Remove(const char* filePath) override;
+            Result Copy(const char* sourceFilePath, const char* destinationFilePath) override;
+            Result Rename(const char* originalFilePath, const char* newFilePath) override;
+            Result FindFiles(const char* filePath, const char* filter, FindFilesCallbackType callback) override;
+
+            void SetAlias(const char* alias, const char* path) override;
+            void ClearAlias(const char* alias) override;
+            const char* GetAlias(const char* alias) const override;
+            AZStd::optional<AZ::u64> ConvertToAlias(char* inOutBuffer, AZ::u64 bufferLength) const override;
+            bool ConvertToAlias(AZ::IO::FixedMaxPath& convertedPath, const AZ::IO::PathView& path) const override;
+            using FileIOBase::ConvertToAlias;
+            bool ResolvePath(const char* path, char* resolvedPath, AZ::u64 resolvedPathSize) const override;
+            bool ResolvePath(AZ::IO::FixedMaxPath& resolvedPath, const AZ::IO::PathView& path) const override;
+            using FileIOBase::ResolvePath;
+
+            bool GetFilename(HandleType fileHandle, char* filename, AZ::u64 filenameSize) const override;
+            bool ConvertToAbsolutePath(const char* path, char* absolutePath, AZ::u64 maxLength) const;
+            
+        private:
+            typedef AZStd::pair<AZ::OSString, AZ::OSString> AliasType;
+
+            SystemFile* GetFilePointerFromHandle(HandleType fileHandle);
+
+            HandleType GetNextHandle();
+
+            AZStd::optional<AZ::u64> ConvertToAliasBuffer(char* outBuffer, AZ::u64 outBufferLength, AZStd::string_view inBuffer) const;
+            bool ResolveAliases(const char* path, char* resolvedPath, AZ::u64 resolvedPathSize) const;
+            bool IsAbsolutePath(const char* path) const;
+
+            bool LowerIfBeginsWith(char* inOutBuffer, AZ::u64 bufferLen, const char* alias) const;
+
+        private:
+            static AZ::OSString RemoveTrailingSlash(const AZ::OSString& pathStr);
+            static AZ::OSString CheckForTrailingSlash(const AZ::OSString& pathStr);
+
+            mutable AZStd::recursive_mutex m_openFileGuard;
+            AZStd::atomic<HandleType> m_nextHandle;
+            AZStd::map<HandleType, SystemFile, AZStd::less<HandleType>, AZ::OSStdAllocator> m_openFiles;
+            AZStd::vector<AliasType, AZ::OSStdAllocator> m_aliases;
+
+            void CheckInvalidWrite(const char* path);
+        };
+    } // namespace IO
+} // namespace AZ
+

Code/Framework/AzFramework/AzFramework/IO/RemoteFileIO.h

@@ -0,0 +1,233 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#pragma once
+
+#include <AzCore/IO/FileIO.h>
+#include <AzCore/std/containers/unordered_map.h>
+#include <AzCore/std/parallel/atomic.h>
+#include <AzCore/std/parallel/mutex.h>
+#include <AzCore/std/string/string.h>
+#include <AzCore/std/string/osstring.h>
+#include <AzCore/Memory/OSAllocator.h>
+#include <AzCore/std/containers/vector.h>
+#include <AzCore/RTTI/RTTI.h>
+
+////////////////////////////////////////////////////////////////////////////////////
+//NetworkFileIO implements FileIOBase for serving all file system requests via the asset
+//processor connection. The asset processor on the other side of the connection uses LocalFileIO
+//to complete the task and sends the results back.
+//NetworkFileIO uses no caching at all.
+//RemoteFileIO derives from NetworkFileIO and adds caching to speed things up.
+
+//This option defines RemoteFileIO as NetworkFileIO, an easy way to test with no caching at all
+//#define REMOTEFILEIO_IS_NETWORKFILEIO
+
+//REMOTEFILEIO_CACHE_FILETREE is an option that turns on file metadata caching of the cache tree.
+//When on it queries the AP at startup for the state of the cache files and caches it. It also
+//registers itself as a listener for file changes and recaches on changes.
+//#define REMOTEFILEIO_CACHE_FILETREE
+
+//REMOTEFILEIO_CACHE_FILETREE_FALLBACK is an option you can turn on that in the event the instance
+//asks about file meta data that doesnt exist in the cache, we can fallback to NetworkFileIO and
+//see if something changed. For example if someone adds a directory and the code asks
+//if that directory exists, it wont be in the cache so it would normally report no, with the fall
+//it would secondary query NetworkFileIO to get the real state and return it in an effort to be
+//safer in edge cases in dynamic development.
+//#define REMOTEFILEIO_CACHE_FILETREE_FALLBACK
+
+///////////////////////////////////////////////////////////////////////////////////
+//NETWORKFILEIO_LOG is a logging option that must tolerate high speed logging, so
+//no output and no allocations, to not alter the timing of calls. On start it allocs
+//a chunk of memmory and writes into it. It is a super verbose logging every file call.
+//#define NETWORKFILEIO_LOG
+
+//REMOTEFILEIO_SYNC_CHECK is an option to enable sync checks between the readahead cache
+//and where the server currently thinks the file is at. It is very useful when debugging
+//a combination of virtual file actions that result in a slightly different state. For example
+//seek()'ing before the begining or past the end of a file is undefined in the standard, and
+//different platforms handle what happens differently. One may allow it, others may not. When
+//it is allowed sometime it means padding the file to a new size with zeros, sometime it just
+//stops at the bounds... this can be used to figure out what is going on in wierd out of sync
+//cases that arrise from a wierd combination of calls on different platforms.
+//#define REMOTEFILEIO_SYNC_CHECK
+
+
+#ifdef REMOTEFILEIO_CACHE_FILETREE
+#include <AzFramework/Asset/AssetCatalogBus.h>
+#endif
+///////////////////////////////////////////////////////////////////////////////////
+namespace AZ
+{
+    namespace IO
+    {
+        class NetworkFileIO
+            : public FileIOBase
+        {
+        public:
+            AZ_RTTI(NetworkFileIO, "{A863335E-9330-44E2-AD89-B5309F3B8B93}", FileIOBase);
+            AZ_CLASS_ALLOCATOR(NetworkFileIO, OSAllocator, 0);
+
+            NetworkFileIO();
+            virtual ~NetworkFileIO();
+
+            ////////////////////////////////////////////////////////////////////////////////////////
+            //implementation of FileIOBase
+
+            Result Open(const char* filePath, OpenMode mode, HandleType& fileHandle) override;
+            Result Close(HandleType fileHandle) override;
+            Result Tell(HandleType fileHandle, AZ::u64& offset) override;
+            Result Seek(HandleType fileHandle, AZ::s64 offset, SeekType type) override;
+            Result Size(HandleType fileHandle, AZ::u64& size) override;
+            Result Read(HandleType fileHandle, void* buffer, AZ::u64 size, bool failOnFewerThanSizeBytesRead = false, AZ::u64* bytesRead = nullptr) override;
+            Result Write(HandleType fileHandle, const void* buffer, AZ::u64 size, AZ::u64* bytesWritten = nullptr) override;
+            Result Flush(HandleType fileHandle) override;
+            bool Eof(HandleType fileHandle) override;
+            AZ::u64 ModificationTime(HandleType fileHandle) override;
+            bool Exists(const char* filePath) override;
+            Result Size(const char* filePath, AZ::u64& size) override;
+            AZ::u64 ModificationTime(const char* filePath) override;
+            bool IsDirectory(const char* filePath) override;
+            bool IsReadOnly(const char* filePath) override;
+            Result CreatePath(const char* filePath) override;
+            Result DestroyPath(const char* filePath) override;
+            Result Remove(const char* filePath) override;
+            Result Copy(const char* sourceFilePath, const char* destinationFilePath) override;
+            Result Rename(const char* sourceFilePath, const char* destinationFilePath) override;
+            Result FindFiles(const char* filePath, const char* filter, FindFilesCallbackType callback) override;
+            void SetAlias(const char* alias, const char* path) override;
+            void ClearAlias(const char* alias) override;
+            AZStd::optional<AZ::u64> ConvertToAlias(char* inOutBuffer, AZ::u64 bufferLength) const override;
+            bool ConvertToAlias(AZ::IO::FixedMaxPath& convertedPath, const AZ::IO::PathView& path) const override;
+            using FileIOBase::ConvertToAlias;
+            const char* GetAlias(const char* alias) const override;
+            bool ResolvePath(const char* path, char* resolvedPath, AZ::u64 resolvedPathSize) const override;
+            bool ResolvePath(AZ::IO::FixedMaxPath& resolvedPath, const AZ::IO::PathView& path) const override;
+            using FileIOBase::ResolvePath;
+            bool GetFilename(HandleType fileHandle, char* filename, AZ::u64 filenameSize) const override;
+            bool IsRemoteIOEnabled() override;
+            ////////////////////////////////////////////////////////////////////////////////////////////
+
+        protected:
+            mutable AZStd::recursive_mutex m_remoteFilesGuard;
+            AZStd::unordered_map<HandleType, AZ::OSString, AZStd::hash<HandleType>, AZStd::equal_to<HandleType>, AZ::OSStdAllocator> m_remoteFiles;
+        };
+
+#ifdef REMOTEFILEIO_IS_NETWORKFILEIO
+        #define RemoteFileIO NetworkFileIO 
+#else
+        //////////////////////////////////////////////////////////////////////////
+        class RemoteFileCache
+        {
+        public:
+            AZ_CLASS_ALLOCATOR(RemoteFileCache, OSAllocator, 0);
+
+            RemoteFileCache() = default;
+            RemoteFileCache(const RemoteFileCache& other) = default;
+            RemoteFileCache(RemoteFileCache&& other);
+            RemoteFileCache& operator=(RemoteFileCache&& other);
+
+            void Invalidate();
+            AZ::u64 RemainingBytes();
+            AZ::u64 CacheFilePosition();
+            AZ::u64 CacheStartFilePosition();
+            AZ::u64 CacheEndFilePosition();
+            bool IsFilePositionInCache(AZ::u64 filePosition);
+            void SetCachePositionFromFilePosition(AZ::u64 filePosition);
+            void SetFilePosition(AZ::u64 filePosition);
+            void OffsetFilePosition(AZ::s64 offset);
+            bool Eof();
+
+            void SyncCheck();
+
+            AZStd::vector<char, AZ::OSStdAllocator> m_cacheLookaheadBuffer;
+            AZ::u64 m_cacheLookaheadPos = 0;
+
+            AZ::u64 m_fileSize = 0;
+            AZ::u64 m_fileSizeTime = 0;
+
+            // note that m_filePosition caches the actual physical pointer location of the file - not the cache pos.
+            // the actual 'tell position' should be this number minus the number of bytes its ahead by in the cache.
+            // Whenever something happens that will actually move the cursor on the remote host occurs, like
+            // NetworkFileIO read, this should be updated. 
+            AZ::u64 m_filePosition = 0;
+
+            HandleType m_fileHandle = AZ::IO::InvalidHandle;
+
+            OpenMode m_openMode = OpenMode::Invalid;
+        };
+
+
+        //////////////////////////////////////////////////////////////////////////
+
+        class RemoteFileIO
+            : public NetworkFileIO
+#ifdef REMOTEFILEIO_CACHE_FILETREE
+            , private AzFramework::AssetCatalogEventBus::Handler
+#endif
+        {
+        public:
+            AZ_RTTI(RemoteFileIO, "{E2939E15-3B83-402A-A6DA-A436EDAB2ED2}", NetworkFileIO);
+            AZ_CLASS_ALLOCATOR(RemoteFileIO, OSAllocator, 0);
+
+            RemoteFileIO(FileIOBase* excludedFileIO = nullptr);
+            virtual ~RemoteFileIO();
+            
+            ////////////////////////////////////////////////////////////////////////////////////////
+            //implementation of NetworkFileIO
+
+            Result Open(const char* filePath, OpenMode mode, HandleType& fileHandle) override;
+            Result Close(HandleType fileHandle) override;
+            Result Tell(HandleType fileHandle, AZ::u64& offset) override;
+            Result Seek(HandleType fileHandle, AZ::s64 offset, SeekType type) override;
+            Result Size(HandleType fileHandle, AZ::u64& size) override;
+            Result Read(HandleType fileHandle, void* buffer, AZ::u64 size, bool failOnFewerThanSizeBytesRead = false, AZ::u64* bytesRead = nullptr) override;
+            Result Write(HandleType fileHandle, const void* buffer, AZ::u64 size, AZ::u64* bytesWritten = nullptr) override;
+            bool Eof(HandleType fileHandle) override;
+            Result Size(const char* filePath, AZ::u64& size) override { return NetworkFileIO::Size(filePath, size); }
+            void SetAlias(const char* alias, const char* path) override;
+            const char* GetAlias(const char* alias) const override;
+            void ClearAlias(const char* alias) override;
+            AZStd::optional<AZ::u64> ConvertToAlias(char* inOutBuffer, AZ::u64 bufferLength) const override;
+            bool ConvertToAlias(AZ::IO::FixedMaxPath& convertedPath, const AZ::IO::PathView& path) const override;
+            using FileIOBase::ConvertToAlias;
+            bool ResolvePath(const char* path, char* resolvedPath, AZ::u64 resolvedPathSize) const override;
+            bool ResolvePath(AZ::IO::FixedMaxPath& resolvedPath, const AZ::IO::PathView& path) const override;
+            using FileIOBase::ResolvePath;
+
+#ifdef REMOTEFILEIO_CACHE_FILETREE
+            bool Exists(const char* filePath) override;
+            bool IsDirectory(const char* filePath) override;
+            Result FindFiles(const char* filePath, const char* filter, FindFilesCallbackType callback) override;
+            
+            Result CacheFileTree();
+            AZStd::vector<AZStd::string> m_remoteFileTreeCache;
+            AZStd::vector<AZStd::string> m_remoteFolderTreeCache;
+            //////////////////////////////////////////////////////////////////////////
+
+            //////////////////////////////////////////////////////////////////////////
+            //implementation of AssetCatalogEventBus
+            void OnCatalogAssetChanged(const AZ::Data::AssetId& assetId) override;
+            void OnCatalogAssetRemoved(const AZ::Data::AssetId& assetId) override;
+            //////////////////////////////////////////////////////////////////////////
+#endif
+            mutable AZStd::recursive_mutex m_remoteFileCacheGuard;
+            AZStd::unordered_map<HandleType, RemoteFileCache, AZStd::hash<HandleType>, AZStd::equal_to<HandleType>, AZ::OSStdAllocator> m_remoteFileCache;
+            // (assumes you're inside a remote files guard lock already for creation, which is true since we create one on open)
+            RemoteFileCache& GetCache(HandleType fileHandle);
+
+        private:
+            FileIOBase* m_excludedFileIO;
+        };
+#endif
+    } // namespace IO
+} // namespace AZ

Code/Framework/AzFramework/AzFramework/IO/RemoteStorageDrive.cpp

@@ -0,0 +1,549 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <inttypes.h>
+#include <AzCore/Casting/numeric_cast.h>
+#include <AzCore/Debug/Profiler.h>
+#include <AzCore/IO/Streamer/FileRequest.h>
+#include <AzCore/IO/Streamer/Statistics.h>
+#include <AzCore/IO/Streamer/StreamerContext.h>
+#include <AzCore/Settings/SettingsRegistry.h>
+#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
+#include <AzCore/Serialization/SerializeContext.h>
+#include <AzCore/std/smart_ptr/make_shared.h>
+#include <AzCore/std/typetraits/decay.h>
+#include <AzFramework/IO/RemoteStorageDrive.h>
+
+namespace AzFramework
+{
+    AZStd::shared_ptr<AZ::IO::StreamStackEntry> RemoteStorageDriveConfig::AddStreamStackEntry(
+        [[maybe_unused]] const AZ::IO::HardwareInformation& hardware, AZStd::shared_ptr<AZ::IO::StreamStackEntry> parent)
+    {
+        AZ::SettingsRegistryInterface* settingsRegistry = AZ::SettingsRegistry::Get();
+        AZ::s64 allowRemoteFilesystem{};
+
+        AZ::SettingsRegistryMergeUtils::PlatformGet(*settingsRegistry, allowRemoteFilesystem,
+            AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey, "remote_filesystem");
+        if (allowRemoteFilesystem != 0)
+        {
+            auto result = AZStd::make_shared<AzFramework::RemoteStorageDrive>(m_maxFileHandles);
+            result->SetNext(AZStd::move(parent));
+            return result;
+        }
+        else
+        {
+            return parent;
+        }
+    }
+
+    void RemoteStorageDriveConfig::Reflect(AZ::ReflectContext* context)
+    {
+        if (auto serializeContext = azrtti_cast<AZ::SerializeContext*>(context); serializeContext != nullptr)
+        {
+            serializeContext->Class<RemoteStorageDriveConfig, AZ::IO::IStreamerStackConfig>()
+                ->Version(1)
+                ->Field("MaxFileHandles", &RemoteStorageDriveConfig::m_maxFileHandles);
+        }
+    }
+
+    RemoteStorageDrive::RemoteStorageDrive(AZ::u32 maxFileHandles)
+        : StreamStackEntry("Storage drive(VFS)")
+    {
+        m_fileLastUsed.resize(maxFileHandles, AZStd::chrono::system_clock::time_point::min());
+        m_filePaths.resize(maxFileHandles);
+        m_fileHandles.resize(maxFileHandles, AZ::IO::InvalidHandle);
+
+        // Add initial dummy values to the stats to avoid division by zero later on and avoid needing branches.
+        m_readSizeAverage.PushEntry(1);
+        m_readTimeAverage.PushEntry(AZStd::chrono::microseconds(1));
+    }
+
+    RemoteStorageDrive::~RemoteStorageDrive()
+    {
+        using namespace AZ::IO;
+
+        for (HandleType handle : m_fileHandles)
+        {
+            if (handle != InvalidHandle)
+            {
+                m_fileIO.Close(handle);
+            }
+        }
+    }
+
+    void RemoteStorageDrive::PrepareRequest(AZ::IO::FileRequest* request)
+    {
+        using namespace AZ::IO;
+
+        AZ_PROFILE_FUNCTION(AZ::Debug::ProfileCategory::AzCore);
+        AZ_Assert(request, "PrepareRequest was provided a null request.");
+
+        if (AZStd::holds_alternative<FileRequest::ReadRequestData>(request->GetCommand()))
+        {
+            auto& readRequest = AZStd::get<FileRequest::ReadRequestData>(request->GetCommand());
+
+            FileRequest* read = m_context->GetNewInternalRequest();
+            read->CreateRead(request, readRequest.m_output, readRequest.m_outputSize, readRequest.m_path,
+                readRequest.m_offset, readRequest.m_size);
+            m_context->PushPreparedRequest(read);
+            return;
+        }
+        StreamStackEntry::PrepareRequest(request);
+    }
+
+    void RemoteStorageDrive::QueueRequest(AZ::IO::FileRequest* request)
+    {
+        using namespace AZ::IO;
+
+        AZ_Assert(request, "QueueRequest was provided a null request.");
+
+        AZStd::visit([this, request](auto&& args)
+        {
+            using namespace AZ::IO;
+            using Command = AZStd::decay_t<decltype(args)>;
+            if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData> ||
+                AZStd::is_same_v<Command, FileRequest::FileExistsCheckData> ||
+                AZStd::is_same_v<Command, FileRequest::FileMetaDataRetrievalData>)
+            {
+                m_pendingRequests.push_back(request);
+                return;
+            }
+            else if constexpr (AZStd::is_same_v<Command, FileRequest::CancelData>)
+            {
+                if (CancelRequest(request, args.m_target))
+                {
+                    // Only forward if this isn't part of the request chain, otherwise the storage device should
+                    // be the last step as it doesn't forward any (sub)requests.
+                    return;
+                }
+            }
+            else if constexpr (AZStd::is_same_v<Command, FileRequest::FlushData>)
+            {
+                FlushCache(args.m_path);
+            }
+            else if constexpr (AZStd::is_same_v<Command, FileRequest::FlushAllData>)
+            {
+                FlushEntireCache();
+            }
+            else if constexpr (AZStd::is_same_v<Command, FileRequest::ReportData>)
+            {
+                Report(args);
+            }
+            StreamStackEntry::QueueRequest(request);
+        }, request->GetCommand());
+    }
+
+    bool RemoteStorageDrive::ExecuteRequests()
+    {
+        using namespace AZ::IO;
+
+        if (!m_pendingRequests.empty())
+        {
+            FileRequest* request = m_pendingRequests.front();
+            AZStd::visit([this, request](auto&& args)
+            {
+                using namespace AZ::IO;
+                using Command = AZStd::decay_t<decltype(args)>;
+                if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
+                {
+                    ReadFile(request);
+                }
+                else if constexpr (AZStd::is_same_v<Command, FileRequest::FileExistsCheckData>)
+                {
+                    FileExistsRequest(request);
+                }
+                else if constexpr (AZStd::is_same_v<Command, FileRequest::FileMetaDataRetrievalData>)
+                {
+                    FileMetaDataRetrievalRequest(request);
+                }
+            }, request->GetCommand());
+            m_pendingRequests.pop_front();
+            return true;
+        }
+        else
+        {
+            return StreamStackEntry::ExecuteRequests();
+        }
+    }
+
+    void RemoteStorageDrive::UpdateStatus(Status& status) const
+    {
+        // Only participate if there are actually any reads done.
+        if (m_fileOpenCloseTimeAverage.GetNumRecorded() > 0)
+        {
+            AZ::s32 availableSlots = s_maxRequests - aznumeric_cast<AZ::s32>(m_pendingRequests.size());
+            StreamStackEntry::UpdateStatus(status);
+            status.m_numAvailableSlots = AZStd::min(status.m_numAvailableSlots, availableSlots);
+            status.m_isIdle = status.m_isIdle && m_pendingRequests.empty();
+        }
+    }
+
+    void RemoteStorageDrive::UpdateCompletionEstimates(AZStd::chrono::system_clock::time_point now,
+        AZStd::vector<AZ::IO::FileRequest*>& internalPending, AZ::IO::StreamerContext::PreparedQueue::iterator pendingBegin,
+        AZ::IO::StreamerContext::PreparedQueue::iterator pendingEnd)
+    {
+        using namespace AZ::IO;
+
+        StreamStackEntry::UpdateCompletionEstimates(now, internalPending, pendingBegin, pendingEnd);
+
+        const RequestPath* activeFile = nullptr;
+        if (m_activeCacheSlot != s_fileNotFound)
+        {
+            activeFile = &m_filePaths[m_activeCacheSlot];
+        }
+            
+        // Estimate requests in this stack entry.
+        for (FileRequest* request : m_pendingRequests)
+        {
+            EstimateCompletionTimeForRequest(request, now, activeFile);
+        }
+
+        // Estimate internally pending requests. Because this call will go from the top of the stack to the bottom,
+        // but estimation is calculated from the bottom to the top, this list should be processed in reverse order.
+        for (auto requestIt = internalPending.rbegin(); requestIt != internalPending.rend(); ++requestIt)
+        {
+            EstimateCompletionTimeForRequest(*requestIt, now, activeFile);
+        }
+
+        // Estimate pending requests that have not been queued yet.
+        for (auto requestIt = pendingBegin; requestIt != pendingEnd; ++requestIt)
+        {
+            EstimateCompletionTimeForRequest(*requestIt, now, activeFile);
+        }
+    }
+
+    void RemoteStorageDrive::EstimateCompletionTimeForRequest(AZ::IO::FileRequest* request,
+        AZStd::chrono::system_clock::time_point& startTime, const AZ::IO::RequestPath*& activeFile) const
+    {
+        using namespace AZ::IO;
+
+        AZ::u64 readSize = 0;
+        const RequestPath* targetFile = nullptr;
+
+        AZStd::visit([&](auto&& args)
+        {
+            using namespace AZ::IO;
+            using Command = AZStd::decay_t<decltype(args)>;
+            if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
+            {
+                targetFile = &args.m_path;
+                readSize = args.m_size;
+            }
+            else if constexpr (AZStd::is_same_v<Command, FileRequest::CompressedReadData>)
+            {
+                targetFile = &args.m_compressionInfo.m_archiveFilename;
+                readSize = args.m_compressionInfo.m_compressedSize;
+            }
+            else if constexpr (AZStd::is_same_v<Command, FileRequest::FileExistsCheckData>)
+            {
+                readSize = 0;
+                AZStd::chrono::microseconds averageTime = m_getFileExistsTimeAverage.CalculateAverage();
+                startTime += averageTime;
+            }
+            else if constexpr (AZStd::is_same_v<Command, FileRequest::FileMetaDataRetrievalData>)
+            {
+                readSize = 0;
+                AZStd::chrono::microseconds averageTime = m_getFileMetaDataTimeAverage.CalculateAverage();
+                startTime += averageTime;
+            }
+        }, request->GetCommand());
+
+        if (readSize > 0)
+        {
+            if (activeFile && activeFile != targetFile)
+            {
+                if (FindFileInCache(*targetFile) == s_fileNotFound)
+                {
+                    AZStd::chrono::microseconds fileOpenCloseTimeAverage = m_fileOpenCloseTimeAverage.CalculateAverage();
+                    startTime += fileOpenCloseTimeAverage;
+                }
+            }
+
+            AZ::u64 totalBytesRead = m_readSizeAverage.GetTotal();
+            double totalReadTimeUSec = aznumeric_caster(m_readTimeAverage.GetTotal().count());
+            startTime += AZStd::chrono::microseconds(aznumeric_cast<AZ::u64>((readSize * totalReadTimeUSec) / totalBytesRead));
+        }
+        request->SetEstimatedCompletion(startTime);
+    }
+
+    void RemoteStorageDrive::ReadFile(AZ::IO::FileRequest* request)
+    {
+        using namespace AZ::IO;
+
+        AZ_PROFILE_FUNCTION(AZ::Debug::ProfileCategory::AzCore);
+            
+        auto data = AZStd::get_if<FileRequest::ReadData>(&request->GetCommand());
+        AZ_Assert(data, "Request doing reading in the RemoteStorageDrive didn't contain read data.")
+
+        HandleType file = InvalidHandle;
+
+        // If the file is already open, use that file handle and update it's last touched time.
+        size_t cacheIndex = FindFileInCache(data->m_path);
+        if (cacheIndex != s_fileNotFound)
+        {
+            file = m_fileHandles[cacheIndex];
+            m_fileLastUsed[cacheIndex] = AZStd::chrono::high_resolution_clock::now();
+        }
+            
+        // If the file is not open, eject the oldest entry from the cache and open the file for reading.
+        if (file == InvalidHandle)
+        {
+            AZStd::chrono::system_clock::time_point oldest = m_fileLastUsed[0];
+            cacheIndex = 0;
+            size_t numFiles = m_filePaths.size();
+            for (size_t i = 1; i < numFiles; ++i)
+            {
+                if (m_fileLastUsed[i] < oldest)
+                {
+                    oldest = m_fileLastUsed[i];
+                    cacheIndex = i;
+                }
+            }
+
+            TIMED_AVERAGE_WINDOW_SCOPE(m_fileOpenCloseTimeAverage);
+            if (!m_fileIO.Open(data->m_path.GetRelativePath(), OpenMode::ModeRead, file))
+            {
+                StreamStackEntry::QueueRequest(request);
+                return;
+            }
+
+            m_fileLastUsed[cacheIndex] = AZStd::chrono::high_resolution_clock::now();
+            if (m_fileHandles[cacheIndex] != InvalidHandle)
+            {
+                m_fileIO.Close(m_fileHandles[cacheIndex]);
+            }
+            m_fileHandles[cacheIndex] = file;
+            m_filePaths[cacheIndex] = data->m_path;
+        }
+        m_activeCacheSlot = cacheIndex;
+
+        AZ_Assert(file != InvalidHandle, 
+            "While searching for file '%s' RemoteStorageDevice::ReadFile encountered a problem that wasn't reported.", data->m_path.GetRelativePath());
+        {
+            TIMED_AVERAGE_WINDOW_SCOPE(m_readTimeAverage);
+            AZ::u64 currentOffset = 0;
+            if (!m_fileIO.Tell(file, currentOffset))
+            {
+                AZ_Warning("IO", false, "RemoteIO failed to tell the offset for a valid file handle for file '%s'.", data->m_path.GetRelativePath());
+                m_readSizeAverage.PushEntry(0);
+                request->SetStatus(IStreamerTypes::RequestStatus::Failed);
+                m_context->MarkRequestAsCompleted(request);
+            }
+            if (currentOffset != data->m_offset)
+            {
+                if (!m_fileIO.Seek(file, data->m_offset, SeekType::SeekFromStart))
+                {
+                    AZ_Warning("IO", false, "RemoteIO failed to tell to seek to %" PRIu64 " in '%s'.", data->m_offset, data->m_path.GetRelativePath());
+                    m_readSizeAverage.PushEntry(0);
+                    request->SetStatus(IStreamerTypes::RequestStatus::Failed);
+                    m_context->MarkRequestAsCompleted(request);
+                }
+            }
+            if (!m_fileIO.Read(file, data->m_output, data->m_size, true))
+            {
+                AZ_Warning("IO", false, "RemoteIO failed to read %i bytes at offset %" PRIu64 " from '%s'.",
+                    data->m_size, data->m_offset, data->m_path.GetRelativePath());
+                m_readSizeAverage.PushEntry(0);
+                request->SetStatus(IStreamerTypes::RequestStatus::Failed);
+                m_context->MarkRequestAsCompleted(request);
+            }
+        }
+        m_readSizeAverage.PushEntry(data->m_size);
+            
+        request->SetStatus(IStreamerTypes::RequestStatus::Completed);
+        m_context->MarkRequestAsCompleted(request);
+    }
+
+    bool RemoteStorageDrive::CancelRequest(AZ::IO::FileRequest* cancelRequest, AZ::IO::FileRequestPtr& target)
+    {
+        using namespace AZ::IO;
+
+        bool ownsRequestChain = false;
+        for (auto it = m_pendingRequests.begin(); it != m_pendingRequests.end();)
+        {
+            if ((*it)->WorksOn(target))
+            {
+                (*it)->SetStatus(IStreamerTypes::RequestStatus::Canceled);
+                m_context->MarkRequestAsCompleted(*it);
+                it = m_pendingRequests.erase(it);
+                ownsRequestChain = true;
+            }
+            else
+            {
+                ++it;
+            }
+        }
+
+        if (ownsRequestChain)
+        {
+            cancelRequest->SetStatus(IStreamerTypes::RequestStatus::Completed);
+            m_context->MarkRequestAsCompleted(cancelRequest);
+        }
+
+        return ownsRequestChain;
+    }
+
+    void RemoteStorageDrive::FileExistsRequest(AZ::IO::FileRequest* request)
+    {
+        using namespace AZ::IO;
+
+        TIMED_AVERAGE_WINDOW_SCOPE(m_getFileExistsTimeAverage);
+
+        auto& fileExists = AZStd::get<FileRequest::FileExistsCheckData>(request->GetCommand());
+        size_t cacheIndex = FindFileInCache(fileExists.m_path);
+        if (cacheIndex != s_fileNotFound)
+        {
+            fileExists.m_found = true;
+            m_context->MarkRequestAsCompleted(request);
+        }
+        else
+        {
+            if (m_fileIO.Exists(fileExists.m_path.GetAbsolutePath()))
+            {
+                fileExists.m_found = true;
+                request->SetStatus(IStreamerTypes::RequestStatus::Completed);
+                m_context->MarkRequestAsCompleted(request);
+            }
+            else
+            {
+                // File couldn't be found through VFS so let the next node try locally.
+                StreamStackEntry::QueueRequest(request);
+            }
+        }
+    }
+
+    void RemoteStorageDrive::FileMetaDataRetrievalRequest(AZ::IO::FileRequest* request)
+    {
+        using namespace AZ::IO;
+
+        AZ_PROFILE_FUNCTION(AZ::Debug::ProfileCategory::AzCore);
+        TIMED_AVERAGE_WINDOW_SCOPE(m_getFileMetaDataTimeAverage);
+
+        AZ::u64 fileSize = 0;
+        bool found = false;
+
+        auto& command = AZStd::get<FileRequest::FileMetaDataRetrievalData>(request->GetCommand());
+        // If the file is already open, use the file handle which usually is cheaper than asking for the file by name.
+        size_t cacheIndex = FindFileInCache(command.m_path);
+        if (cacheIndex != s_fileNotFound)
+        {
+            AZ_Assert(m_fileHandles[cacheIndex] != InvalidHandle,
+                "File path '%s' doesn't have an associated file handle.", m_filePaths[cacheIndex].GetRelativePath());
+            found = m_fileIO.Size(m_fileHandles[cacheIndex], fileSize);
+        }
+        else
+        {
+            // The file is not open yet, so try to get the file size by name.
+            m_fileIO.Size(command.m_path.GetAbsolutePath(), fileSize);
+        }
+
+        if (found)
+        {
+            command.m_fileSize = fileSize;
+            command.m_found = true;
+            request->SetStatus(IStreamerTypes::RequestStatus::Completed);
+            m_context->MarkRequestAsCompleted(request);
+        }
+        else
+        {
+            StreamStackEntry::QueueRequest(request);
+        }
+    }
+
+    void RemoteStorageDrive::FlushCache(const AZ::IO::RequestPath& filePath)
+    {
+        using namespace AZ::IO;
+
+        size_t cacheIndex = FindFileInCache(filePath);
+        if (cacheIndex != s_fileNotFound)
+        {
+            m_fileLastUsed[cacheIndex] = AZStd::chrono::system_clock::time_point();
+            m_filePaths[cacheIndex].Clear();
+            AZ_Assert(m_fileHandles[cacheIndex] != AZ::IO::InvalidHandle,
+                "File path '%s' doesn't have an associated file handle.", m_filePaths[cacheIndex].GetRelativePath());
+            m_fileIO.Close(m_fileHandles[cacheIndex]);
+            m_fileHandles[cacheIndex] = InvalidHandle;
+        }
+    }
+
+    void RemoteStorageDrive::FlushEntireCache()
+    {
+        size_t numFiles = m_filePaths.size();
+        for (size_t i = 0; i < numFiles; ++i)
+        {
+            m_fileLastUsed[i] = AZStd::chrono::system_clock::time_point();
+            m_filePaths[i].Clear();
+            if (m_fileHandles[i] != AZ::IO::InvalidHandle)
+            {
+                m_fileIO.Close(m_fileHandles[i]);
+                m_fileHandles[i] = AZ::IO::InvalidHandle;
+            }
+        }
+    }
+
+    size_t RemoteStorageDrive::FindFileInCache(const AZ::IO::RequestPath& filePath) const
+    {
+        size_t numFiles = m_filePaths.size();
+        for (size_t i = 0; i < numFiles; ++i)
+        {
+            if (m_filePaths[i] == filePath)
+            {
+                return i;
+            }
+        }
+        return AZ::IO::s_fileNotFound;
+    }
+
+    void RemoteStorageDrive::CollectStatistics(AZStd::vector<AZ::IO::Statistic>& statistics) const
+    {
+        using namespace AZ::IO;
+
+        using DoubleSeconds = AZStd::chrono::duration<double>;
+            
+        double totalBytesReadMB = m_readSizeAverage.GetTotal() / (1024.0 * 1024.0);
+        double totalReadTimeSec = AZStd::chrono::duration_cast<DoubleSeconds>(m_readTimeAverage.GetTotal()).count();
+        if (m_readSizeAverage.GetTotal() > 1) // A default is always added.
+        {
+            statistics.push_back(Statistic::CreateFloat(m_name, "Read Speed (avg. mbps)", totalBytesReadMB / totalReadTimeSec));
+        }
+
+        if (m_fileOpenCloseTimeAverage.GetNumRecorded() > 0)
+        {
+            statistics.push_back(Statistic::CreateInteger(m_name, "File Open & Close (avg. us)", m_fileOpenCloseTimeAverage.CalculateAverage().count()));
+            statistics.push_back(Statistic::CreateInteger(m_name, "Get file exists (avg. us)", m_getFileExistsTimeAverage.CalculateAverage().count()));
+            statistics.push_back(Statistic::CreateInteger(m_name, "Get file meta data (avg. us)", m_getFileMetaDataTimeAverage.CalculateAverage().count()));
+            statistics.push_back(Statistic::CreateInteger(m_name, "Available slots", AZ::s64{ s_maxRequests } - m_pendingRequests.size()));
+        }
+
+        StreamStackEntry::CollectStatistics(statistics);
+    }
+
+    void RemoteStorageDrive::Report(const AZ::IO::FileRequest::ReportData& data) const
+    {
+        using namespace AZ::IO;
+
+        switch (data.m_reportType)
+        {
+        case FileRequest::ReportData::ReportType::FileLocks:
+            for (AZ::u32 i = 0; i < m_fileHandles.size(); ++i)
+            {
+                if (m_fileHandles[i] != InvalidHandle)
+                {
+                    AZ_Printf("Streamer", "File lock in %s : '%s'.\n", m_name.c_str(), m_filePaths[i].GetRelativePath());
+                }
+            }
+            break;
+        default:
+            break;
+        }
+    }
+} // namespace AzFramework

Code/Framework/AzFramework/AzFramework/IO/RemoteStorageDrive.h

@@ -0,0 +1,86 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#pragma once
+
+#include <AzCore/IO/Streamer/Statistics.h>
+#include <AzCore/IO/Streamer/StreamerConfiguration.h>
+#include <AzCore/IO/Streamer/StreamStackEntry.h>
+#include <AzCore/std/containers/deque.h>
+#include <AzCore/std/containers/vector.h>
+#include <AzCore/std/chrono/clocks.h>
+#include <AzFramework/IO/RemoteFileIO.h>
+
+namespace AzFramework
+{
+    struct RemoteStorageDriveConfig final :
+        public AZ::IO::IStreamerStackConfig
+    {
+        AZ_RTTI(AzFramework::RemoteStorageDriveConfig, "{A37AC957-AC1F-4CAC-BB7B-0CAD5ABAD416}", AZ::IO::IStreamerStackConfig);
+        AZ_CLASS_ALLOCATOR(RemoteStorageDriveConfig, AZ::SystemAllocator, 0);
+
+        ~RemoteStorageDriveConfig() override = default;
+        AZStd::shared_ptr<AZ::IO::StreamStackEntry> AddStreamStackEntry(
+            const AZ::IO::HardwareInformation& hardware, AZStd::shared_ptr<AZ::IO::StreamStackEntry> parent) override;
+        static void Reflect(AZ::ReflectContext* context);
+
+        AZ::u32 m_maxFileHandles{ 1024 };
+    };
+
+    class RemoteStorageDrive
+        : public AZ::IO::StreamStackEntry
+    {
+    public:
+        explicit RemoteStorageDrive(AZ::u32 maxFileHandles);
+        ~RemoteStorageDrive() override;
+
+        void PrepareRequest(AZ::IO::FileRequest* request) override;
+        void QueueRequest(AZ::IO::FileRequest* request) override;
+        bool ExecuteRequests() override;
+
+        void UpdateStatus(Status& status) const override;
+        void UpdateCompletionEstimates(AZStd::chrono::system_clock::time_point now,
+            AZStd::vector<AZ::IO::FileRequest*>& internalPending,
+            AZ::IO::StreamerContext::PreparedQueue::iterator pendingBegin,
+            AZ::IO::StreamerContext::PreparedQueue::iterator pendingEnd) override;
+
+        void CollectStatistics(AZStd::vector<AZ::IO::Statistic>& statistics) const override;
+
+    protected:
+        static constexpr AZ::s32 s_maxRequests = 1;
+            
+        void ReadFile(AZ::IO::FileRequest* request);
+        bool CancelRequest(AZ::IO::FileRequest* cancelRequest, AZ::IO::FileRequestPtr& target);
+        void FileExistsRequest(AZ::IO::FileRequest* request);
+        void FileMetaDataRetrievalRequest(AZ::IO::FileRequest* request);
+        size_t FindFileInCache(const AZ::IO::RequestPath& filePath) const;
+        void EstimateCompletionTimeForRequest(AZ::IO::FileRequest* request, AZStd::chrono::system_clock::time_point& startTime,
+            const AZ::IO::RequestPath*& activeFile) const;
+        void FlushCache(const AZ::IO::RequestPath& filePath);
+        void FlushEntireCache();
+        void Report(const AZ::IO::FileRequest::ReportData& data) const;
+
+        AZ::IO::RemoteFileIO m_fileIO;
+        AZ::IO::TimedAverageWindow<AZ::IO::s_statisticsWindowSize> m_fileOpenCloseTimeAverage;
+        AZ::IO::TimedAverageWindow<AZ::IO::s_statisticsWindowSize> m_getFileExistsTimeAverage;
+        AZ::IO::TimedAverageWindow<AZ::IO::s_statisticsWindowSize> m_getFileMetaDataTimeAverage;
+        AZ::IO::TimedAverageWindow<AZ::IO::s_statisticsWindowSize> m_readTimeAverage;
+        AZ::IO::AverageWindow<AZ::u64, float, AZ::IO::s_statisticsWindowSize> m_readSizeAverage;
+        AZStd::deque<AZ::IO::FileRequest*> m_pendingRequests;
+
+        AZStd::vector<AZStd::chrono::system_clock::time_point> m_fileLastUsed;
+        AZStd::vector<AZ::IO::RequestPath> m_filePaths;
+        AZStd::vector<AZ::IO::HandleType> m_fileHandles;
+
+        size_t m_activeCacheSlot = AZ::IO::s_fileNotFound;
+    };
+} // namespace AzFramework