birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #6225 from aws-lumberyard-dev/burelc/inotify

Browse Source 
[AssetProcessor] Refactor the FileWatcher to use only one watch thread
This commit is contained in:
Chris Burel
2022-01-04 18:18:29 -08:00
committed by GitHub
parent 5eb6c2d24b 648a21ab5c
commit bfb1d0ad9e
23 changed files with 708 additions and 944 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/AssetProcessor/Platform/Linux/assetprocessor_linux_files.cmake
+2
View File
@@ -8,4 +8,6 @@
set(FILES
native/FileWatcher/FileWatcher_linux.cpp
native/FileWatcher/FileWatcher_linux.h
native/FileWatcher/FileWatcher_platform.h
)
Code/Tools/AssetProcessor/Platform/Linux/native/FileWatcher/FileWatcher_linux.cpp
+152 -163
View File
@@ -5,7 +5,10 @@
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <AzCore/std/string/fixed_string.h>
#include <native/FileWatcher/FileWatcher.h>
#include <native/FileWatcher/FileWatcher_platform.h>
#include <QDirIterator>
#include <QHash>
@@ -15,155 +18,127 @@
#include <sys/inotify.h>
static constexpr int s_handleToFolderMapLockTimeout = 1000; // 1 sec timeout for obtaining the handle to folder map lock
static constexpr size_t s_iNotifyMaxEntries = 1024 * 16; // Control the maximum number of entries (from inotify) that can be read at one time
static constexpr size_t s_iNotifyEventSize = sizeof(struct inotify_event);
static constexpr size_t s_iNotifyReadBufferSize = s_iNotifyMaxEntries * s_iNotifyEventSize;
static constexpr size_t s_inotifyMaxEntries = 1024 * 16; // Control the maximum number of entries (from inotify) that can be read at one time
static constexpr size_t s_inotifyEventSize = sizeof(struct inotify_event);
static constexpr size_t s_inotifyReadBufferSize = s_inotifyMaxEntries * s_inotifyEventSize;
struct FolderRootWatch::PlatformImplementation
bool FileWatcher::PlatformImplementation::Initialize()
{
PlatformImplementation() = default;
int m_iNotifyHandle = -1;
QMutex m_handleToFolderMapLock;
QHash<int, QString> m_handleToFolderMap;
bool Initialize()
if (m_inotifyHandle < 0)
{
if (m_iNotifyHandle < 0)
{
// The CLOEXEC flag prevents the inotify watchers from copying on fork/exec
m_iNotifyHandle = inotify_init1(IN_CLOEXEC);
}
return (m_iNotifyHandle >= 0);
// The CLOEXEC flag prevents the inotify watchers from copying on fork/exec
m_inotifyHandle = inotify_init1(IN_CLOEXEC);
[[maybe_unused]] const auto err = errno;
[[maybe_unused]] AZStd::fixed_string<255> errorString;
AZ_Warning("FileWatcher", (m_inotifyHandle >= 0), "Unable to initialize inotify, file monitoring will not be available: %s\n", strerror_r(err, errorString.data(), errorString.capacity()));
}
void Finalize()
{
if (m_iNotifyHandle >= 0)
{
if (!m_handleToFolderMapLock.tryLock(s_handleToFolderMapLockTimeout))
{
AZ_Error("FileWatcher", false, "Unable to obtain inotify handle lock on thread");
return;
}
QHashIterator<int, QString> iter(m_handleToFolderMap);
while (iter.hasNext())
{
iter.next();
int watchHandle = iter.key();
inotify_rm_watch(m_iNotifyHandle, watchHandle);
}
m_handleToFolderMap.clear();
m_handleToFolderMapLock.unlock();
::close(m_iNotifyHandle);
m_iNotifyHandle = -1;
}
}
void AddWatchFolder(QString folder)
{
if (m_iNotifyHandle >= 0)
{
// Clean up the path before accepting it as a watch folder
QString cleanPath = QDir::cleanPath(folder);
// Add the folder to watch and track it
int watchHandle = inotify_add_watch(m_iNotifyHandle,
cleanPath.toUtf8().constData(),
IN_CREATE | IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY | IN_MOVE);
if (!m_handleToFolderMapLock.tryLock(s_handleToFolderMapLockTimeout))
{
AZ_Error("FileWatcher", false, "Unable to obtain inotify handle lock on thread");
return;
}
m_handleToFolderMap[watchHandle] = cleanPath;
m_handleToFolderMapLock.unlock();
// Add all the subfolders to watch and track them
QDirIterator dirIter(folder, QDirIterator::Subdirectories | QDirIterator::FollowSymlinks);
while (dirIter.hasNext())
{
QString dirName = dirIter.next();
if (dirName.endsWith("/.") || dirName.endsWith("/.."))
{
continue;
}
int watchHandle = inotify_add_watch(m_iNotifyHandle,
dirName.toUtf8().constData(),
IN_CREATE | IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY | IN_MOVE);
if (!m_handleToFolderMapLock.tryLock(s_handleToFolderMapLockTimeout))
{
AZ_Error("FileWatcher", false, "Unable to obtain inotify handle lock on thread");
return;
}
m_handleToFolderMap[watchHandle] = dirName;
m_handleToFolderMapLock.unlock();
}
}
}
void RemoveWatchFolder(int watchHandle)
{
if (m_iNotifyHandle >= 0)
{
if (!m_handleToFolderMapLock.tryLock(s_handleToFolderMapLockTimeout))
{
AZ_Error("FileWatcher", false, "Unable to obtain inotify handle lock on thread");
return;
}
QHash<int, QString>::iterator handleToRemove = m_handleToFolderMap.find(watchHandle);
if (handleToRemove != m_handleToFolderMap.end())
{
inotify_rm_watch(m_iNotifyHandle, watchHandle);
m_handleToFolderMap.erase(handleToRemove);
}
m_handleToFolderMapLock.unlock();
}
}
};
//////////////////////////////////////////////////////////////////////////////
/// FolderWatchRoot
FolderRootWatch::FolderRootWatch(const QString rootFolder)
: m_root(rootFolder)
, m_shutdownThreadSignal(false)
, m_fileWatcher(nullptr)
, m_platformImpl(new PlatformImplementation())
{
return (m_inotifyHandle >= 0);
}
FolderRootWatch::~FolderRootWatch()
void FileWatcher::PlatformImplementation::Finalize()
{
// Destructor is required in here since this file contains the definition of struct PlatformImplementation
Stop();
if (m_inotifyHandle < 0)
{
return;
}
delete m_platformImpl;
{
QMutexLocker lock{&m_handleToFolderMapLock};
for (const auto& watchHandle : m_handleToFolderMap.keys())
{
inotify_rm_watch(m_inotifyHandle, watchHandle);
}
m_handleToFolderMap.clear();
}
::close(m_inotifyHandle);
m_inotifyHandle = -1;
}
bool FolderRootWatch::Start()
void FileWatcher::PlatformImplementation::AddWatchFolder(QString folder, bool recursive)
{
if (m_inotifyHandle < 0)
{
return;
}
// Clean up the path before accepting it as a watch folder
QString cleanPath = QDir::cleanPath(folder);
// Add the folder to watch and track it
int watchHandle = inotify_add_watch(m_inotifyHandle,
cleanPath.toUtf8().constData(),
IN_CREATE | IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY | IN_MOVE);
if (watchHandle < 0)
{
[[maybe_unused]] const auto err = errno;
[[maybe_unused]] AZStd::fixed_string<255> errorString;
AZ_Warning("FileWatcher", false, "inotify_add_watch failed for path %s: %s", cleanPath.toUtf8().constData(), strerror_r(err, errorString.data(), errorString.capacity()));
return;
}
{
QMutexLocker lock{&m_handleToFolderMapLock};
m_handleToFolderMap[watchHandle] = cleanPath;
}
// Add all the contents (files and directories) to watch and track them
QDirIterator dirIter(folder, QDir::NoDotAndDotDot | QDir::Dirs | QDir::Files, (recursive ? QDirIterator::Subdirectories : QDirIterator::NoIteratorFlags) | QDirIterator::FollowSymlinks);
while (dirIter.hasNext())
{
QString dirName = dirIter.next();
watchHandle = inotify_add_watch(m_inotifyHandle,
dirName.toUtf8().constData(),
IN_CREATE | IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY | IN_MOVE);
if (watchHandle < 0)
{
[[maybe_unused]] const auto err = errno;
[[maybe_unused]] AZStd::fixed_string<255> errorString;
AZ_Warning("FileWatcher", false, "inotify_add_watch failed for path %s: %s", dirName.toUtf8().constData(), strerror_r(err, errorString.data(), errorString.capacity()));
return;
}
QMutexLocker lock{&m_handleToFolderMapLock};
m_handleToFolderMap[watchHandle] = dirName;
}
}
void FileWatcher::PlatformImplementation::RemoveWatchFolder(int watchHandle)
{
if (m_inotifyHandle < 0)
{
return;
}
QMutexLocker lock{&m_handleToFolderMapLock};
if (m_handleToFolderMap.remove(watchHandle))
{
inotify_rm_watch(m_inotifyHandle, watchHandle);
}
}
bool FileWatcher::PlatformStart()
{
// inotify will be used by linux to monitor file changes within directories under the root folder
if (!m_platformImpl->Initialize())
{
return false;
}
m_platformImpl->AddWatchFolder(m_root);
for (const auto& [directory, recursive] : m_folderWatchRoots)
{
if (QDir(directory).exists())
{
m_platformImpl->AddWatchFolder(directory, recursive);
}
}
m_shutdownThreadSignal = false;
m_thread = std::thread([this]() { WatchFolderLoop(); });
return true;
}
void FolderRootWatch::Stop()
void FileWatcher::PlatformStop()
{
m_shutdownThreadSignal = true;
@@ -172,64 +147,78 @@ void FolderRootWatch::Stop()
if (m_thread.joinable())
{
m_thread.join(); // wait for the thread to finish
m_thread = std::thread(); //destroy
}
}
void FolderRootWatch::WatchFolderLoop()
void FileWatcher::WatchFolderLoop()
{
char eventBuffer[s_iNotifyReadBufferSize];
char eventBuffer[s_inotifyReadBufferSize];
while (!m_shutdownThreadSignal)
{
ssize_t bytesRead = ::read(m_platformImpl->m_iNotifyHandle, eventBuffer, s_iNotifyReadBufferSize);
ssize_t bytesRead = ::read(m_platformImpl->m_inotifyHandle, eventBuffer, s_inotifyReadBufferSize);
if (bytesRead < 0)
{
// Break out of the loop when the notify handle was closed (outside of this thread)
break;
}
else if (bytesRead > 0)
if (!bytesRead)
{
for (size_t index=0; index<bytesRead;)
continue;
}
for (size_t index=0; index<bytesRead;)
{
const auto* event = reinterpret_cast<inotify_event*>(&eventBuffer[index]);
if (event->mask & (IN_CREATE | IN_DELETE | IN_MODIFY | IN_MOVE))
{
struct inotify_event *event = ( struct inotify_event * ) &eventBuffer[ index ];
const QString pathStr = QDir(m_platformImpl->m_handleToFolderMap[event->wd]).absoluteFilePath(event->name);
if (event->mask & (IN_CREATE | IN_DELETE | IN_MODIFY | IN_MOVE ))
if (event->mask & (IN_CREATE | IN_MOVED_TO))
{
QString pathStr = QString("%1%2%3").arg(m_platformImpl->m_handleToFolderMap[event->wd], QDir::separator(), event->name);
if (event->mask & IN_ISDIR)
{
// New Directory, see if it should be added to the watched directories
// It is only added if it is a child of a recursively watched directory
const auto found = AZStd::find_if(begin(m_folderWatchRoots), end(m_folderWatchRoots), [this, event](const WatchRoot& watchRoot)
{
return watchRoot.m_directory == m_platformImpl->m_handleToFolderMap[event->wd];
});
if (event->mask & (IN_CREATE | IN_MOVED_TO))
{
if ( event->mask & IN_ISDIR )
// If the path is not in m_folderWatchRoots, it must
// be a new subdirectory of a subdirectory of some
// other root that is being watched recursively.
// Maintain the recursive nature of that root.
const bool shouldAddFolder = (found == end(m_folderWatchRoots)) ? true : found->m_recursive;
if (shouldAddFolder)
{
// New Directory, add it to the watch
m_platformImpl->AddWatchFolder(pathStr);
}
else
{
ProcessNewFileEvent(pathStr);
m_platformImpl->AddWatchFolder(pathStr, true);
}
}
else if (event->mask & (IN_DELETE | IN_MOVED_FROM))
else
{
if (event->mask & IN_ISDIR)
{
// Directory Deleted, remove it from the watch
m_platformImpl->RemoveWatchFolder(event->wd);
}
else
{
ProcessDeleteFileEvent(pathStr);
}
}
else if ((event->mask & IN_MODIFY) && ((event->mask & IN_ISDIR) != IN_ISDIR))
{
ProcessModifyFileEvent(pathStr);
rawFileAdded(pathStr, {});
}
}
index += s_iNotifyEventSize + event->len;
else if (event->mask & (IN_DELETE | IN_MOVED_FROM))
{
if (event->mask & IN_ISDIR)
{
// Directory Deleted, remove it from the watch
m_platformImpl->RemoveWatchFolder(event->wd);
}
else
{
rawFileRemoved(pathStr, {});
}
}
else if ((event->mask & IN_MODIFY) && ((event->mask & IN_ISDIR) != IN_ISDIR))
{
rawFileModified(pathStr, {});
}
}
index += s_inotifyEventSize + event->len;
}
}
}
Code/Tools/AssetProcessor/Platform/Linux/native/FileWatcher/FileWatcher_linux.h
+26
View File
@@ -0,0 +1,26 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <FileWatcher/FileWatcher.h>
#include <QMutex>
#include <QHash>
class FileWatcher::PlatformImplementation
{
public:
bool Initialize();
void Finalize();
void AddWatchFolder(QString folder, bool recursive);
void RemoveWatchFolder(int watchHandle);
int m_inotifyHandle = -1;
QMutex m_handleToFolderMapLock;
QHash<int, QString> m_handleToFolderMap;
};
Code/Tools/AssetProcessor/Platform/Linux/native/FileWatcher/FileWatcher_platform.h
+11
View File
@@ -0,0 +1,11 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <native/FileWatcher/FileWatcher_linux.h>
Code/Tools/AssetProcessor/Platform/Mac/assetprocessor_mac_files.cmake
+2
View File
@@ -8,4 +8,6 @@
set(FILES
native/FileWatcher/FileWatcher_macos.cpp
native/FileWatcher/FileWatcher_mac.h
native/FileWatcher/FileWatcher_platform.h
)
Code/Tools/AssetProcessor/Platform/Mac/native/FileWatcher/FileWatcher_mac.h
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <native/FileWatcher/FileWatcher.h>
#include <CoreServices/CoreServices.h>
class FileWatcher::PlatformImplementation
{
public:
FSEventStreamRef m_stream = nullptr;
CFRunLoopRef m_runLoop = nullptr;
};
Code/Tools/AssetProcessor/Platform/Mac/native/FileWatcher/FileWatcher_macos.cpp
+31 -53
View File
@@ -6,47 +6,23 @@
*
*/
#include <native/FileWatcher/FileWatcher.h>
#include <native/FileWatcher/FileWatcher_platform.h>
#include <native/utilities/BatchApplicationManager.h>
#include <AzCore/Debug/Trace.h>
#include <CoreServices/CoreServices.h>
void FileEventStreamCallback(ConstFSEventStreamRef streamRef, void *clientCallBackInfo, size_t numEvents, void *eventPaths, const FSEventStreamEventFlags eventFlags[], const FSEventStreamEventId eventIds[]);
struct FolderRootWatch::PlatformImplementation
{
PlatformImplementation() : m_stream(nullptr), m_runLoop(nullptr) { }
FSEventStreamRef m_stream;
CFRunLoopRef m_runLoop;
QString m_renameFileDirectory;
};
//////////////////////////////////////////////////////////////////////////////
/// FolderWatchRoot
FolderRootWatch::FolderRootWatch(const QString rootFolder)
: m_root(rootFolder)
, m_shutdownThreadSignal(false)
, m_fileWatcher(nullptr)
, m_platformImpl(new PlatformImplementation())
{
}
FolderRootWatch::~FolderRootWatch()
{
// Destructor is required in here since this file contains the definition of struct PlatformImplementation
Stop();
delete m_platformImpl;
}
bool FolderRootWatch::Start()
bool FileWatcher::PlatformStart()
{
m_shutdownThreadSignal = false;
CFStringRef rootPath = CFStringCreateWithCString(kCFAllocatorDefault, m_root.toStdString().data(), kCFStringEncodingMacRoman);
CFArrayRef pathsToWatch = CFArrayCreate(NULL, (const void **)&rootPath, 1, NULL);
CFMutableArrayRef pathsToWatch = CFArrayCreateMutable(nullptr, this->m_folderWatchRoots.size(), nullptr);
for (const auto& root : this->m_folderWatchRoots)
{
CFArrayAppendValue(pathsToWatch, root.m_directory.toCFString());
}
// The larger this number, the larger the delay between the kernel knowing a file changed
// and us actually consuming the event. It is very important for asset processor to deal with
@@ -60,11 +36,12 @@ bool FolderRootWatch::Start()
// Set ourselves as the value for the context info field so that in the callback
// we get passed into it and the callback can call our public API to handle
// the file change events
FSEventStreamContext streamContext;
::memset(&streamContext, 0, sizeof(streamContext));
streamContext.info = this;
FSEventStreamContext streamContext{
/*.version =*/ 0,
/*.info =*/ this,
};
m_platformImpl->m_stream = FSEventStreamCreate(NULL,
m_platformImpl->m_stream = FSEventStreamCreate(nullptr,
FileEventStreamCallback,
&streamContext,
pathsToWatch,
@@ -72,24 +49,25 @@ bool FolderRootWatch::Start()
timeBetweenKernelUpdateAndNotification,
kFSEventStreamCreateFlagFileEvents);
AZ_Error("FileWatcher", (m_platformImpl->m_stream != nullptr), "FSEventStreamCreate returned a nullptr. No file events will be reported for %s", m_root.toStdString().c_str());
m_thread = std::thread(std::bind(&FolderRootWatch::WatchFolderLoop, this));
AZ_Error("FileWatcher", (m_platformImpl->m_stream != nullptr), "FSEventStreamCreate returned a nullptr. No file events will be reported.");
const CFIndex pathCount = CFArrayGetCount(pathsToWatch);
for(CFIndex i = 0; i < pathCount; ++i)
{
CFRelease(CFArrayGetValueAtIndex(pathsToWatch, i));
}
CFRelease(pathsToWatch);
CFRelease(rootPath);
return (m_platformImpl->m_stream != nullptr);
return m_platformImpl->m_stream != nullptr;
}
void FolderRootWatch::Stop()
void FileWatcher::PlatformStop()
{
m_shutdownThreadSignal = true;
if (m_thread.joinable())
{
m_thread.join(); // wait for the thread to finish
m_thread = std::thread(); //destroy
}
FSEventStreamStop(m_platformImpl->m_stream);
@@ -97,7 +75,7 @@ void FolderRootWatch::Stop()
FSEventStreamRelease(m_platformImpl->m_stream);
}
void FolderRootWatch::WatchFolderLoop()
void FileWatcher::WatchFolderLoop()
{
// Use a half second timeout interval so that we can check if
// m_shutdownThreadSignal has been changed while we were running the RunLoop
@@ -117,14 +95,14 @@ void FolderRootWatch::WatchFolderLoop()
void FileEventStreamCallback(ConstFSEventStreamRef streamRef, void *clientCallBackInfo, size_t numEvents, void *eventPaths, const FSEventStreamEventFlags eventFlags[], const FSEventStreamEventId eventIds[])
{
FolderRootWatch* watcher = reinterpret_cast<FolderRootWatch*>(clientCallBackInfo);
auto* watcher = reinterpret_cast<FileWatcher*>(clientCallBackInfo);
const char** filePaths = reinterpret_cast<const char**>(eventPaths);
for (int i = 0; i < numEvents; ++i)
{
QFileInfo fileInfo(QDir::cleanPath(filePaths[i]));
QString fileAndPath = fileInfo.absoluteFilePath();
const QFileInfo fileInfo(QDir::cleanPath(filePaths[i]));
const QString fileAndPath = fileInfo.absoluteFilePath();
if (!fileInfo.isHidden())
{
@@ -133,38 +111,38 @@ void FileEventStreamCallback(ConstFSEventStreamRef streamRef, void *clientCallBa
// so check for all of them
if (eventFlags[i] & kFSEventStreamEventFlagItemCreated)
{
watcher->ProcessNewFileEvent(fileAndPath);
watcher->rawFileAdded(fileAndPath, {});
}
if (eventFlags[i] & kFSEventStreamEventFlagItemModified)
{
watcher->ProcessModifyFileEvent(fileAndPath);
watcher->rawFileModified(fileAndPath, {});
}
if (eventFlags[i] & kFSEventStreamEventFlagItemRemoved)
{
watcher->ProcessDeleteFileEvent(fileAndPath);
watcher->rawFileRemoved(fileAndPath, {});
}
if (eventFlags[i] & kFSEventStreamEventFlagItemRenamed)
{
if (fileInfo.exists())
{
watcher->ProcessNewFileEvent(fileAndPath);
watcher->rawFileAdded(fileAndPath, {});
// macOS does not send out an event for the directory being
// modified when a file has been renamed but the FileWatcher
// API expects it so send out the modification event ourselves.
watcher->ProcessModifyFileEvent(fileInfo.absolutePath());
watcher->rawFileModified(fileInfo.absolutePath(), {});
}
else
{
watcher->ProcessDeleteFileEvent(fileAndPath);
watcher->rawFileRemoved(fileAndPath, {});
// macOS does not send out an event for the directory being
// modified when a file has been renamed but the FileWatcher
// API expects it so send out the modification event ourselves.
watcher->ProcessModifyFileEvent(fileInfo.absolutePath());
watcher->rawFileModified(fileInfo.absolutePath(), {});
}
}
}
Code/Tools/AssetProcessor/Platform/Mac/native/FileWatcher/FileWatcher_platform.h
+11
View File
@@ -0,0 +1,11 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <native/FileWatcher/FileWatcher_mac.h>
Code/Tools/AssetProcessor/Platform/Mac/native/FileWatcher/FileWatcher_win.cpp
-130
View File
@@ -1,130 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <native/FileWatcher/FileWatcher.h>
#include <AzCore/PlatformIncl.h>
struct FolderRootWatch::PlatformImplementation
{
PlatformImplementation() : m_directoryHandle(nullptr), m_ioHandle(nullptr) { }
HANDLE m_directoryHandle;
HANDLE m_ioHandle;
};
//////////////////////////////////////////////////////////////////////////////
/// FolderWatchRoot
FolderRootWatch::FolderRootWatch(const QString rootFolder)
: m_root(rootFolder)
, m_shutdownThreadSignal(false)
, m_fileWatcher(nullptr)
, m_platformImpl(new PlatformImplementation())
{
}
FolderRootWatch::~FolderRootWatch()
{
// Destructor is required in here since this file contains the definition of struct PlatformImplementation
Stop();
delete m_platformImpl;
}
bool FolderRootWatch::Start()
{
m_platformImpl->m_directoryHandle = ::CreateFileW(m_root.toStdWString().data(), FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, nullptr);
if (m_platformImpl->m_directoryHandle != INVALID_HANDLE_VALUE)
{
m_platformImpl->m_ioHandle = ::CreateIoCompletionPort(m_platformImpl->m_directoryHandle, nullptr, 1, 0);
if (m_platformImpl->m_ioHandle != INVALID_HANDLE_VALUE)
{
m_shutdownThreadSignal = false;
m_thread = std::thread(std::bind(&FolderRootWatch::WatchFolderLoop, this));
return true;
}
}
return false;
}
void FolderRootWatch::Stop()
{
m_shutdownThreadSignal = true;
CloseHandle(m_platformImpl->m_ioHandle);
m_platformImpl->m_ioHandle = nullptr;
if (m_thread.joinable())
{
m_thread.join(); // wait for the thread to finish
m_thread = std::thread(); //destroy
}
CloseHandle(m_platformImpl->m_directoryHandle);
m_platformImpl->m_directoryHandle = nullptr;
}
void FolderRootWatch::WatchFolderLoop()
{
FILE_NOTIFY_INFORMATION aFileNotifyInformationList[50000];
QString path;
OVERLAPPED aOverlapped;
LPOVERLAPPED pOverlapped;
DWORD dwByteCount;
ULONG_PTR ulKey;
while (!m_shutdownThreadSignal)
{
::memset(aFileNotifyInformationList, 0, sizeof(aFileNotifyInformationList));
::memset(&aOverlapped, 0, sizeof(aOverlapped));
if (::ReadDirectoryChangesW(m_platformImpl->m_directoryHandle, aFileNotifyInformationList, sizeof(aFileNotifyInformationList), true, FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_ATTRIBUTES | FILE_NOTIFY_CHANGE_FILE_NAME, nullptr, &aOverlapped, nullptr))
{
//wait for up to a second for I/O to signal
dwByteCount = 0;
if (::GetQueuedCompletionStatus(m_platformImpl->m_ioHandle, &dwByteCount, &ulKey, &pOverlapped, INFINITE))
{
//if we are signaled to shutdown bypass
if (!m_shutdownThreadSignal && ulKey)
{
if (dwByteCount)
{
int offset = 0;
FILE_NOTIFY_INFORMATION* pFileNotifyInformation = aFileNotifyInformationList;
do
{
pFileNotifyInformation = (FILE_NOTIFY_INFORMATION*)((char*)pFileNotifyInformation + offset);
path.clear();
path.append(m_root);
path.append(QString::fromWCharArray(pFileNotifyInformation->FileName, pFileNotifyInformation->FileNameLength / 2));
QString file = QDir::toNativeSeparators(QDir::cleanPath(path));
switch (pFileNotifyInformation->Action)
{
case FILE_ACTION_ADDED:
case FILE_ACTION_RENAMED_NEW_NAME:
ProcessNewFileEvent(file);
break;
case FILE_ACTION_REMOVED:
case FILE_ACTION_RENAMED_OLD_NAME:
ProcessDeleteFileEvent(file);
break;
case FILE_ACTION_MODIFIED:
ProcessModifyFileEvent(file);
break;
}
offset = pFileNotifyInformation->NextEntryOffset;
} while (offset);
}
}
}
}
}
}
Code/Tools/AssetProcessor/Platform/Windows/assetprocessor_windows_files.cmake
+3 -1
View File
@@ -7,6 +7,8 @@
#
set(FILES
native/FileWatcher/FileWatcher_win.cpp
native/FileWatcher/FileWatcher_platform.h
native/FileWatcher/FileWatcher_windows.cpp
native/FileWatcher/FileWatcher_windows.h
native/resource.h
)
Code/Tools/AssetProcessor/Platform/Windows/native/FileWatcher/FileWatcher_platform.h
+11
View File
@@ -0,0 +1,11 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <native/FileWatcher/FileWatcher_windows.h>
Code/Tools/AssetProcessor/Platform/Windows/native/FileWatcher/FileWatcher_win.cpp
-130
View File
@@ -1,130 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <native/FileWatcher/FileWatcher.h>
#include <AzCore/PlatformIncl.h>
struct FolderRootWatch::PlatformImplementation
{
PlatformImplementation() : m_directoryHandle(nullptr), m_ioHandle(nullptr) { }
HANDLE m_directoryHandle;
HANDLE m_ioHandle;
};
//////////////////////////////////////////////////////////////////////////////
/// FolderWatchRoot
FolderRootWatch::FolderRootWatch(const QString rootFolder)
: m_root(rootFolder)
, m_shutdownThreadSignal(false)
, m_fileWatcher(nullptr)
, m_platformImpl(new PlatformImplementation())
{
}
FolderRootWatch::~FolderRootWatch()
{
// Destructor is required in here since this file contains the definition of struct PlatformImplementation
Stop();
delete m_platformImpl;
}
bool FolderRootWatch::Start()
{
m_platformImpl->m_directoryHandle = ::CreateFileW(m_root.toStdWString().data(), FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, nullptr);
if (m_platformImpl->m_directoryHandle != INVALID_HANDLE_VALUE)
{
m_platformImpl->m_ioHandle = ::CreateIoCompletionPort(m_platformImpl->m_directoryHandle, nullptr, 1, 0);
if (m_platformImpl->m_ioHandle != INVALID_HANDLE_VALUE)
{
m_shutdownThreadSignal = false;
m_thread = std::thread(std::bind(&FolderRootWatch::WatchFolderLoop, this));
return true;
}
}
return false;
}
void FolderRootWatch::Stop()
{
m_shutdownThreadSignal = true;
CloseHandle(m_platformImpl->m_ioHandle);
m_platformImpl->m_ioHandle = nullptr;
if (m_thread.joinable())
{
m_thread.join(); // wait for the thread to finish
m_thread = std::thread(); //destroy
}
CloseHandle(m_platformImpl->m_directoryHandle);
m_platformImpl->m_directoryHandle = nullptr;
}
void FolderRootWatch::WatchFolderLoop()
{
FILE_NOTIFY_INFORMATION aFileNotifyInformationList[50000];
QString path;
OVERLAPPED aOverlapped;
LPOVERLAPPED pOverlapped;
DWORD dwByteCount;
ULONG_PTR ulKey;
while (!m_shutdownThreadSignal)
{
::memset(aFileNotifyInformationList, 0, sizeof(aFileNotifyInformationList));
::memset(&aOverlapped, 0, sizeof(aOverlapped));
if (::ReadDirectoryChangesW(m_platformImpl->m_directoryHandle, aFileNotifyInformationList, sizeof(aFileNotifyInformationList), true, FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_ATTRIBUTES | FILE_NOTIFY_CHANGE_FILE_NAME, nullptr, &aOverlapped, nullptr))
{
//wait for up to a second for I/O to signal
dwByteCount = 0;
if (::GetQueuedCompletionStatus(m_platformImpl->m_ioHandle, &dwByteCount, &ulKey, &pOverlapped, INFINITE))
{
//if we are signaled to shutdown bypass
if (!m_shutdownThreadSignal && ulKey)
{
if (dwByteCount)
{
int offset = 0;
FILE_NOTIFY_INFORMATION* pFileNotifyInformation = aFileNotifyInformationList;
do
{
pFileNotifyInformation = (FILE_NOTIFY_INFORMATION*)((char*)pFileNotifyInformation + offset);
path.clear();
path.append(m_root);
path.append(QString::fromWCharArray(pFileNotifyInformation->FileName, pFileNotifyInformation->FileNameLength / 2));
QString file = QDir::toNativeSeparators(QDir::cleanPath(path));
switch (pFileNotifyInformation->Action)
{
case FILE_ACTION_ADDED:
case FILE_ACTION_RENAMED_NEW_NAME:
ProcessNewFileEvent(file);
break;
case FILE_ACTION_REMOVED:
case FILE_ACTION_RENAMED_OLD_NAME:
ProcessDeleteFileEvent(file);
break;
case FILE_ACTION_MODIFIED:
ProcessModifyFileEvent(file);
break;
}
offset = pFileNotifyInformation->NextEntryOffset;
} while (offset);
}
}
}
}
}
}
Code/Tools/AssetProcessor/Platform/Windows/native/FileWatcher/FileWatcher_windows.cpp
+153
View File
@@ -0,0 +1,153 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <AzCore/std/tuple.h>
#include <AzCore/std/utils.h>
#include <native/FileWatcher/FileWatcher.h>
#include <native/FileWatcher/FileWatcher_platform.h>
#include <QDir>
bool FileWatcher::PlatformStart()
{
m_shutdownThreadSignal = false;
bool allSucceeded = true;
for (const auto& [directory, recursive] : m_folderWatchRoots)
{
if (QDir(directory).exists())
{
allSucceeded &= m_platformImpl->AddWatchFolder(directory, recursive);
}
}
return allSucceeded;
}
bool FileWatcher::PlatformImplementation::AddWatchFolder(QString root, bool recursive)
{
HandleUniquePtr directoryHandle{::CreateFileW(
root.toStdWString().data(),
FILE_LIST_DIRECTORY,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
nullptr,
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED,
nullptr
)};
if (directoryHandle.get() == INVALID_HANDLE_VALUE)
{
AZ_Warning("FileWatcher", false, "Failed to start watching %s", root.toUtf8().constData());
return false;
}
// Associate this file handle with our existing io completion port handle
if (!::CreateIoCompletionPort(directoryHandle.get(), m_ioHandle.get(), /*CompletionKey =*/ static_cast<ULONG_PTR>(PlatformImplementation::EventType::FileRead), 1))
{
return false;
}
auto id = AZStd::make_unique<OVERLAPPED>();
auto* idp = id.get();
const auto& [folderWatch, inserted] = m_folderRootWatches.emplace(AZStd::piecewise_construct, AZStd::forward_as_tuple(idp),
AZStd::forward_as_tuple(AZStd::move(id), AZStd::move(directoryHandle), root, recursive));
if (!inserted)
{
return false;
}
return folderWatch->second.ReadChanges();
}
bool FileWatcher::PlatformImplementation::FolderRootWatch::ReadChanges()
{
// Register to get directory change notifications for our directory handle
return ::ReadDirectoryChangesW(
m_directoryHandle.get(),
&m_fileNotifyInformationList,
sizeof(m_fileNotifyInformationList),
m_recursive,
FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_ATTRIBUTES | FILE_NOTIFY_CHANGE_FILE_NAME,
nullptr,
m_overlapped.get(),
nullptr
);
}
void FileWatcher::PlatformStop()
{
m_shutdownThreadSignal = true;
// Send a special signal to the child thread, that is blocked in a GetQueuedCompletionStatus call, with a completion
// key set to Shutdown. The child thread will stop its processing when it receives this value for the completion key
PostQueuedCompletionStatus(m_platformImpl->m_ioHandle.get(), 0, /*CompletionKey =*/ static_cast<ULONG_PTR>(PlatformImplementation::EventType::Shutdown), nullptr);
if (m_thread.joinable())
{
m_thread.join(); // wait for the thread to finish
}
}
void FileWatcher::WatchFolderLoop()
{
LPOVERLAPPED directoryId = nullptr;
ULONG_PTR completionKey = 0;
while (!m_shutdownThreadSignal)
{
DWORD dwByteCount = 0;
if (::GetQueuedCompletionStatus(m_platformImpl->m_ioHandle.get(), &dwByteCount, &completionKey, &directoryId, INFINITE))
{
if (m_shutdownThreadSignal || completionKey == static_cast<ULONG_PTR>(PlatformImplementation::EventType::Shutdown))
{
break;
}
if (dwByteCount == 0)
{
continue;
}
const auto foundFolderRoot = m_platformImpl->m_folderRootWatches.find(directoryId);
if (foundFolderRoot == end(m_platformImpl->m_folderRootWatches))
{
continue;
}
PlatformImplementation::FolderRootWatch& folderRoot = foundFolderRoot->second;
// Initialize offset to 1 to ensure that the first iteration is always processed
DWORD offset = 1;
for (
const FILE_NOTIFY_INFORMATION* pFileNotifyInformation = reinterpret_cast<const FILE_NOTIFY_INFORMATION*>(&folderRoot.m_fileNotifyInformationList);
offset;
pFileNotifyInformation = reinterpret_cast<const FILE_NOTIFY_INFORMATION*>(reinterpret_cast<const char*>(pFileNotifyInformation) + offset)
){
const QString file = QDir::toNativeSeparators(QDir(folderRoot.m_directoryRoot)
.filePath(QString::fromWCharArray(pFileNotifyInformation->FileName, pFileNotifyInformation->FileNameLength / 2)));
switch (pFileNotifyInformation->Action)
{
case FILE_ACTION_ADDED:
case FILE_ACTION_RENAMED_NEW_NAME:
rawFileAdded(file, {});
break;
case FILE_ACTION_REMOVED:
case FILE_ACTION_RENAMED_OLD_NAME:
rawFileRemoved(file, {});
break;
case FILE_ACTION_MODIFIED:
rawFileModified(file, {});
break;
}
offset = pFileNotifyInformation->NextEntryOffset;
}
folderRoot.ReadChanges();
}
}
}
Code/Tools/AssetProcessor/Platform/Windows/native/FileWatcher/FileWatcher_windows.h
+65
View File
@@ -0,0 +1,65 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/typetraits/aligned_storage.h>
#include <AzCore/std/typetraits/remove_pointer.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <native/FileWatcher/FileWatcher.h>
#include <AzCore/PlatformIncl.h>
struct HandleDeleter
{
void operator()(HANDLE handle)
{
if (handle && handle != INVALID_HANDLE_VALUE)
{
CloseHandle(handle);
}
}
};
using HandleUniquePtr = AZStd::unique_ptr<AZStd::remove_pointer_t<HANDLE>, HandleDeleter>;
class FileWatcher::PlatformImplementation
{
public:
bool AddWatchFolder(QString folder, bool recursive);
struct FolderRootWatch
{
FolderRootWatch(AZStd::unique_ptr<OVERLAPPED>&& overlapped, HandleUniquePtr&& directoryHandle, QString root, bool recursive)
: m_overlapped(AZStd::move(overlapped))
, m_directoryHandle(AZStd::move(directoryHandle))
, m_directoryRoot(AZStd::move(root))
, m_recursive(recursive)
{
}
bool ReadChanges();
AZStd::unique_ptr<OVERLAPPED> m_overlapped; // Identifies this root watch
HandleUniquePtr m_directoryHandle;
QString m_directoryRoot;
bool m_recursive;
AZStd::aligned_storage_t<64 * 1024, sizeof(DWORD)> m_fileNotifyInformationList{};
};
enum class EventType
{
FileRead,
Shutdown
};
AZStd::unordered_map<LPOVERLAPPED, FolderRootWatch> m_folderRootWatches;
HandleUniquePtr m_ioHandle{CreateIoCompletionPort(INVALID_HANDLE_VALUE, nullptr, /*CompletionKey =*/ static_cast<ULONG_PTR>(EventType::FileRead), 1)};
};
Code/Tools/AssetProcessor/assetprocessor_static_files.cmake
-1
View File
@@ -44,7 +44,6 @@ set(FILES
native/FileProcessor/FileProcessor.h
native/FileWatcher/FileWatcher.cpp
native/FileWatcher/FileWatcher.h
native/FileWatcher/FileWatcherAPI.h
native/InternalBuilders/SettingsRegistryBuilder.cpp
native/InternalBuilders/SettingsRegistryBuilder.h
native/resourcecompiler/JobsModel.cpp
Code/Tools/AssetProcessor/native/FileWatcher/FileWatcher.cpp
+119 -130
View File
@@ -6,154 +6,119 @@
*
*/
#include "FileWatcher.h"
#include "AzCore/std/containers/vector.h"
#include <native/assetprocessor.h>
#include <native/FileWatcher/FileWatcher_platform.h>
#include <QFileInfo>
//////////////////////////////////////////////////////////////////////////////
/// FolderWatchRoot
void FolderRootWatch::ProcessNewFileEvent(const QString& file)
//! IsSubfolder(folderA, folderB)
//! returns whether folderA is a subfolder of folderB
//! assumptions: absolute paths
static bool IsSubfolder(const QString& folderA, const QString& folderB)
{
FileChangeInfo info;
info.m_action = FileAction::FileAction_Added;
info.m_filePath = file;
const bool invoked = QMetaObject::invokeMethod(m_fileWatcher, "AnyFileChange", Qt::QueuedConnection, Q_ARG(FileChangeInfo, info));
Q_ASSERT(invoked);
}
// lets avoid allocating or messing with memory - this is a MAJOR hotspot as it is called for any file change even in the cache!
if (folderA.length() <= folderB.length())
{
return false;
}
void FolderRootWatch::ProcessDeleteFileEvent(const QString& file)
{
FileChangeInfo info;
info.m_action = FileAction::FileAction_Removed;
info.m_filePath = file;
const bool invoked = QMetaObject::invokeMethod(m_fileWatcher, "AnyFileChange", Qt::QueuedConnection, Q_ARG(FileChangeInfo, info));
Q_ASSERT(invoked);
}
using AZStd::begin;
using AZStd::end;
void FolderRootWatch::ProcessModifyFileEvent(const QString& file)
{
FileChangeInfo info;
info.m_action = FileAction::FileAction_Modified;
info.m_filePath = file;
const bool invoked = QMetaObject::invokeMethod(m_fileWatcher, "AnyFileChange", Qt::QueuedConnection, Q_ARG(FileChangeInfo, info));
Q_ASSERT(invoked);
constexpr auto isSlash = [](const QChar c) constexpr
{
return c == AZ::IO::WindowsPathSeparator || c == AZ::IO::PosixPathSeparator;
};
const auto firstPathSeparator = AZStd::find_if(begin(folderB), end(folderB), [&isSlash](const QChar c)
{
return isSlash(c);
});
// Follow the convention used by AZ::IO::Path, and use a case-sensitive comparison on Posix paths
const bool useCaseSensitiveCompare = (firstPathSeparator == end(folderB)) ? true : (*firstPathSeparator == AZ::IO::PosixPathSeparator);
return AZStd::equal(begin(folderB), end(folderB), begin(folderA), [isSlash, useCaseSensitiveCompare](const QChar charAtB, const QChar charAtA)
{
if (isSlash(charAtA))
{
return isSlash(charAtB);
}
if (useCaseSensitiveCompare)
{
return charAtA == charAtB;
}
return charAtA.toLower() == charAtB.toLower();
});
}
//////////////////////////////////////////////////////////////////////////
/// FileWatcher
FileWatcher::FileWatcher()
: m_nextHandle(0)
: m_platformImpl(AZStd::make_unique<PlatformImplementation>())
{
qRegisterMetaType<FileChangeInfo>("FileChangeInfo");
auto makeFilter = [this](auto signal)
{
return [this, signal](QString path)
{
const auto foundWatchRoot = AZStd::find_if(begin(m_folderWatchRoots), end(m_folderWatchRoots), [path](const WatchRoot& watchRoot)
{
return Filter(path, watchRoot);
});
if (foundWatchRoot == end(m_folderWatchRoots))
{
return;
}
AZStd::invoke(signal, this, path);
};
};
// The rawFileAdded signals are emitted by the watcher thread. Use a queued
// connection so that the consumers of the notification process the
// notification on the main thread.
connect(this, &FileWatcher::rawFileAdded, this, makeFilter(&FileWatcher::fileAdded), Qt::QueuedConnection);
connect(this, &FileWatcher::rawFileRemoved, this, makeFilter(&FileWatcher::fileRemoved), Qt::QueuedConnection);
connect(this, &FileWatcher::rawFileModified, this, makeFilter(&FileWatcher::fileModified), Qt::QueuedConnection);
}
FileWatcher::~FileWatcher()
{
disconnect();
StopWatching();
}
int FileWatcher::AddFolderWatch(FolderWatchBase* pFolderWatch)
void FileWatcher::AddFolderWatch(QString directory, bool recursive)
{
if (!pFolderWatch)
// Search for an already monitored root that is a parent of `directory`,
// that is already watching subdirectories recursively
const auto found = AZStd::find_if(begin(m_folderWatchRoots), end(m_folderWatchRoots), [directory](const WatchRoot& root)
{
return -1;
return root.m_recursive && IsSubfolder(directory, root.m_directory);
});
if (found != end(m_folderWatchRoots))
{
// This directory is already watched
return;
}
FolderRootWatch* pFolderRootWatch = nullptr;
//create a new root and start listening for changes
m_folderWatchRoots.push_back({directory, recursive});
//see if this a sub folder of an already watched root
for (auto rootsIter = m_folderWatchRoots.begin(); !pFolderRootWatch && rootsIter != m_folderWatchRoots.end(); ++rootsIter)
//since we created a new root, see if the new root is a super folder
//of other roots, if it is then then fold those roots into the new super root
if (recursive)
{
if (FolderWatchBase::IsSubfolder(pFolderWatch->m_folder, (*rootsIter)->m_root))
AZStd::erase_if(m_folderWatchRoots, [directory](const WatchRoot& root)
{
pFolderRootWatch = *rootsIter;
}
return IsSubfolder(root.m_directory, directory);
});
}
bool bCreatedNewRoot = false;
//if its not a sub folder
if (!pFolderRootWatch)
{
//create a new root and start listening for changes
pFolderRootWatch = new FolderRootWatch(pFolderWatch->m_folder);
//make sure the folder watcher(s) get deleted before this
pFolderRootWatch->setParent(this);
bCreatedNewRoot = true;
}
pFolderRootWatch->m_fileWatcher = this;
QObject::connect(this, &FileWatcher::AnyFileChange, pFolderWatch, &FolderWatchBase::OnAnyFileChange);
if (bCreatedNewRoot)
{
if (m_startedWatching)
{
pFolderRootWatch->Start();
}
//since we created a new root, see if the new root is a super folder
//of other roots, if it is then then fold those roots into the new super root
for (auto rootsIter = m_folderWatchRoots.begin(); rootsIter != m_folderWatchRoots.end(); )
{
if (FolderWatchBase::IsSubfolder((*rootsIter)->m_root, pFolderWatch->m_folder))
{
//union the sub folder map over to the new root
pFolderRootWatch->m_subFolderWatchesMap.insert((*rootsIter)->m_subFolderWatchesMap);
//clear the old root sub folders map so they don't get deleted when we
//delete the old root as they are now pointed to by the new root
(*rootsIter)->m_subFolderWatchesMap.clear();
//delete the empty old root, deleting a root will call Stop()
//automatically which kills the thread
delete *rootsIter;
//remove the old root pointer form the watched list
rootsIter = m_folderWatchRoots.erase(rootsIter);
}
else
{
++rootsIter;
}
}
//add the new root to the watched roots
m_folderWatchRoots.push_back(pFolderRootWatch);
}
//add to the root
pFolderRootWatch->m_subFolderWatchesMap.insert(m_nextHandle, pFolderWatch);
m_nextHandle++;
return m_nextHandle - 1;
}
void FileWatcher::RemoveFolderWatch(int handle)
void FileWatcher::ClearFolderWatches()
{
for (auto rootsIter = m_folderWatchRoots.begin(); rootsIter != m_folderWatchRoots.end(); )
{
//find an element by the handle
auto foundIter = (*rootsIter)->m_subFolderWatchesMap.find(handle);
if (foundIter != (*rootsIter)->m_subFolderWatchesMap.end())
{
//remove the element
(*rootsIter)->m_subFolderWatchesMap.erase(foundIter);
//we removed a folder watch, if it's empty then there is no reason to keep watching it.
if ((*rootsIter)->m_subFolderWatchesMap.empty())
{
delete(*rootsIter);
rootsIter = m_folderWatchRoots.erase(rootsIter);
}
else
{
++rootsIter;
}
}
else
{
++rootsIter;
}
}
m_folderWatchRoots.clear();
}
void FileWatcher::StartWatching()
@@ -164,12 +129,18 @@ void FileWatcher::StartWatching()
return;
}
for (FolderRootWatch* root : m_folderWatchRoots)
if (PlatformStart())
{
root->Start();
m_thread = AZStd::thread({/*.name=*/ "AssetProcessor FileWatcher thread"}, [this]{
WatchFolderLoop();
});
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "File Change Monitoring started.\n");
}
else
{
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "File Change Monitoring failed to start.\n");
}
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "File Change Monitoring started.\n");
m_startedWatching = true;
}
@@ -177,17 +148,35 @@ void FileWatcher::StopWatching()
{
if (!m_startedWatching)
{
AZ_Warning("FileWatcher", false, "StartWatching() called when is not watching for file changes.");
AZ_Warning("FileWatcher", false, "StopWatching() called when is not watching for file changes.");
return;
}
for (FolderRootWatch* root : m_folderWatchRoots)
{
root->Stop();
}
PlatformStop();
m_startedWatching = false;
}
#include "native/FileWatcher/moc_FileWatcher.cpp"
#include "native/FileWatcher/moc_FileWatcherAPI.cpp"
bool FileWatcher::Filter(QString path, const WatchRoot& watchRoot)
{
if (!IsSubfolder(path, watchRoot.m_directory))
{
return false;
}
if (!watchRoot.m_recursive)
{
// filter out subtrees too.
QStringRef subRef = path.rightRef(path.length() - watchRoot.m_directory.length());
if ((subRef.indexOf('/') != -1) || (subRef.indexOf('\\') != -1))
{
return false; // filter this out.
}
// we don't care about subdirs. IsDir is more expensive so we do it after the above filter.
if (QFileInfo(path).isDir())
{
return false;
}
}
return true;
}
Code/Tools/AssetProcessor/native/FileWatcher/FileWatcher.h
+40 -57
View File
@@ -5,62 +5,21 @@
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#ifndef FILEWATCHER_COMPONENT_H
#define FILEWATCHER_COMPONENT_H
//////////////////////////////////////////////////////////////////////////
#pragma once
#if !defined(Q_MOC_RUN)
#include "FileWatcherAPI.h"
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/parallel/atomic.h>
#include <AzCore/std/parallel/thread.h>
#include <QMap>
#include <QVector>
#include <QString>
#include <QObject>
#include <thread>
#endif
class FileWatcher;
//////////////////////////////////////////////////////////////////////////
//! FolderRootWatch
/*! Class used for holding a point in the files system from which file changes are tracked.
* */
class FolderRootWatch
: public QObject
{
Q_OBJECT
friend class FileWatcher;
public:
FolderRootWatch(const QString rootFolder);
virtual ~FolderRootWatch();
void ProcessNewFileEvent(const QString& file);
void ProcessDeleteFileEvent(const QString& file);
void ProcessModifyFileEvent(const QString& file);
void ProcessRenameFileEvent(const QString& fileOld, const QString& fileNew);
public Q_SLOTS:
bool Start();
void Stop();
private:
void WatchFolderLoop();
private:
std::thread m_thread;
QString m_root;
QMap<int, FolderWatchBase*> m_subFolderWatchesMap;
volatile bool m_shutdownThreadSignal;
FileWatcher* m_fileWatcher;
// Can't use unique_ptr because this is a QObject and Qt's magic sauce is
// unable to determine the size of the unique_ptr and so fails to compile
struct PlatformImplementation;
PlatformImplementation* m_platformImpl;
};
//////////////////////////////////////////////////////////////////////////
//! FileWatcher
/*! Class that handles creation and deletion of FolderRootWatches based on
@@ -73,23 +32,47 @@ class FileWatcher
public:
FileWatcher();
virtual ~FileWatcher();
~FileWatcher() override;
//////////////////////////////////////////////////////////////////////////
virtual int AddFolderWatch(FolderWatchBase* pFolderWatch);
virtual void RemoveFolderWatch(int handle);
void AddFolderWatch(QString directory, bool recursive = true);
void ClearFolderWatches();
//////////////////////////////////////////////////////////////////////////
void StartWatching();
void StopWatching();
Q_SIGNALS:
void AnyFileChange(FileChangeInfo info);
// These signals are emitted when a file under a watched path changes
void fileAdded(QString filePath);
void fileRemoved(QString filePath);
void fileModified(QString filePath);
// These signals are emitted by the platform implementations when files
// change. Some platforms' file watch APIs do not support non-recursive
// watches, so the signals are filtered before being forwarded to the
// non-"raw" fileAdded/Removed/Modified signals above.
void rawFileAdded(QString filePath, QPrivateSignal);
void rawFileRemoved(QString filePath, QPrivateSignal);
void rawFileModified(QString filePath, QPrivateSignal);
private:
int m_nextHandle;
AZStd::vector<FolderRootWatch*> m_folderWatchRoots;
bool m_startedWatching = false;
};
bool PlatformStart();
void PlatformStop();
void WatchFolderLoop();
#endif//FILEWATCHER_COMPONENT_H
class PlatformImplementation;
friend class PlatformImplementation;
struct WatchRoot
{
QString m_directory;
bool m_recursive;
};
static bool Filter(QString path, const WatchRoot& watchRoot);
AZStd::unique_ptr<PlatformImplementation> m_platformImpl;
AZStd::vector<WatchRoot> m_folderWatchRoots;
AZStd::thread m_thread;
bool m_startedWatching = false;
AZStd::atomic_bool m_shutdownThreadSignal = false;
};
Code/Tools/AssetProcessor/native/FileWatcher/FileWatcherAPI.h
-222
View File
@@ -1,222 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#ifndef FILEWATCHERAPI_H
#define FILEWATCHERAPI_H
#include <QString>
#include <QObject>
#include <QDir>
//////////////////////////////////////////////////////////////////////////
//! FileAction
/*! Enum for which file changes are tracked.
* */
enum FileAction
{
FileAction_None = 0x00,
FileAction_Added = 0x01,
FileAction_Removed = 0x02,
FileAction_Modified = 0x04,
FileAction_Any = 0xFF,
};
inline FileAction operator | (FileAction a, FileAction b)
{
return static_cast<FileAction>(static_cast<int>(a) | static_cast<int>(b));
}
inline FileAction operator & (FileAction a, FileAction b)
{
return static_cast<FileAction>(static_cast<int>(a) & static_cast<int>(b));
}
//////////////////////////////////////////////////////////////////////////
//! FileChangeInfo
/*! Struct for passing along information about file changes.
* */
struct FileChangeInfo
{
FileChangeInfo()
: m_action(FileAction::FileAction_None)
{}
FileChangeInfo(const FileChangeInfo& rhs)
: m_action(rhs.m_action)
, m_filePath(rhs.m_filePath)
, m_filePathOld(rhs.m_filePathOld)
{
}
FileAction m_action;
QString m_filePath;
QString m_filePathOld;
};
Q_DECLARE_METATYPE(FileChangeInfo)
//////////////////////////////////////////////////////////////////////////
//! FolderWatchBase
/*! Class for filtering file changes generated from a root watch. Define your own
*! custom filtering by deriving from this base class and implement your own
*! custom code for what to do when receiving a file change notification.
* */
class FolderWatchBase
: public QObject
{
Q_OBJECT
public:
FolderWatchBase(const QString strFolder, bool bWatchSubtree = true, FileAction fileAction = FileAction::FileAction_Any)
: m_folder(strFolder)
, m_watchSubtree(bWatchSubtree)
, m_fileAction(fileAction)
{
m_folder = QDir::toNativeSeparators(QDir::cleanPath(m_folder) + "/");
}
//! IsSubfolder(folderA, folderB)
//! returns whether folderA is a subfolder of folderB
//! assumptions: absolute paths, case insensitive
static bool IsSubfolder(const QString& folderA, const QString& folderB)
{
// lets avoid allocating or messing with memory - this is a MAJOR hotspot as it is called for any file change even in the cache!
int sizeB = folderB.length();
int sizeA = folderA.length();
if (sizeA <= sizeB)
{
return false;
}
QChar slash1 = QChar('\\');
QChar slash2 = QChar('/');
int posA = 0;
// A is going to be the longer one, so use B:
for (int idx = 0; idx < sizeB; ++idx)
{
QChar charAtA = folderA.at(posA);
QChar charAtB = folderB.at(idx);
if ((charAtB == slash1) || (charAtB == slash2))
{
if ((charAtA != slash1) && (charAtA != slash2))
{
return false;
}
++posA;
}
else
{
if (charAtA.toLower() != charAtB.toLower())
{
return false;
}
++posA;
}
}
return true;
}
QString m_folder;
bool m_watchSubtree;
FileAction m_fileAction;
public Q_SLOTS:
void OnAnyFileChange(FileChangeInfo info)
{
//if they set a file action then respect it by rejecting non matching file actions
if (info.m_action & m_fileAction)
{
//is the file is in the folder or subtree (if specified) then call OnFileChange
if (FolderWatchBase::IsSubfolder(info.m_filePath, m_folder))
{
OnFileChange(info);
}
}
}
virtual void OnFileChange(const FileChangeInfo& info) = 0;
};
//////////////////////////////////////////////////////////////////////////
//! FolderWatchCallbackEx
/*! Class implements a more complex filtering that can optionally filter for file
*! extension and call different callback for different kinds of file changes
*! generated from a root watch.
*! Notes:
*! - empty extension "" catches all file changes
*! - extension should not include the leading "."
* */
class FolderWatchCallbackEx
: public FolderWatchBase
{
Q_OBJECT
public:
FolderWatchCallbackEx(const QString strFolder, const QString extension, bool bWatchSubtree)
: FolderWatchBase(strFolder, bWatchSubtree)
, m_extension(extension)
{
}
QString m_extension;
//on file change call the change callback if passes extension then route
//to specific file action type callback
virtual void OnFileChange(const FileChangeInfo& info)
{
//if they set an extension to watch for only let matching extensions through
QFileInfo fileInfo(info.m_filePath);
if (!m_watchSubtree)
{
// filter out subtrees too.
QStringRef subRef = info.m_filePath.rightRef(info.m_filePath.length() - m_folder.length());
if ((subRef.indexOf('/') != -1) || (subRef.indexOf('\\') != -1))
{
return; // filter this out.
}
// we don't care about subdirs. IsDir is more expensive so we do it after the above filter.
if (fileInfo.isDir())
{
return;
}
}
if (m_extension.isEmpty() || fileInfo.completeSuffix().compare(m_extension, Qt::CaseInsensitive) == 0)
{
if (info.m_action & FileAction::FileAction_Any)
{
Q_EMIT fileChange(info);
}
if (info.m_action & FileAction::FileAction_Added)
{
Q_EMIT fileAdded(info.m_filePath);
}
if (info.m_action & FileAction::FileAction_Removed)
{
Q_EMIT fileRemoved(info.m_filePath);
}
if (info.m_action & FileAction::FileAction_Modified)
{
Q_EMIT fileModified(info.m_filePath);
}
}
}
Q_SIGNALS:
void fileChange(FileChangeInfo info);
void fileAdded(QString filePath);
void fileRemoved(QString filePath);
void fileModified(QString filePath);
};
#endif//FILEWATCHERAPI_H
Code/Tools/AssetProcessor/native/unittests/FileWatcherUnitTests.cpp
+7 -9
View File
@@ -24,14 +24,12 @@ void FileWatcherUnitTestRunner::StartTest()
FileWatcher fileWatcher;
FolderWatchCallbackEx folderWatch(tempPath, "", true);
fileWatcher.AddFolderWatch(&folderWatch);
fileWatcher.AddFolderWatch(tempPath);
fileWatcher.StartWatching();
{ // test a single file create/write
bool foundFile = false;
auto connection = QObject::connect(&folderWatch, &FolderWatchCallbackEx::fileAdded, this, [&](QString filename)
auto connection = QObject::connect(&fileWatcher, &FileWatcher::fileAdded, this, [&](QString filename)
{
AZ_TracePrintf(AssetProcessor::DebugChannel, "Single file test Found asset: %s.\n", filename.toUtf8().data());
foundFile = true;
@@ -66,7 +64,7 @@ void FileWatcherUnitTestRunner::StartTest()
const unsigned long maxFiles = 10000;
QSet<QString> outstandingFiles;
auto connection = QObject::connect(&folderWatch, &FolderWatchCallbackEx::fileAdded, this, [&](QString filename)
auto connection = QObject::connect(&fileWatcher, &FileWatcher::fileAdded, this, [&](QString filename)
{
outstandingFiles.remove(filename);
});
@@ -122,7 +120,7 @@ void FileWatcherUnitTestRunner::StartTest()
{ // test deletion
bool foundFile = false;
auto connection = QObject::connect(&folderWatch, &FolderWatchCallbackEx::fileRemoved, this, [&](QString filename)
auto connection = QObject::connect(&fileWatcher, &FileWatcher::fileRemoved, this, [&](QString filename)
{
AZ_TracePrintf(AssetProcessor::DebugChannel, "Deleted asset: %s...\n", filename.toUtf8().data());
foundFile = true;
@@ -155,7 +153,7 @@ void FileWatcherUnitTestRunner::StartTest()
{
bool fileAddCalled = false;
QString fileAddName;
auto connectionAdd = QObject::connect(&folderWatch, &FolderWatchCallbackEx::fileAdded, this, [&](QString filename)
auto connectionAdd = QObject::connect(&fileWatcher, &FileWatcher::fileAdded, this, [&](QString filename)
{
fileAddCalled = true;
fileAddName = filename;
@@ -163,7 +161,7 @@ void FileWatcherUnitTestRunner::StartTest()
bool fileRemoveCalled = false;
QString fileRemoveName;
auto connectionRemove = QObject::connect(&folderWatch, &FolderWatchCallbackEx::fileRemoved, this, [&](QString filename)
auto connectionRemove = QObject::connect(&fileWatcher, &FileWatcher::fileRemoved, this, [&](QString filename)
{
fileRemoveCalled = true;
fileRemoveName = filename;
@@ -171,7 +169,7 @@ void FileWatcherUnitTestRunner::StartTest()
QStringList fileModifiedNames;
bool fileModifiedCalled = false;
auto connectionModified = QObject::connect(&folderWatch, &FolderWatchCallbackEx::fileModified, this, [&](QString filename)
auto connectionModified = QObject::connect(&fileWatcher, &FileWatcher::fileModified, this, [&](QString filename)
{
fileModifiedCalled = true;
fileModifiedNames.append(filename);
Code/Tools/AssetProcessor/native/utilities/ApplicationManager.h
-1
View File
@@ -21,7 +21,6 @@
#include "native/assetprocessor.h"
#endif
class FolderWatchCallbackEx;
class QCoreApplication;
namespace AZ
Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp
+55 -43
View File
@@ -433,63 +433,77 @@ void ApplicationManagerBase::DestroyPlatformConfiguration()
void ApplicationManagerBase::InitFileMonitor()
{
m_folderWatches.reserve(m_platformConfiguration->GetScanFolderCount());
m_watchHandles.reserve(m_platformConfiguration->GetScanFolderCount());
for (int folderIdx = 0; folderIdx < m_platformConfiguration->GetScanFolderCount(); ++folderIdx)
{
const AssetProcessor::ScanFolderInfo& info = m_platformConfiguration->GetScanFolderAt(folderIdx);
FolderWatchCallbackEx* newFolderWatch = new FolderWatchCallbackEx(info.ScanPath(), "", info.RecurseSubFolders());
// hook folder watcher to assess files on add/modify
// relevant files will be sent to resource compiler
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded,
m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssessAddedFile);
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileModified,
m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssessModifiedFile);
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved,
m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssessDeletedFile);
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded, [this](QString path) { m_fileStateCache->AddFile(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileModified, [this](QString path) { m_fileStateCache->UpdateFile(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved, [this](QString path) { m_fileStateCache->RemoveFile(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded, [](QString path) { AZ::Interface<AssetProcessor::ExcludedFolderCacheInterface>::Get()->FileAdded(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded,
m_fileProcessor.get(), &AssetProcessor::FileProcessor::AssessAddedFile);
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved,
m_fileProcessor.get(), &AssetProcessor::FileProcessor::AssessDeletedFile);
m_folderWatches.push_back(AZStd::unique_ptr<FolderWatchCallbackEx>(newFolderWatch));
m_watchHandles.push_back(m_fileWatcher.AddFolderWatch(newFolderWatch));
m_fileWatcher.AddFolderWatch(info.ScanPath(), info.RecurseSubFolders());
}
// also hookup monitoring for the cache (output directory)
QDir cacheRoot;
if (AssetUtilities::ComputeProjectCacheRoot(cacheRoot))
{
FolderWatchCallbackEx* newFolderWatch = new FolderWatchCallbackEx(cacheRoot.absolutePath(), "", true);
m_fileWatcher.AddFolderWatch(cacheRoot.absolutePath(), true);
}
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded, [this](QString path) { m_fileStateCache->AddFile(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileModified, [this](QString path) { m_fileStateCache->UpdateFile(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved, [this](QString path) { m_fileStateCache->RemoveFile(path); });
if (m_platformConfiguration->GetScanFolderCount() || !cacheRoot.path().isEmpty())
{
const auto cachePath = QDir::toNativeSeparators(cacheRoot.absolutePath());
// we only care about cache root deletions.
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved,
m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssessDeletedFile);
const auto OnFileAdded = [this, cachePath](QString path)
{
const bool isCacheRoot = path.startsWith(cachePath);
if (isCacheRoot)
{
m_fileStateCache->AddFile(path);
}
else
{
m_assetProcessorManager->AssessAddedFile(path);
m_fileStateCache->AddFile(path);
AZ::Interface<AssetProcessor::ExcludedFolderCacheInterface>::Get()->FileAdded(path);
m_fileProcessor->AssetProcessor::FileProcessor::AssessAddedFile(path);
}
};
m_folderWatches.push_back(AZStd::unique_ptr<FolderWatchCallbackEx>(newFolderWatch));
m_watchHandles.push_back(m_fileWatcher.AddFolderWatch(newFolderWatch));
const auto OnFileModified = [this, cachePath](QString path)
{
const bool isCacheRoot = path.startsWith(cachePath);
if (isCacheRoot)
{
m_assetProcessorManager->AssessModifiedFile(path);
}
else
{
m_assetProcessorManager->AssessModifiedFile(path);
m_fileStateCache->UpdateFile(path);
}
};
const auto OnFileRemoved = [this, cachePath](QString path)
{
const bool isCacheRoot = path.startsWith(cachePath);
if (isCacheRoot)
{
m_fileStateCache->RemoveFile(path);
m_assetProcessorManager->AssessDeletedFile(path);
}
else
{
m_assetProcessorManager->AssessDeletedFile(path);
m_fileStateCache->RemoveFile(path);
m_fileProcessor->AssessDeletedFile(path);
}
};
connect(&m_fileWatcher, &FileWatcher::fileAdded, OnFileAdded);
connect(&m_fileWatcher, &FileWatcher::fileModified, OnFileModified);
connect(&m_fileWatcher, &FileWatcher::fileRemoved, OnFileRemoved);
}
}
void ApplicationManagerBase::DestroyFileMonitor()
{
for (int watchHandle : m_watchHandles)
{
m_fileWatcher.RemoveFolderWatch(watchHandle);
}
m_folderWatches.resize(0);
m_fileWatcher.ClearFolderWatches();
}
void ApplicationManagerBase::DestroyApplicationServer()
@@ -798,8 +812,6 @@ ApplicationManager::BeforeRunStatus ApplicationManagerBase::BeforeRun()
qRegisterMetaType<AzFramework::AssetSystem::AssetStatus>("AzFramework::AssetSystem::AssetStatus");
qRegisterMetaType<AzFramework::AssetSystem::AssetStatus>("AssetStatus");
qRegisterMetaType<FileChangeInfo>("FileChangeInfo");
qRegisterMetaType<AssetProcessor::AssetScanningStatus>("AssetScanningStatus");
qRegisterMetaType<AssetProcessor::NetworkRequestID>("NetworkRequestID");
Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h
-3
View File
@@ -47,7 +47,6 @@ namespace AssetProcessor
class ApplicationServer;
class ConnectionManager;
class FolderWatchCallbackEx;
class ControlRequestHandler;
class ApplicationManagerBase
@@ -192,9 +191,7 @@ protected:
bool m_sourceControlReady = false;
bool m_fullIdle = false;
AZStd::vector<AZStd::unique_ptr<FolderWatchCallbackEx> > m_folderWatches;
FileWatcher m_fileWatcher;
AZStd::vector<int> m_watchHandles;
AssetProcessor::PlatformConfiguration* m_platformConfiguration = nullptr;
AssetProcessor::AssetProcessorManager* m_assetProcessorManager = nullptr;
AssetProcessor::AssetCatalog* m_assetCatalog = nullptr;
Code/Tools/AssetProcessor/native/utilities/AssetBuilderInfo.h
-1
View File
@@ -23,7 +23,6 @@
#include <native/resourcecompiler/RCBuilder.h>
#include <AssetBuilder/AssetBuilderInfo.h>
class FolderWatchCallbackEx;
class QCoreApplication;
namespace AssetProcessor