birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[AssetProcessor] Refactor the FileWatcher to use only one watch thread

Browse Source 
This change reworks the AssetProcessor's FileWatcher so that it only uses
one thread. This is motivated by getting better support for inotify on
Linux. The previous architecture required calling `inotify_init` once for
each directory that was being watched, and using separate inotify instances
for each watched tree. In addition, having separate threads per watched
tree is not necessary, and just consumes system resources. Each platform
supports watching multiple directories with the same platform-specific
watcher API, so each platform has been updated accordingly.

The interface to the FileWatcher class is greatly simplified. Previously,
it supported client-supplied filtering of the paths that would generate
notifications. This was done by subclassing `FolderWatchBase` and
implementing `OnFileChange`. However, only one filter was ever used, so
that filter is now hard-coded in the FileWatcher class, and the classes
driving the old filtering mechanism are removed. Users of the interface
now have a much easier time, they just call `AddFolderWatch` with the path
to watch, and only have to connect to one set of signals, instead of
separate signals per watched directory.

Signed-off-by: Chris Burel <burelc@amazon.com>
This commit is contained in:
Chris Burel
2021-11-15 11:30:37 -08:00
parent 7ac5bc3d5c
commit ce0bb1ca2b
22 changed files with 658 additions and 932 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/AssetProcessor/Platform/Linux/assetprocessor_linux_files.cmake
+2
View File
@@ -8,4 +8,6 @@
set(FILES
native/FileWatcher/FileWatcher_linux.cpp
native/FileWatcher/FileWatcher_linux.h
native/FileWatcher/FileWatcher_platform.h
)
Code/Tools/AssetProcessor/Platform/Linux/native/FileWatcher/FileWatcher_linux.cpp
+137 -188
View File
@@ -5,7 +5,10 @@
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <AzCore/std/string/fixed_string.h>
#include <native/FileWatcher/FileWatcher.h>
#include <native/FileWatcher/FileWatcher_platform.h>
#include <QDirIterator>
#include <QHash>
@@ -15,180 +18,127 @@
#include <sys/inotify.h>
static constexpr int s_handleToFolderMapLockTimeout = 1000; // 1 sec timeout for obtaining the handle to folder map lock
static constexpr size_t s_iNotifyMaxEntries = 1024 * 16; // Control the maximum number of entries (from inotify) that can be read at one time
static constexpr size_t s_iNotifyEventSize = sizeof(struct inotify_event);
static constexpr size_t s_iNotifyReadBufferSize = s_iNotifyMaxEntries * s_iNotifyEventSize;
static constexpr size_t s_inotifyMaxEntries = 1024 * 16; // Control the maximum number of entries (from inotify) that can be read at one time
static constexpr size_t s_inotifyEventSize = sizeof(struct inotify_event);
static constexpr size_t s_inotifyReadBufferSize = s_inotifyMaxEntries * s_inotifyEventSize;
struct FolderRootWatch::PlatformImplementation
bool FileWatcher::PlatformImplementation::Initialize()
{
PlatformImplementation() = default;
int m_iNotifyHandle = -1;
QMutex m_handleToFolderMapLock;
QHash<int, QString> m_handleToFolderMap;
bool Initialize()
if (m_inotifyHandle < 0)
{
if (m_iNotifyHandle < 0)
{
// The CLOEXEC flag prevents the inotify watchers from copying on fork/exec
m_iNotifyHandle = inotify_init1(IN_CLOEXEC);
const auto err = errno;
// The CLOEXEC flag prevents the inotify watchers from copying on fork/exec
m_inotifyHandle = inotify_init1(IN_CLOEXEC);
if (m_iNotifyHandle < 0)
{
AZ_Warning("FileWatcher", false, "Unable to initialize inotify, file monitoring will not be available: %s\n", strerror(err));
}
}
return (m_iNotifyHandle >= 0);
[[maybe_unused]] const auto err = errno;
[[maybe_unused]] AZStd::fixed_string<255> errorString;
AZ_Warning("FileWatcher", (m_inotifyHandle >= 0), "Unable to initialize inotify, file monitoring will not be available: %s\n", strerror_r(err, errorString.data(), errorString.capacity()));
}
void Finalize()
{
if (m_iNotifyHandle >= 0)
{
if (!m_handleToFolderMapLock.tryLock(s_handleToFolderMapLockTimeout))
{
AZ_Error("FileWatcher", false, "Unable to obtain inotify handle lock on thread");
return;
}
QHashIterator<int, QString> iter(m_handleToFolderMap);
while (iter.hasNext())
{
iter.next();
int watchHandle = iter.key();
inotify_rm_watch(m_iNotifyHandle, watchHandle);
}
m_handleToFolderMap.clear();
m_handleToFolderMapLock.unlock();
::close(m_iNotifyHandle);
m_iNotifyHandle = -1;
}
}
void AddWatchFolder(QString folder, bool recursive)
{
if (m_iNotifyHandle >= 0)
{
// Clean up the path before accepting it as a watch folder
QString cleanPath = QDir::cleanPath(folder);
// Add the folder to watch and track it
int watchHandle = inotify_add_watch(m_iNotifyHandle,
cleanPath.toUtf8().constData(),
IN_CREATE | IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY | IN_MOVE);
if (watchHandle < 0)
{
AZ_Error("FileWatcher", false, "inotify_add_watch failed for path %s", cleanPath.toUtf8().constData());
return;
}
if (!m_handleToFolderMapLock.tryLock(s_handleToFolderMapLockTimeout))
{
AZ_Error("FileWatcher", false, "Unable to obtain inotify handle lock on thread");
return;
}
m_handleToFolderMap[watchHandle] = cleanPath;
m_handleToFolderMapLock.unlock();
if (!recursive)
{
return;
}
// Add all the subfolders to watch and track them
QDirIterator dirIter(folder, QDirIterator::Subdirectories | QDirIterator::FollowSymlinks);
while (dirIter.hasNext())
{
QString dirName = dirIter.next();
if (dirName.endsWith("/.") || dirName.endsWith("/.."))
{
continue;
}
int watchHandle = inotify_add_watch(m_iNotifyHandle,
dirName.toUtf8().constData(),
IN_CREATE | IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY | IN_MOVE);
if (watchHandle < 0)
{
AZ_Error("FileWatcher", false, "inotify_add_watch failed for path %s", dirName.toUtf8().constData());
return;
}
if (!m_handleToFolderMapLock.tryLock(s_handleToFolderMapLockTimeout))
{
AZ_Error("FileWatcher", false, "Unable to obtain inotify handle lock on thread");
return;
}
m_handleToFolderMap[watchHandle] = dirName;
m_handleToFolderMapLock.unlock();
}
}
}
void RemoveWatchFolder(int watchHandle)
{
if (m_iNotifyHandle >= 0)
{
if (!m_handleToFolderMapLock.tryLock(s_handleToFolderMapLockTimeout))
{
AZ_Error("FileWatcher", false, "Unable to obtain inotify handle lock on thread");
return;
}
QHash<int, QString>::iterator handleToRemove = m_handleToFolderMap.find(watchHandle);
if (handleToRemove != m_handleToFolderMap.end())
{
inotify_rm_watch(m_iNotifyHandle, watchHandle);
m_handleToFolderMap.erase(handleToRemove);
}
m_handleToFolderMapLock.unlock();
}
}
};
//////////////////////////////////////////////////////////////////////////////
/// FolderWatchRoot
FolderRootWatch::FolderRootWatch(const QString rootFolder, bool recursive)
: m_root(rootFolder)
, m_shutdownThreadSignal(false)
, m_fileWatcher(nullptr)
, m_recursive(recursive)
, m_platformImpl(new PlatformImplementation())
{
return (m_inotifyHandle >= 0);
}
FolderRootWatch::~FolderRootWatch()
void FileWatcher::PlatformImplementation::Finalize()
{
// Destructor is required in here since this file contains the definition of struct PlatformImplementation
Stop();
if (m_inotifyHandle < 0)
{
return;
}
delete m_platformImpl;
{
QMutexLocker lock{&m_handleToFolderMapLock};
for (const auto& watchHandle : m_handleToFolderMap.keys())
{
inotify_rm_watch(m_inotifyHandle, watchHandle);
}
m_handleToFolderMap.clear();
}
::close(m_inotifyHandle);
m_inotifyHandle = -1;
}
bool FolderRootWatch::Start()
void FileWatcher::PlatformImplementation::AddWatchFolder(QString folder, bool recursive)
{
if (m_inotifyHandle < 0)
{
return;
}
// Clean up the path before accepting it as a watch folder
QString cleanPath = QDir::cleanPath(folder);
// Add the folder to watch and track it
int watchHandle = inotify_add_watch(m_inotifyHandle,
cleanPath.toUtf8().constData(),
IN_CREATE | IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY | IN_MOVE);
if (watchHandle < 0)
{
[[maybe_unused]] const auto err = errno;
[[maybe_unused]] AZStd::fixed_string<255> errorString;
AZ_Warning("FileWatcher", false, "inotify_add_watch failed for path %s: %s", cleanPath.toUtf8().constData(), strerror_r(err, errorString.data(), errorString.capacity()));
return;
}
{
QMutexLocker lock{&m_handleToFolderMapLock};
m_handleToFolderMap[watchHandle] = cleanPath;
}
// Add all the contents (files and directories) to watch and track them
QDirIterator dirIter(folder, QDir::NoDotAndDotDot | QDir::Dirs | QDir::Files, (recursive ? QDirIterator::Subdirectories : QDirIterator::NoIteratorFlags) | QDirIterator::FollowSymlinks);
while (dirIter.hasNext())
{
QString dirName = dirIter.next();
watchHandle = inotify_add_watch(m_inotifyHandle,
dirName.toUtf8().constData(),
IN_CREATE | IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY | IN_MOVE);
if (watchHandle < 0)
{
[[maybe_unused]] const auto err = errno;
[[maybe_unused]] AZStd::fixed_string<255> errorString;
AZ_Warning("FileWatcher", false, "inotify_add_watch failed for path %s: %s", dirName.toUtf8().constData(), strerror_r(err, errorString.data(), errorString.capacity()));
return;
}
QMutexLocker lock{&m_handleToFolderMapLock};
m_handleToFolderMap[watchHandle] = dirName;
}
}
void FileWatcher::PlatformImplementation::RemoveWatchFolder(int watchHandle)
{
if (m_inotifyHandle < 0)
{
return;
}
QMutexLocker lock{&m_handleToFolderMapLock};
if (m_handleToFolderMap.remove(watchHandle))
{
inotify_rm_watch(m_inotifyHandle, watchHandle);
}
}
bool FileWatcher::PlatformStart()
{
// inotify will be used by linux to monitor file changes within directories under the root folder
if (!m_platformImpl->Initialize())
{
return false;
}
m_platformImpl->AddWatchFolder(m_root, m_recursive);
m_shutdownThreadSignal = false;
if (m_platformImpl->m_iNotifyHandle >= 0)
for (const auto& [directory, recursive] : m_folderWatchRoots)
{
m_thread = std::thread([this]() { WatchFolderLoop(); });
if (QDir(directory).exists())
{
m_platformImpl->AddWatchFolder(directory, recursive);
}
}
return true;
}
void FolderRootWatch::Stop()
void FileWatcher::PlatformStop()
{
m_shutdownThreadSignal = true;
@@ -197,64 +147,63 @@ void FolderRootWatch::Stop()
if (m_thread.joinable())
{
m_thread.join(); // wait for the thread to finish
m_thread = std::thread(); //destroy
}
}
void FolderRootWatch::WatchFolderLoop()
void FileWatcher::WatchFolderLoop()
{
char eventBuffer[s_iNotifyReadBufferSize];
char eventBuffer[s_inotifyReadBufferSize];
while (!m_shutdownThreadSignal)
{
ssize_t bytesRead = ::read(m_platformImpl->m_iNotifyHandle, eventBuffer, s_iNotifyReadBufferSize);
ssize_t bytesRead = ::read(m_platformImpl->m_inotifyHandle, eventBuffer, s_inotifyReadBufferSize);
if (bytesRead < 0)
{
// Break out of the loop when the notify handle was closed (outside of this thread)
break;
}
else if (bytesRead > 0)
if (!bytesRead)
{
for (size_t index=0; index<bytesRead;)
continue;
}
for (size_t index=0; index<bytesRead;)
{
const auto* event = reinterpret_cast<inotify_event*>(&eventBuffer[index]);
if (event->mask & (IN_CREATE | IN_DELETE | IN_MODIFY | IN_MOVE))
{
struct inotify_event *event = ( struct inotify_event * ) &eventBuffer[ index ];
const QString pathStr = QDir(m_platformImpl->m_handleToFolderMap[event->wd]).absoluteFilePath(event->name);
if (event->mask & (IN_CREATE | IN_DELETE | IN_MODIFY | IN_MOVE ))
if (event->mask & (IN_CREATE | IN_MOVED_TO))
{
QString pathStr = QString("%1%2%3").arg(m_platformImpl->m_handleToFolderMap[event->wd], QDir::separator(), event->name);
if (event->mask & (IN_CREATE | IN_MOVED_TO))
if (event->mask & IN_ISDIR /*&& m_recursive*/)
{
if ( event->mask & IN_ISDIR && m_recursive)
{
// New Directory, add it to the watch
m_platformImpl->AddWatchFolder(pathStr, true);
}
else
{
ProcessNewFileEvent(pathStr);
}
// New Directory, add it to the watch
m_platformImpl->AddWatchFolder(pathStr, true);
}
else if (event->mask & (IN_DELETE | IN_MOVED_FROM))
else
{
if (event->mask & IN_ISDIR)
{
// Directory Deleted, remove it from the watch
m_platformImpl->RemoveWatchFolder(event->wd);
}
else
{
ProcessDeleteFileEvent(pathStr);
}
}
else if ((event->mask & IN_MODIFY) && ((event->mask & IN_ISDIR) != IN_ISDIR))
{
ProcessModifyFileEvent(pathStr);
rawFileAdded(pathStr, {});
}
}
index += s_iNotifyEventSize + event->len;
else if (event->mask & (IN_DELETE | IN_MOVED_FROM))
{
if (event->mask & IN_ISDIR)
{
// Directory Deleted, remove it from the watch
m_platformImpl->RemoveWatchFolder(event->wd);
}
else
{
rawFileRemoved(pathStr, {});
}
}
else if ((event->mask & IN_MODIFY) && ((event->mask & IN_ISDIR) != IN_ISDIR))
{
rawFileModified(pathStr, {});
}
}
index += s_inotifyEventSize + event->len;
}
}
}
Code/Tools/AssetProcessor/Platform/Linux/native/FileWatcher/FileWatcher_linux.h
+26
View File
@@ -0,0 +1,26 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <FileWatcher/FileWatcher.h>
#include <QMutex>
#include <QHash>
class FileWatcher::PlatformImplementation
{
public:
bool Initialize();
void Finalize();
void AddWatchFolder(QString folder, bool recursive);
void RemoveWatchFolder(int watchHandle);
int m_inotifyHandle = -1;
QMutex m_handleToFolderMapLock;
QHash<int, QString> m_handleToFolderMap;
};
Code/Tools/AssetProcessor/Platform/Linux/native/FileWatcher/FileWatcher_platform.h
+11
View File
@@ -0,0 +1,11 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <native/FileWatcher/FileWatcher_linux.h>
Code/Tools/AssetProcessor/Platform/Mac/assetprocessor_mac_files.cmake
+2
View File
@@ -8,4 +8,6 @@
set(FILES
native/FileWatcher/FileWatcher_macos.cpp
native/FileWatcher/FileWatcher_mac.h
native/FileWatcher/FileWatcher_platform.h
)
Code/Tools/AssetProcessor/Platform/Mac/native/FileWatcher/FileWatcher_mac.h
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <native/FileWatcher/FileWatcher.h>
#include <CoreServices/CoreServices.h>
class FileWatcher::PlatformImplementation
{
public:
FSEventStreamRef m_stream = nullptr;
CFRunLoopRef m_runLoop = nullptr;
};
Code/Tools/AssetProcessor/Platform/Mac/native/FileWatcher/FileWatcher_macos.cpp
+31 -53
View File
@@ -6,47 +6,23 @@
*
*/
#include <native/FileWatcher/FileWatcher.h>
#include <native/FileWatcher/FileWatcher_platform.h>
#include <native/utilities/BatchApplicationManager.h>
#include <AzCore/Debug/Trace.h>
#include <CoreServices/CoreServices.h>
void FileEventStreamCallback(ConstFSEventStreamRef streamRef, void *clientCallBackInfo, size_t numEvents, void *eventPaths, const FSEventStreamEventFlags eventFlags[], const FSEventStreamEventId eventIds[]);
struct FolderRootWatch::PlatformImplementation
{
PlatformImplementation() : m_stream(nullptr), m_runLoop(nullptr) { }
FSEventStreamRef m_stream;
CFRunLoopRef m_runLoop;
QString m_renameFileDirectory;
};
//////////////////////////////////////////////////////////////////////////////
/// FolderWatchRoot
FolderRootWatch::FolderRootWatch(const QString rootFolder)
: m_root(rootFolder)
, m_shutdownThreadSignal(false)
, m_fileWatcher(nullptr)
, m_platformImpl(new PlatformImplementation())
{
}
FolderRootWatch::~FolderRootWatch()
{
// Destructor is required in here since this file contains the definition of struct PlatformImplementation
Stop();
delete m_platformImpl;
}
bool FolderRootWatch::Start()
bool FileWatcher::PlatformStart()
{
m_shutdownThreadSignal = false;
CFStringRef rootPath = CFStringCreateWithCString(kCFAllocatorDefault, m_root.toStdString().data(), kCFStringEncodingMacRoman);
CFArrayRef pathsToWatch = CFArrayCreate(NULL, (const void **)&rootPath, 1, NULL);
CFMutableArrayRef pathsToWatch = CFArrayCreateMutable(nullptr, this->m_folderWatchRoots.size(), nullptr);
for (const auto& root : this->m_folderWatchRoots)
{
CFArrayAppendValue(pathsToWatch, root.m_directory.toCFString());
}
// The larger this number, the larger the delay between the kernel knowing a file changed
// and us actually consuming the event. It is very important for asset processor to deal with
@@ -60,11 +36,12 @@ bool FolderRootWatch::Start()
// Set ourselves as the value for the context info field so that in the callback
// we get passed into it and the callback can call our public API to handle
// the file change events
FSEventStreamContext streamContext;
::memset(&streamContext, 0, sizeof(streamContext));
streamContext.info = this;
FSEventStreamContext streamContext{
/*.version =*/ 0,
/*.info =*/ this,
};
m_platformImpl->m_stream = FSEventStreamCreate(NULL,
m_platformImpl->m_stream = FSEventStreamCreate(nullptr,
FileEventStreamCallback,
&streamContext,
pathsToWatch,
@@ -72,24 +49,25 @@ bool FolderRootWatch::Start()
timeBetweenKernelUpdateAndNotification,
kFSEventStreamCreateFlagFileEvents);
AZ_Error("FileWatcher", (m_platformImpl->m_stream != nullptr), "FSEventStreamCreate returned a nullptr. No file events will be reported for %s", m_root.toStdString().c_str());
m_thread = std::thread(std::bind(&FolderRootWatch::WatchFolderLoop, this));
AZ_Error("FileWatcher", (m_platformImpl->m_stream != nullptr), "FSEventStreamCreate returned a nullptr. No file events will be reported.");
const CFIndex pathCount = CFArrayGetCount(pathsToWatch);
for(CFIndex i = 0; i < pathCount; ++i)
{
CFRelease(CFArrayGetValueAtIndex(pathsToWatch, i));
}
CFRelease(pathsToWatch);
CFRelease(rootPath);
return (m_platformImpl->m_stream != nullptr);
return m_platformImpl->m_stream != nullptr;
}
void FolderRootWatch::Stop()
void FileWatcher::PlatformStop()
{
m_shutdownThreadSignal = true;
if (m_thread.joinable())
{
m_thread.join(); // wait for the thread to finish
m_thread = std::thread(); //destroy
}
FSEventStreamStop(m_platformImpl->m_stream);
@@ -97,7 +75,7 @@ void FolderRootWatch::Stop()
FSEventStreamRelease(m_platformImpl->m_stream);
}
void FolderRootWatch::WatchFolderLoop()
void FileWatcher::WatchFolderLoop()
{
// Use a half second timeout interval so that we can check if
// m_shutdownThreadSignal has been changed while we were running the RunLoop
@@ -117,14 +95,14 @@ void FolderRootWatch::WatchFolderLoop()
void FileEventStreamCallback(ConstFSEventStreamRef streamRef, void *clientCallBackInfo, size_t numEvents, void *eventPaths, const FSEventStreamEventFlags eventFlags[], const FSEventStreamEventId eventIds[])
{
FolderRootWatch* watcher = reinterpret_cast<FolderRootWatch*>(clientCallBackInfo);
auto* watcher = reinterpret_cast<FileWatcher*>(clientCallBackInfo);
const char** filePaths = reinterpret_cast<const char**>(eventPaths);
for (int i = 0; i < numEvents; ++i)
{
QFileInfo fileInfo(QDir::cleanPath(filePaths[i]));
QString fileAndPath = fileInfo.absoluteFilePath();
const QFileInfo fileInfo(QDir::cleanPath(filePaths[i]));
const QString fileAndPath = fileInfo.absoluteFilePath();
if (!fileInfo.isHidden())
{
@@ -133,38 +111,38 @@ void FileEventStreamCallback(ConstFSEventStreamRef streamRef, void *clientCallBa
// so check for all of them
if (eventFlags[i] & kFSEventStreamEventFlagItemCreated)
{
watcher->ProcessNewFileEvent(fileAndPath);
watcher->rawFileAdded(fileAndPath, {});
}
if (eventFlags[i] & kFSEventStreamEventFlagItemModified)
{
watcher->ProcessModifyFileEvent(fileAndPath);
watcher->rawFileModified(fileAndPath, {});
}
if (eventFlags[i] & kFSEventStreamEventFlagItemRemoved)
{
watcher->ProcessDeleteFileEvent(fileAndPath);
watcher->rawFileRemoved(fileAndPath, {});
}
if (eventFlags[i] & kFSEventStreamEventFlagItemRenamed)
{
if (fileInfo.exists())
{
watcher->ProcessNewFileEvent(fileAndPath);
watcher->rawFileAdded(fileAndPath, {});
// macOS does not send out an event for the directory being
// modified when a file has been renamed but the FileWatcher
// API expects it so send out the modification event ourselves.
watcher->ProcessModifyFileEvent(fileInfo.absolutePath());
watcher->rawFileModified(fileInfo.absolutePath(), {});
}
else
{
watcher->ProcessDeleteFileEvent(fileAndPath);
watcher->rawFileRemoved(fileAndPath, {});
// macOS does not send out an event for the directory being
// modified when a file has been renamed but the FileWatcher
// API expects it so send out the modification event ourselves.
watcher->ProcessModifyFileEvent(fileInfo.absolutePath());
watcher->rawFileModified(fileInfo.absolutePath(), {});
}
}
}
Code/Tools/AssetProcessor/Platform/Mac/native/FileWatcher/FileWatcher_platform.h
+11
View File
@@ -0,0 +1,11 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <native/FileWatcher/FileWatcher_mac.h>
Code/Tools/AssetProcessor/Platform/Mac/native/FileWatcher/FileWatcher_win.cpp
-130
View File
@@ -1,130 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <native/FileWatcher/FileWatcher.h>
#include <AzCore/PlatformIncl.h>
struct FolderRootWatch::PlatformImplementation
{
PlatformImplementation() : m_directoryHandle(nullptr), m_ioHandle(nullptr) { }
HANDLE m_directoryHandle;
HANDLE m_ioHandle;
};
//////////////////////////////////////////////////////////////////////////////
/// FolderWatchRoot
FolderRootWatch::FolderRootWatch(const QString rootFolder)
: m_root(rootFolder)
, m_shutdownThreadSignal(false)
, m_fileWatcher(nullptr)
, m_platformImpl(new PlatformImplementation())
{
}
FolderRootWatch::~FolderRootWatch()
{
// Destructor is required in here since this file contains the definition of struct PlatformImplementation
Stop();
delete m_platformImpl;
}
bool FolderRootWatch::Start()
{
m_platformImpl->m_directoryHandle = ::CreateFileW(m_root.toStdWString().data(), FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, nullptr);
if (m_platformImpl->m_directoryHandle != INVALID_HANDLE_VALUE)
{
m_platformImpl->m_ioHandle = ::CreateIoCompletionPort(m_platformImpl->m_directoryHandle, nullptr, 1, 0);
if (m_platformImpl->m_ioHandle != INVALID_HANDLE_VALUE)
{
m_shutdownThreadSignal = false;
m_thread = std::thread(std::bind(&FolderRootWatch::WatchFolderLoop, this));
return true;
}
}
return false;
}
void FolderRootWatch::Stop()
{
m_shutdownThreadSignal = true;
CloseHandle(m_platformImpl->m_ioHandle);
m_platformImpl->m_ioHandle = nullptr;
if (m_thread.joinable())
{
m_thread.join(); // wait for the thread to finish
m_thread = std::thread(); //destroy
}
CloseHandle(m_platformImpl->m_directoryHandle);
m_platformImpl->m_directoryHandle = nullptr;
}
void FolderRootWatch::WatchFolderLoop()
{
FILE_NOTIFY_INFORMATION aFileNotifyInformationList[50000];
QString path;
OVERLAPPED aOverlapped;
LPOVERLAPPED pOverlapped;
DWORD dwByteCount;
ULONG_PTR ulKey;
while (!m_shutdownThreadSignal)
{
::memset(aFileNotifyInformationList, 0, sizeof(aFileNotifyInformationList));
::memset(&aOverlapped, 0, sizeof(aOverlapped));
if (::ReadDirectoryChangesW(m_platformImpl->m_directoryHandle, aFileNotifyInformationList, sizeof(aFileNotifyInformationList), true, FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_ATTRIBUTES | FILE_NOTIFY_CHANGE_FILE_NAME, nullptr, &aOverlapped, nullptr))
{
//wait for up to a second for I/O to signal
dwByteCount = 0;
if (::GetQueuedCompletionStatus(m_platformImpl->m_ioHandle, &dwByteCount, &ulKey, &pOverlapped, INFINITE))
{
//if we are signaled to shutdown bypass
if (!m_shutdownThreadSignal && ulKey)
{
if (dwByteCount)
{
int offset = 0;
FILE_NOTIFY_INFORMATION* pFileNotifyInformation = aFileNotifyInformationList;
do
{
pFileNotifyInformation = (FILE_NOTIFY_INFORMATION*)((char*)pFileNotifyInformation + offset);
path.clear();
path.append(m_root);
path.append(QString::fromWCharArray(pFileNotifyInformation->FileName, pFileNotifyInformation->FileNameLength / 2));
QString file = QDir::toNativeSeparators(QDir::cleanPath(path));
switch (pFileNotifyInformation->Action)
{
case FILE_ACTION_ADDED:
case FILE_ACTION_RENAMED_NEW_NAME:
ProcessNewFileEvent(file);
break;
case FILE_ACTION_REMOVED:
case FILE_ACTION_RENAMED_OLD_NAME:
ProcessDeleteFileEvent(file);
break;
case FILE_ACTION_MODIFIED:
ProcessModifyFileEvent(file);
break;
}
offset = pFileNotifyInformation->NextEntryOffset;
} while (offset);
}
}
}
}
}
}
Code/Tools/AssetProcessor/Platform/Windows/assetprocessor_windows_files.cmake
+2
View File
@@ -7,6 +7,8 @@
#
set(FILES
native/FileWatcher/FileWatcher_platform.h
native/FileWatcher/FileWatcher_win.cpp
native/FileWatcher/FileWatcher_windows.h
native/resource.h
)
Code/Tools/AssetProcessor/Platform/Windows/native/FileWatcher/FileWatcher_platform.h
+11
View File
@@ -0,0 +1,11 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <native/FileWatcher/FileWatcher_windows.h>
Code/Tools/AssetProcessor/Platform/Windows/native/FileWatcher/FileWatcher_win.cpp
+116 -93
View File
@@ -6,125 +6,148 @@
*
*/
#include <AzCore/std/tuple.h>
#include <AzCore/std/utils.h>
#include <native/FileWatcher/FileWatcher.h>
#include <native/FileWatcher/FileWatcher_platform.h>
#include <QDir>
#include <AzCore/PlatformIncl.h>
struct FolderRootWatch::PlatformImplementation
bool FileWatcher::PlatformStart()
{
PlatformImplementation() : m_directoryHandle(nullptr), m_ioHandle(nullptr) { }
HANDLE m_directoryHandle;
HANDLE m_ioHandle;
};
m_shutdownThreadSignal = false;
//////////////////////////////////////////////////////////////////////////////
/// FolderWatchRoot
FolderRootWatch::FolderRootWatch(const QString rootFolder)
: m_root(rootFolder)
, m_shutdownThreadSignal(false)
, m_fileWatcher(nullptr)
, m_platformImpl(new PlatformImplementation())
{
}
FolderRootWatch::~FolderRootWatch()
{
// Destructor is required in here since this file contains the definition of struct PlatformImplementation
Stop();
delete m_platformImpl;
}
bool FolderRootWatch::Start()
{
m_platformImpl->m_directoryHandle = ::CreateFileW(m_root.toStdWString().data(), FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, nullptr);
if (m_platformImpl->m_directoryHandle != INVALID_HANDLE_VALUE)
bool allSucceeded = true;
for (const auto& [directory, recursive] : m_folderWatchRoots)
{
m_platformImpl->m_ioHandle = ::CreateIoCompletionPort(m_platformImpl->m_directoryHandle, nullptr, 1, 0);
if (m_platformImpl->m_ioHandle != INVALID_HANDLE_VALUE)
if (QDir(directory).exists())
{
m_shutdownThreadSignal = false;
m_thread = std::thread(std::bind(&FolderRootWatch::WatchFolderLoop, this));
return true;
allSucceeded &= m_platformImpl->AddWatchFolder(directory, recursive);
}
}
return false;
return allSucceeded;
}
void FolderRootWatch::Stop()
bool FileWatcher::PlatformImplementation::AddWatchFolder(QString root, bool recursive)
{
HandleUniquePtr directoryHandle{::CreateFileW(
root.toStdWString().data(),
FILE_LIST_DIRECTORY,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
nullptr,
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED,
nullptr
)};
if (directoryHandle.get() == INVALID_HANDLE_VALUE)
{
AZ_Warning("FileWatcher", false, "Failed to start watching %s", root.toUtf8().constData());
return false;
}
// Associate this file handle with our existing io completion port handle
if (!::CreateIoCompletionPort(directoryHandle.get(), m_ioHandle.get(), /*CompletionKey =*/ static_cast<ULONG_PTR>(PlatformImplementation::EventType::FileRead), 1))
{
return false;
}
auto id = AZStd::make_unique<OVERLAPPED>();
auto* idp = id.get();
const auto& [folderWatch, inserted] = m_folderRootWatches.emplace(AZStd::piecewise_construct, AZStd::forward_as_tuple(idp),
AZStd::forward_as_tuple(AZStd::move(id), AZStd::move(directoryHandle), root, recursive));
if (!inserted)
{
return false;
}
return folderWatch->second.ReadChanges();
}
bool FileWatcher::PlatformImplementation::FolderRootWatch::ReadChanges()
{
// Register to get directory change notifications for our directory handle
return ::ReadDirectoryChangesW(
m_directoryHandle.get(),
&m_fileNotifyInformationList,
sizeof(m_fileNotifyInformationList),
m_recursive,
FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_ATTRIBUTES | FILE_NOTIFY_CHANGE_FILE_NAME,
nullptr,
m_overlapped.get(),
nullptr
);
}
void FileWatcher::PlatformStop()
{
m_shutdownThreadSignal = true;
CloseHandle(m_platformImpl->m_ioHandle);
m_platformImpl->m_ioHandle = nullptr;
// Send a special signal to the child thread, that is blocked in a GetQueuedCompletionStatus call, with a completion
// key set to Shutdown. The child thread will stop its processing when it receives this value for the completion key
PostQueuedCompletionStatus(m_platformImpl->m_ioHandle.get(), 0, /*CompletionKey =*/ static_cast<ULONG_PTR>(PlatformImplementation::EventType::Shutdown), nullptr);
if (m_thread.joinable())
{
m_thread.join(); // wait for the thread to finish
m_thread = std::thread(); //destroy
}
CloseHandle(m_platformImpl->m_directoryHandle);
m_platformImpl->m_directoryHandle = nullptr;
}
void FolderRootWatch::WatchFolderLoop()
void FileWatcher::WatchFolderLoop()
{
FILE_NOTIFY_INFORMATION aFileNotifyInformationList[50000];
QString path;
OVERLAPPED aOverlapped;
LPOVERLAPPED pOverlapped;
DWORD dwByteCount;
ULONG_PTR ulKey;
LPOVERLAPPED directoryId = nullptr;
ULONG_PTR completionKey = 0;
while (!m_shutdownThreadSignal)
{
::memset(aFileNotifyInformationList, 0, sizeof(aFileNotifyInformationList));
::memset(&aOverlapped, 0, sizeof(aOverlapped));
if (::ReadDirectoryChangesW(m_platformImpl->m_directoryHandle, aFileNotifyInformationList, sizeof(aFileNotifyInformationList), true, FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_ATTRIBUTES | FILE_NOTIFY_CHANGE_FILE_NAME, nullptr, &aOverlapped, nullptr))
DWORD dwByteCount = 0;
if (::GetQueuedCompletionStatus(m_platformImpl->m_ioHandle.get(), &dwByteCount, &completionKey, &directoryId, INFINITE))
{
//wait for up to a second for I/O to signal
dwByteCount = 0;
if (::GetQueuedCompletionStatus(m_platformImpl->m_ioHandle, &dwByteCount, &ulKey, &pOverlapped, INFINITE))
if (m_shutdownThreadSignal || completionKey == static_cast<ULONG_PTR>(PlatformImplementation::EventType::Shutdown))
{
//if we are signaled to shutdown bypass
if (!m_shutdownThreadSignal && ulKey)
{
if (dwByteCount)
{
int offset = 0;
FILE_NOTIFY_INFORMATION* pFileNotifyInformation = aFileNotifyInformationList;
do
{
pFileNotifyInformation = (FILE_NOTIFY_INFORMATION*)((char*)pFileNotifyInformation + offset);
path.clear();
path.append(m_root);
path.append(QString::fromWCharArray(pFileNotifyInformation->FileName, pFileNotifyInformation->FileNameLength / 2));
QString file = QDir::toNativeSeparators(QDir::cleanPath(path));
switch (pFileNotifyInformation->Action)
{
case FILE_ACTION_ADDED:
case FILE_ACTION_RENAMED_NEW_NAME:
ProcessNewFileEvent(file);
break;
case FILE_ACTION_REMOVED:
case FILE_ACTION_RENAMED_OLD_NAME:
ProcessDeleteFileEvent(file);
break;
case FILE_ACTION_MODIFIED:
ProcessModifyFileEvent(file);
break;
}
offset = pFileNotifyInformation->NextEntryOffset;
} while (offset);
}
}
break;
}
if (dwByteCount == 0)
{
continue;
}
const auto foundFolderRoot = m_platformImpl->m_folderRootWatches.find(directoryId);
if (foundFolderRoot == end(m_platformImpl->m_folderRootWatches))
{
continue;
}
PlatformImplementation::FolderRootWatch& folderRoot = foundFolderRoot->second;
// Initialize offset to 1 to ensure that the first iteration is always processed
DWORD offset = 1;
for (
const FILE_NOTIFY_INFORMATION* pFileNotifyInformation = reinterpret_cast<const FILE_NOTIFY_INFORMATION*>(&folderRoot.m_fileNotifyInformationList);
offset;
pFileNotifyInformation = reinterpret_cast<const FILE_NOTIFY_INFORMATION*>(reinterpret_cast<const char*>(pFileNotifyInformation) + offset)
){
const QString file = QDir::toNativeSeparators(QDir(folderRoot.m_directoryRoot)
.filePath(QString::fromWCharArray(pFileNotifyInformation->FileName, pFileNotifyInformation->FileNameLength / 2)));
switch (pFileNotifyInformation->Action)
{
case FILE_ACTION_ADDED:
case FILE_ACTION_RENAMED_NEW_NAME:
rawFileAdded(file, {});
break;
case FILE_ACTION_REMOVED:
case FILE_ACTION_RENAMED_OLD_NAME:
rawFileRemoved(file, {});
break;
case FILE_ACTION_MODIFIED:
rawFileModified(file, {});
break;
}
offset = pFileNotifyInformation->NextEntryOffset;
}
folderRoot.ReadChanges();
}
}
}
Code/Tools/AssetProcessor/Platform/Windows/native/FileWatcher/FileWatcher_windows.h
+65
View File
@@ -0,0 +1,65 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/typetraits/aligned_storage.h>
#include <AzCore/std/typetraits/remove_pointer.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <native/FileWatcher/FileWatcher.h>
#include <AzCore/PlatformIncl.h>
struct HandleDeleter
{
void operator()(HANDLE handle)
{
if (handle && handle != INVALID_HANDLE_VALUE)
{
CloseHandle(handle);
}
}
};
using HandleUniquePtr = AZStd::unique_ptr<AZStd::remove_pointer_t<HANDLE>, HandleDeleter>;
class FileWatcher::PlatformImplementation
{
public:
bool AddWatchFolder(QString folder, bool recursive);
struct FolderRootWatch
{
FolderRootWatch(AZStd::unique_ptr<OVERLAPPED>&& overlapped, HandleUniquePtr&& directoryHandle, QString root, bool recursive)
: m_overlapped(AZStd::move(overlapped))
, m_directoryHandle(AZStd::move(directoryHandle))
, m_directoryRoot(AZStd::move(root))
, m_recursive(recursive)
{
}
bool ReadChanges();
AZStd::unique_ptr<OVERLAPPED> m_overlapped; // Identifies this root watch
HandleUniquePtr m_directoryHandle;
QString m_directoryRoot;
bool m_recursive;
AZStd::aligned_storage_t<64 * 1024, sizeof(DWORD)> m_fileNotifyInformationList{};
};
enum class EventType
{
FileRead,
Shutdown
};
AZStd::unordered_map<LPOVERLAPPED, FolderRootWatch> m_folderRootWatches;
HandleUniquePtr m_ioHandle{CreateIoCompletionPort(INVALID_HANDLE_VALUE, nullptr, /*CompletionKey =*/ static_cast<ULONG_PTR>(EventType::FileRead), 1)};
};