birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

PythonProxyNotificationHandler::OnEventGenericHook acquires the Python GIL before executing (#3904)

Browse Source 
- Protected python execution of OnEventGenericHook by trying to lock the GIL and show a descriptive error when it was not possible to lock.
- Improved mechanism to lock python mutex and GIL in PhytonSystemComponent. Acquiring/releasing GIL once per thread.
- Added unit test to verify errors are fired when trying to execute OnEventGenericHook from another thread (as it should not able to acquire the GIL).
- Improved python threading tests to actually using python buses.

Signed-off-by: moraaar moraaar@amazon.com
This commit is contained in:
moraaar
2021-09-07 15:35:04 +01:00
committed by GitHub
parent 33299399af
commit f551e69b2b
16 changed files with 259 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.cpp
+95 -11
View File
@@ -258,6 +258,85 @@ namespace EditorPythonBindings
void Finalize() override {}
};
// Manages the acquisition and release of the Python GIL (Global Interpreter Lock).
// Used by PythonSystemComponent to lock the GIL when executing python.
class PythonSystemComponent::PythonGILScopedLock final
{
public:
PythonGILScopedLock(AZStd::recursive_mutex& lock, int& lockRecursiveCounter, bool tryLock = false);
~PythonGILScopedLock();
bool IsLocked() const;
protected:
void Lock(bool tryLock);
void Unlock();
AZStd::recursive_mutex& m_lock;
int& m_lockRecursiveCounter;
bool m_locked = false;
AZStd::unique_ptr<pybind11::gil_scoped_release> m_releaseGIL;
AZStd::unique_ptr<pybind11::gil_scoped_acquire> m_acquireGIL;
};
PythonSystemComponent::PythonGILScopedLock::PythonGILScopedLock(AZStd::recursive_mutex& lock, int& lockRecursiveCounter, bool tryLock)
: m_lock(lock)
, m_lockRecursiveCounter(lockRecursiveCounter)
{
Lock(tryLock);
}
PythonSystemComponent::PythonGILScopedLock::~PythonGILScopedLock()
{
Unlock();
}
bool PythonSystemComponent::PythonGILScopedLock::IsLocked() const
{
return m_locked;
}
void PythonSystemComponent::PythonGILScopedLock::Lock(bool tryLock)
{
if (tryLock)
{
if (!m_lock.try_lock())
{
return;
}
}
else
{
m_lock.lock();
}
m_locked = true;
m_lockRecursiveCounter++;
// Only Acquire the GIL when there is no recursion. If there is
// recursion that means it's the same thread (because the mutex was able
// to be locked) and therefore it's already got the GIL acquired.
if (m_lockRecursiveCounter == 1)
{
m_releaseGIL = AZStd::make_unique<pybind11::gil_scoped_release>();
m_acquireGIL = AZStd::make_unique<pybind11::gil_scoped_acquire>();
}
}
void PythonSystemComponent::PythonGILScopedLock::Unlock()
{
if (!m_locked)
{
return;
}
m_acquireGIL.reset();
m_releaseGIL.reset();
m_lockRecursiveCounter--;
m_locked = false;
m_lock.unlock();
}
void PythonSystemComponent::Reflect(AZ::ReflectContext* context)
{
if (AZ::SerializeContext* serialize = azrtti_cast<AZ::SerializeContext*>(context))
@@ -296,9 +375,9 @@ namespace EditorPythonBindings
void PythonSystemComponent::Deactivate()
{
StopPython(true);
AzToolsFramework::EditorPythonRunnerRequestBus::Handler::BusDisconnect();
AZ::Interface<AzToolsFramework::EditorPythonEventsInterface>::Unregister(this);
StopPython(true);
}
bool PythonSystemComponent::StartPython([[maybe_unused]] bool silenceWarnings)
@@ -354,7 +433,6 @@ namespace EditorPythonBindings
bool result = false;
EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindingsNotificationBus::Events::OnPreFinalize);
AzToolsFramework::EditorPythonRunnerRequestBus::Handler::BusDisconnect();
result = StopPythonInterpreter();
EditorPythonBindingsNotificationBus::Broadcast(&EditorPythonBindingsNotificationBus::Events::OnPostFinalize);
@@ -374,12 +452,21 @@ namespace EditorPythonBindings
void PythonSystemComponent::ExecuteWithLock(AZStd::function<void()> executionCallback)
{
AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
pybind11::gil_scoped_release release;
pybind11::gil_scoped_acquire acquire;
PythonGILScopedLock lock(m_lock, m_lockRecursiveCounter);
executionCallback();
}
bool PythonSystemComponent::TryExecuteWithLock(AZStd::function<void()> executionCallback)
{
PythonGILScopedLock lock(m_lock, m_lockRecursiveCounter, true /*tryLock*/);
if (lock.IsLocked())
{
executionCallback();
return true;
}
return false;
}
void PythonSystemComponent::DiscoverPythonPaths(PythonPathStack& pythonPathStack)
{
// the order of the Python paths is the order the Python bootstrap scripts will execute
@@ -548,8 +635,7 @@ namespace EditorPythonBindings
RedirectOutput::Intialize(PyImport_ImportModule("azlmbr_redirect"));
// Acquire GIL before calling Python code
AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
pybind11::gil_scoped_acquire acquire;
PythonGILScopedLock lock(m_lock, m_lockRecursiveCounter);
if (EditorPythonBindings::PythonSymbolEventBus::GetTotalNumOfEventHandlers() == 0)
{
@@ -622,8 +708,7 @@ namespace EditorPythonBindings
&AzToolsFramework::EditorPythonScriptNotificationsBus::Events::OnStartExecuteByString, script);
// Acquire GIL before calling Python code
AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
pybind11::gil_scoped_acquire acquire;
PythonGILScopedLock lock(m_lock, m_lockRecursiveCounter);
// Acquire scope for __main__ for executing our script
pybind11::object scope = pybind11::module::import("__main__").attr("__dict__");
@@ -741,8 +826,7 @@ namespace EditorPythonBindings
try
{
// Acquire GIL before calling Python code
AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
pybind11::gil_scoped_acquire acquire;
PythonGILScopedLock lock(m_lock, m_lockRecursiveCounter);
// Create standard "argc" / "argv" command-line parameters to pass in to the Python script via sys.argv.
// argc = number of parameters. This will always be at least 1, since the first parameter is the script name.