birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

PythonProxyNotificationHandler::OnEventGenericHook acquires the Python GIL before executing (#3904)

Browse Source 
- Protected python execution of OnEventGenericHook by trying to lock the GIL and show a descriptive error when it was not possible to lock.
- Improved mechanism to lock python mutex and GIL in PhytonSystemComponent. Acquiring/releasing GIL once per thread.
- Added unit test to verify errors are fired when trying to execute OnEventGenericHook from another thread (as it should not able to acquire the GIL).
- Improved python threading tests to actually using python buses.

Signed-off-by: moraaar moraaar@amazon.com
This commit is contained in:
moraaar
2021-09-07 15:35:04 +01:00
committed by GitHub
parent 33299399af
commit f551e69b2b
16 changed files with 259 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Gems/EditorPythonBindings/Code/Tests/EditorPythonBindingsTest.cpp
+2 -2
View File
@@ -65,7 +65,7 @@ namespace UnitTest
{
// clearing up memory
m_notificationSink = EditorPythonBindingsNotificationBusSink();
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
// shutdown time!
PythonTestingFixture::TearDown();
@@ -333,7 +333,7 @@ sys.version
{
// clearing up memory
m_notificationSink = EditorPythonBindingsNotificationBusSink();
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
// shutdown time!
PythonTestingFixture::TearDown();
Gems/EditorPythonBindings/Code/Tests/PythonAssetTypesTests.cpp
+1 -1
View File
@@ -366,7 +366,7 @@ namespace UnitTest
void TearDown() override
{
// clearing up memory
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
PythonTestingFixture::TearDown();
}
};
Gems/EditorPythonBindings/Code/Tests/PythonAssociativeTests.cpp
+1 -1
View File
@@ -95,7 +95,7 @@ namespace UnitTest
void TearDown() override
{
// clearing up memory
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
PythonTestingFixture::TearDown();
}
};
Gems/EditorPythonBindings/Code/Tests/PythonContainerAnyTests.cpp
+1 -1
View File
@@ -227,7 +227,7 @@ namespace UnitTest
void TearDown() override
{
// clearing up memory
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
PythonTestingFixture::TearDown();
}
};
Gems/EditorPythonBindings/Code/Tests/PythonDictionaryTests.cpp
+1 -1
View File
@@ -107,7 +107,7 @@ namespace UnitTest
void TearDown() override
{
// clearing up memory
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
PythonTestingFixture::TearDown();
}
};
Gems/EditorPythonBindings/Code/Tests/PythonGlobalsTests.cpp
+1 -1
View File
@@ -175,7 +175,7 @@ namespace UnitTest
void TearDown() override
{
// clearing up memory
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
PythonTestingFixture::TearDown();
}
Gems/EditorPythonBindings/Code/Tests/PythonPairTests.cpp
+1 -1
View File
@@ -121,7 +121,7 @@ namespace UnitTest
void TearDown() override
{
// clearing up memory
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
PythonTestingFixture::TearDown();
}
};
Gems/EditorPythonBindings/Code/Tests/PythonProxyBusTests.cpp
+90 -2
View File
@@ -207,11 +207,13 @@ namespace UnitTest
: AZ::EBusTraits
{
static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::Single;
using MutexType = AZStd::mutex;
virtual ~PythonTestSingleAddressNotifications() = default;
virtual void OnPing(AZ::u64 count) = 0;
virtual void OnPong(AZ::u64 count) = 0;
virtual void MultipleInputs(AZ::u64 one, AZ::s8 two, AZStd::string_view three) = 0;
virtual AZStd::string OnAddFish(AZStd::string_view value) = 0;
virtual void OnFire() = 0;
};
using PythonTestSingleAddressNotificationBus = AZ::EBus<PythonTestSingleAddressNotifications>;
@@ -220,7 +222,7 @@ namespace UnitTest
, public AZ::BehaviorEBusHandler
{
AZ_EBUS_BEHAVIOR_BINDER(PythonTestNotificationHandler, "{97052D15-A4E8-461B-B065-91D16E31C4F7}", AZ::SystemAllocator,
OnPing, OnPong, MultipleInputs, OnAddFish);
OnPing, OnPong, MultipleInputs, OnAddFish, OnFire);
virtual ~PythonTestNotificationHandler() = default;
@@ -246,6 +248,11 @@ namespace UnitTest
return result;
}
void OnFire() override
{
Call(FN_OnFire);
}
static AZ::u64 s_pongCount;
static AZ::u64 s_pingCount;
@@ -270,6 +277,25 @@ namespace UnitTest
return result;
}
static void DoFire()
{
PythonTestSingleAddressNotificationBus::Broadcast(&PythonTestSingleAddressNotificationBus::Events::OnFire);
}
static void DoFiresInParallel(int value)
{
AZStd::vector<AZStd::thread> threads;
threads.reserve(value);
for (size_t i = 0; i < value; ++i)
{
threads.emplace_back(&DoFire);
}
for (AZStd::thread& thread : threads)
{
thread.join();
}
}
static void Reset()
{
s_pingCount = 0;
@@ -288,6 +314,7 @@ namespace UnitTest
->Event("on_pong", &PythonTestSingleAddressNotificationBus::Events::OnPong)
->Event("MultipleInputs", &PythonTestSingleAddressNotificationBus::Events::MultipleInputs)
->Event("OnAddFish", &PythonTestSingleAddressNotificationBus::Events::OnAddFish)
->Event("OnFire", &PythonTestSingleAddressNotificationBus::Events::OnFire)
;
// for testing from Python to send out the events
@@ -297,6 +324,8 @@ namespace UnitTest
->Method("do_ping", &PythonTestNotificationHandler::DoPing)
->Method("do_pong", &PythonTestNotificationHandler::DoPong)
->Method("do_add_fish", &PythonTestNotificationHandler::DoAddFish)
->Method("do_fire", &PythonTestNotificationHandler::DoFire)
->Method("do_fires_in_parallel", &PythonTestNotificationHandler::DoFiresInParallel)
;
}
}
@@ -358,7 +387,7 @@ namespace UnitTest
void TearDown() override
{
// clearing up memory
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
PythonTestingFixture::TearDown();
}
};
@@ -809,6 +838,65 @@ namespace UnitTest
EXPECT_EQ(2, m_testSink.m_evaluationMap[static_cast<int>(LogTypes::AtAddress_Match)]);
}
TEST_F(PythonBusProxyTests, SingleAddressNotifications_InParallel_Errors)
{
PythonTestNotificationHandler pythonTestNotificationHandler;
pythonTestNotificationHandler.Reflect(m_app.GetBehaviorContext());
AZ::Entity e;
Activate(e);
SimulateEditorBecomingInitialized();
enum class LogTypes
{
Skip = 0,
Notifications_OnFire,
};
m_testSink.m_evaluateMessage = [](const char* window, const char* message) -> int
{
if (AzFramework::StringFunc::Equal(window, "python"))
{
if (AzFramework::StringFunc::Equal(message, "Notifications_OnFire"))
{
return static_cast<int>(LogTypes::Notifications_OnFire);
}
}
return static_cast<int>(LogTypes::Skip);
};
UnitTest::PythonTestNotificationHandler::Reset();
const int numFiresInParallel = 220;
const AZStd::string script = AZStd::string::format(
"import azlmbr.bus\n"
"import azlmbr.test\n"
"\n"
"def OnFire(parameters) :\n"
" print('Notifications_OnFire')\n"
"\n"
"handler = azlmbr.bus.NotificationHandler('PythonTestSingleAddressNotificationBus')\n"
"handler.connect(None)\n"
"handler.add_callback('OnFire', OnFire)\n"
"\n"
"azlmbr.test.PythonTestNotificationHandler_do_fire()\n"
"\n"
"azlmbr.test.PythonTestNotificationHandler_do_fires_in_parallel(%d)\n"
"\n"
"handler.disconnect()\n",
numFiresInParallel);
AZ_TEST_START_TRACE_SUPPRESSION;
AzToolsFramework::EditorPythonRunnerRequestBus::Broadcast(&AzToolsFramework::EditorPythonRunnerRequestBus::Events::ExecuteByString, script, false);
AZ_TEST_STOP_TRACE_SUPPRESSION(numFiresInParallel); // Expect numFiresInParallel errors
e.Deactivate();
EXPECT_EQ(1, m_testSink.m_evaluationMap[static_cast<int>(LogTypes::Notifications_OnFire)]);
}
TEST_F(PythonBusProxyTests, NotificationsWithNoAddress)
{
PythonTestNotificationHandler pythonTestNotificationHandler;
Gems/EditorPythonBindings/Code/Tests/PythonProxyObjectTests.cpp
+1 -1
View File
@@ -419,7 +419,7 @@ namespace UnitTest
void TearDown() override
{
// clearing up memory
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
PythonTestingFixture::TearDown();
}
};
Gems/EditorPythonBindings/Code/Tests/PythonReflectionComponentTests.cpp
+1 -1
View File
@@ -532,7 +532,7 @@ namespace UnitTest
void TearDown() override
{
// clearing up memory
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
PythonTestingFixture::TearDown();
}
};
Gems/EditorPythonBindings/Code/Tests/PythonThreadingTests.cpp
+15 -34
View File
@@ -75,7 +75,7 @@ namespace UnitTest
void TearDown() override
{
// clearing up memory
m_testSink = PythonTraceMessageSink();
m_testSink.CleanUp();
PythonTestingFixture::TearDown();
}
};
@@ -114,38 +114,29 @@ namespace UnitTest
try
{
// prepare handler on this thread
pybind11::exec(R"(
import azlmbr.test
const AZStd::string_view script =
"import azlmbr.test\n"
"\n"
"def on_notification(args) :\n"
" value = args[0] + 2\n"
" print('RanInThread')\n"
" return value\n"
"\n"
"handler = azlmbr.test.PythonThreadNotificationBusHandler()\n"
"handler.connect()\n"
"handler.add_callback('OnNotification', on_notification)\n";
def on_notification(args):
value = args[0] + 2
print ('RanInThread')
return value
handler = azlmbr.test.PythonThreadNotificationBusHandler()
handler.connect()
handler.add_callback('OnNotification', on_notification)
)");
AzToolsFramework::EditorPythonRunnerRequestBus::Broadcast(&AzToolsFramework::EditorPythonRunnerRequestBus::Events::ExecuteByString, script, false /*printResult*/);
// start thread; in thread issue notification
auto threadCallback = []()
{
AZ::s64 result = 0;
auto notificationCallback = [&result]()
{
PythonThreadNotificationBus::BroadcastResult(result, &PythonThreadNotificationBus::Events::OnNotification, 40);
};
auto editorPythonEventsInterface = AZ::Interface<AzToolsFramework::EditorPythonEventsInterface>::Get();
if (editorPythonEventsInterface)
{
editorPythonEventsInterface->ExecuteWithLock(notificationCallback);
}
PythonThreadNotificationBus::BroadcastResult(result, &PythonThreadNotificationBus::Events::OnNotification, 40);
EXPECT_EQ(42, result);
};
AZStd::thread theThread(threadCallback);
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(100));
theThread.join();
}
catch ([[maybe_unused]] const std::exception& e)
@@ -187,21 +178,11 @@ namespace UnitTest
auto threadCallback = []()
{
AZ::s64 result = 0;
auto notificationCallback = [&result]()
{
PythonThreadNotificationBus::BroadcastResult(result, &PythonThreadNotificationBus::Events::OnNotification, 40);
};
auto editorPythonEventsInterface = AZ::Interface<AzToolsFramework::EditorPythonEventsInterface>::Get();
if (editorPythonEventsInterface)
{
editorPythonEventsInterface->ExecuteWithLock(notificationCallback);
}
PythonThreadNotificationBus::BroadcastResult(result, &PythonThreadNotificationBus::Events::OnNotification, 40);
EXPECT_EQ(0, result);
};
AZStd::thread theThread(threadCallback);
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(100));
theThread.join();
// the Python script above raises an exception which causes two AZ_Error() message lines:
Gems/EditorPythonBindings/Code/Tests/PythonTraceMessageSink.h
+11
View File
@@ -37,6 +37,15 @@ namespace UnitTest
using EvaluationMap = AZStd::unordered_map<int, int>; // tag to count
EvaluationMap m_evaluationMap;
AZStd::mutex m_lock;
void CleanUp()
{
AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
m_evaluateMessage = {};
m_evaluationMap.clear();
}
//////////////////////////////////////////////////////////////////////////
// TraceMessageDrillerBus
void OnPrintf(const char* window, const char* message) override
@@ -46,6 +55,8 @@ namespace UnitTest
void OnOutput(const char* window, const char* message) override
{
AZStd::lock_guard<decltype(m_lock)> lock(m_lock);
if (m_evaluateMessage)
{
int key = m_evaluateMessage(window, message);