birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(Continuation) Implemented automation paralellization & standarization (#1718)

Browse Source 
Engine improvements/fixes

Fixed behavior that made the editor automated test to be sometimes stuck if lost the focus is lost.
Added support for specifying multiple tests to in batch to the editor, this is achieved by passing --runpythontest with the tests separated by ';'
Added new cmdline argument --project-user-path for overriding the user path. This allows to have multiple editors running writing logs and crash logs in different locations.
Moved responsability of exiting after a test finishes/passes out of ExecuteByFilenameAsTest, callers will use the bool return to know if the test passed.
Editor test batch and parallelization implementation:

Now the external python portion of the editor tests will be specified via test specs which will generate the test. Requiring no code. This is almost a data-driven approach.
Tests can be specified as single tests, parallel, batchable or batchable+parallel
Command line arguments for pytest to override the maximum number of editors, disable parallelization or batching.
Automated tests for testing this new editor testing utility

Signed-off-by: Garcia Ruiz <aljanru@amazon.co.uk>

Co-authored-by: Garcia Ruiz <aljanru@amazon.co.uk>
This commit is contained in:
AMZN-AlexOteiza
2021-07-22 12:57:23 +02:00
committed by GitHub
parent 231f09d899
commit b815c203da
32 changed files with 1653 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Editor/CryEdit.cpp
+74 -14
View File
@@ -581,6 +581,8 @@ public:
{{"project-path", "Supplies the path to the project that the Editor should use", "project-path"}, dummyString},
{{"engine-path", "Supplies the path to the engine", "engine-path"}, dummyString},
{{"project-cache-path", "Path to the project cache", "project-cache-path"}, dummyString},
{{"project-user-path", "Path to the project user path", "project-user-path"}, dummyString},
{{"project-log-path", "Path to the project log path", "project-log-path"}, dummyString}
// add dummy entries here to prevent QCommandLineParser error-ing out on cmd line args that will be parsed later
};
@@ -1483,7 +1485,6 @@ struct PythonTestOutputHandler final
{
PythonOutputHandler::OnExceptionMessage(message);
printf("EXCEPTION: %.*s\n", static_cast<int>(message.size()), message.data());
AZ::Debug::Trace::Terminate(1);
}
};
@@ -1501,34 +1502,91 @@ void CCryEditApp::RunInitPythonScript(CEditCommandLineInfo& cmdInfo)
using namespace AzToolsFramework;
if (cmdInfo.m_bRunPythonScript || cmdInfo.m_bRunPythonTestScript)
{
// cmdInfo data is only available on startup, copy it
QByteArray fileStr = cmdInfo.m_strFileName.toUtf8();
// We support specifying multiple files in the cmdline by separating them with ';'
AZStd::vector<AZStd::string_view> fileList;
AzFramework::StringFunc::TokenizeVisitor(
fileStr.constData(),
[&fileList](AZStd::string_view elem)
{
fileList.push_back(elem);
}, ';', false /* keepEmptyStrings */
);
if (cmdInfo.m_pythonArgs.length() > 0 || cmdInfo.m_bRunPythonTestScript)
{
AZStd::vector<AZStd::string> tokens;
AzFramework::StringFunc::Tokenize(cmdInfo.m_pythonArgs.toUtf8().constData(), tokens, ' ');
QByteArray pythonArgsStr = cmdInfo.m_pythonArgs.toUtf8();
AZStd::vector<AZStd::string_view> pythonArgs;
std::transform(tokens.begin(), tokens.end(), std::back_inserter(pythonArgs), [](auto& tokenData) { return tokenData.c_str(); });
AzFramework::StringFunc::TokenizeVisitor(pythonArgsStr.constData(),
[&pythonArgs](AZStd::string_view elem)
{
pythonArgs.push_back(elem);
}, ' '
);
if (cmdInfo.m_bRunPythonTestScript)
{
AZStd::string pythonTestCase;
if (!cmdInfo.m_pythontTestCase.isEmpty())
// Multiple testcases can be specified them with ';', these should match the files to run
AZStd::vector<AZStd::string_view> testcaseList;
testcaseList.resize(fileList.size());
{
pythonTestCase = cmdInfo.m_pythontTestCase.toUtf8().constData();
int i = 0;
AzFramework::StringFunc::TokenizeVisitor(
fileStr.constData(),
[&i, &testcaseList](AZStd::string_view elem)
{
testcaseList[i++] = (elem);
}, ';', false /* keepEmptyStrings */
);
}
EditorPythonRunnerRequestBus::Broadcast(&EditorPythonRunnerRequestBus::Events::ExecuteByFilenameAsTest, cmdInfo.m_strFileName.toUtf8().constData(), pythonTestCase, pythonArgs);
bool success = true;
auto ExecuteByFilenamesTests = [&pythonArgs, &fileList, &testcaseList, &success](EditorPythonRunnerRequests* pythonRunnerRequests)
{
for (int i = 0; i < fileList.size(); ++i)
{
bool cur_success = pythonRunnerRequests->ExecuteByFilenameAsTest(fileList[i], testcaseList[i], pythonArgs);
success = success && cur_success;
}
};
EditorPythonRunnerRequestBus::Broadcast(ExecuteByFilenamesTests);
// Close the editor gracefully as the test has completed
GetIEditor()->GetDocument()->SetModifiedFlag(false);
QTimer::singleShot(0, qApp, &QApplication::closeAllWindows);
if (success)
{
// Close the editor gracefully as the test has completed
GetIEditor()->GetDocument()->SetModifiedFlag(false);
QTimer::singleShot(0, qApp, &QApplication::closeAllWindows);
}
else
{
// Close down the application with 0xF exit code indicating failure of the test
AZ::Debug::Trace::Terminate(0xF);
}
}
else
{
EditorPythonRunnerRequestBus::Broadcast(&EditorPythonRunnerRequestBus::Events::ExecuteByFilenameWithArgs, cmdInfo.m_strFileName.toUtf8().constData(), pythonArgs);
auto ExecuteByFilenamesWithArgs = [&pythonArgs, &fileList](EditorPythonRunnerRequests* pythonRunnerRequests)
{
for (AZStd::string_view filename : fileList)
{
pythonRunnerRequests->ExecuteByFilenameWithArgs(filename, pythonArgs);
}
};
EditorPythonRunnerRequestBus::Broadcast(ExecuteByFilenamesWithArgs);
}
}
else
{
EditorPythonRunnerRequestBus::Broadcast(&EditorPythonRunnerRequestBus::Events::ExecuteByFilename, cmdInfo.m_strFileName.toUtf8().constData());
auto ExecuteByFilenames = [&fileList](EditorPythonRunnerRequests* pythonRunnerRequests)
{
for (AZStd::string_view filename : fileList)
{
pythonRunnerRequests->ExecuteByFilename(filename);
}
};
EditorPythonRunnerRequestBus::Broadcast(ExecuteByFilenames);
}
}
}
@@ -2271,7 +2329,9 @@ int CCryEditApp::IdleProcessing(bool bBackgroundUpdate)
bool bIsAppWindow = IsWindowInForeground();
bool bActive = false;
int res = 0;
if (bIsAppWindow || m_bForceProcessIdle || m_bKeepEditorActive)
if (bIsAppWindow || m_bForceProcessIdle || m_bKeepEditorActive
// Automated tests must always keep the editor active, or they can get stuck
|| m_bAutotestMode)
{
res = 1;
bActive = true;
Code/Editor/MainWindow.cpp
+2 -7
View File
@@ -278,14 +278,9 @@ namespace
PyExit();
}
void PyReportTest(bool success, const AZStd::string& output)
void PyTestOutput(const AZStd::string& output)
{
CCryEditApp::instance()->PrintAlways(output);
if (!success)
{
gEnv->retCode = 0xF; // Special error code indicating a failure in tests
}
PyExitNoPrompt();
}
}
@@ -1956,7 +1951,7 @@ namespace AzToolsFramework
addLegacyGeneral(behaviorContext->Method("get_pane_class_names", PyGetViewPaneNames, nullptr, "Get all available class names for use with open_pane & close_pane."));
addLegacyGeneral(behaviorContext->Method("exit", PyExit, nullptr, "Exits the editor."));
addLegacyGeneral(behaviorContext->Method("exit_no_prompt", PyExitNoPrompt, nullptr, "Exits the editor without prompting to save first."));
addLegacyGeneral(behaviorContext->Method("report_test_result", PyReportTest, nullptr, "Report test information."));
addLegacyGeneral(behaviorContext->Method("test_output", PyTestOutput, nullptr, "Report test information."));
}
}
}
Code/Framework/AzCore/AzCore/Settings/SettingsRegistryMergeUtils.cpp
+25 -3
View File
@@ -561,9 +561,24 @@ namespace AZ::SettingsRegistryMergeUtils
AZ::IO::FixedMaxPath normalizedProjectPath = path.LexicallyNormal();
registry.Set(FilePathKey_ProjectPath, normalizedProjectPath.Native());
// Add an alias to the project "user" directory
AZ::IO::FixedMaxPath projectUserPath = (normalizedProjectPath / "user").LexicallyNormal();
// Set the user directory with the provided path or using project/user as default
auto projectUserPathKey = FixedValueString::format("%s/project_user_path", BootstrapSettingsRootKey);
AZ::IO::FixedMaxPath projectUserPath;
if (!registry.Get(projectUserPath.Native(), projectUserPathKey))
{
projectUserPath = (normalizedProjectPath / "user").LexicallyNormal();
}
registry.Set(FilePathKey_ProjectUserPath, projectUserPath.Native());
// Set the user directory with the provided path or using project/user as default
auto projectLogPathKey = FixedValueString::format("%s/project_log_path", BootstrapSettingsRootKey);
AZ::IO::FixedMaxPath projectLogPath;
if (!registry.Get(projectLogPath.Native(), projectLogPathKey))
{
projectLogPath = (projectUserPath / "log").LexicallyNormal();
}
registry.Set(FilePathKey_ProjectLogPath, projectLogPath.Native());
// check for a default write storage path, fall back to the project's user/ directory if not
AZStd::optional<AZ::IO::FixedMaxPathString> devWriteStorage = Utils::GetDevWriteStoragePath();
registry.Set(FilePathKey_DevWriteStorage, devWriteStorage.has_value()
@@ -948,7 +963,14 @@ namespace AZ::SettingsRegistryMergeUtils
OptionKeyToRegsetKey{
"project-cache-path",
AZStd::string::format("%s/project_cache_path", AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey)},
OptionKeyToRegsetKey{"project-build-path", ProjectBuildPath} };
OptionKeyToRegsetKey{
"project-user-path",
AZStd::string::format("%s/project_user_path", AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey)},
OptionKeyToRegsetKey{
"project-log-path",
AZStd::string::format("%s/project_log_path", AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey)},
OptionKeyToRegsetKey{"project-build-path", ProjectBuildPath},
};
AZStd::fixed_vector<AZStd::string, commandOptions.size()> overrideArgs;
Code/Framework/AzCore/AzCore/Settings/SettingsRegistryMergeUtils.h
+4
View File
@@ -44,6 +44,10 @@ namespace AZ::SettingsRegistryMergeUtils
//! project settings can be stored
inline static constexpr char FilePathKey_ProjectUserPath[] = "/Amazon/AzCore/Runtime/FilePaths/SourceProjectUserPath";
//! Store the absolute path to the Projects "log" directory, which is a transient directory where per user
//! logs can be stored. By default this would be on "{FilePathKey_ProjectUserPath}/log"
inline static constexpr char FilePathKey_ProjectLogPath[] = "/Amazon/AzCore/Runtime/FilePaths/SourceProjectLogPath";
//! User facing key which represents the root of a project cmake build tree. i.e the ${CMAKE_BINARY_DIR}
//! A relative path is taking relative to the *project* root, NOT *engine* root.
inline constexpr AZStd::string_view ProjectBuildPath = "/Amazon/Project/Settings/Build/project_build_path";
Code/Framework/AzFramework/AzFramework/Application/Application.cpp
+13 -15
View File
@@ -707,25 +707,23 @@ namespace AzFramework
}
}
if (AZ::IO::FixedMaxPath projectUserPath;
m_settingsRegistry->Get(projectUserPath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_ProjectUserPath))
AZ::IO::FixedMaxPath engineRoot = GetEngineRoot();
AZ::IO::FixedMaxPath projectUserPath;
if (!m_settingsRegistry->Get(projectUserPath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_ProjectUserPath))
{
fileIoBase->SetAlias("@user@", projectUserPath.c_str());
AZ::IO::FixedMaxPath projectLogPath = projectUserPath / "log";
fileIoBase->SetAlias("@log@", projectLogPath.c_str());
fileIoBase->CreatePath(projectLogPath.c_str()); // Create the log directory at this point
projectUserPath = engineRoot / "user";
}
fileIoBase->SetAlias("@user@", projectUserPath.c_str());
fileIoBase->CreatePath(projectUserPath.c_str());
CreateUserCache(projectUserPath, *fileIoBase);
CreateUserCache(projectUserPath, *fileIoBase);
}
else
AZ::IO::FixedMaxPath projectLogPath;
if (!m_settingsRegistry->Get(projectLogPath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_ProjectLogPath))
{
AZ::IO::FixedMaxPath fallbackLogPath = GetEngineRoot();
fallbackLogPath /= "user";
fileIoBase->SetAlias("@user@", fallbackLogPath.c_str());
fallbackLogPath /= "log";
fileIoBase->SetAlias("@log@", fallbackLogPath.c_str());
fileIoBase->CreatePath(fallbackLogPath.c_str());
projectLogPath = projectUserPath / "log";
}
fileIoBase->SetAlias("@log@", projectLogPath.c_str());
fileIoBase->CreatePath(projectLogPath.c_str());
}
}
Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorPythonRunnerRequestsBus.h
+2 -1
View File
@@ -35,11 +35,12 @@ namespace AzToolsFramework
[[maybe_unused]] AZStd::string_view filename, [[maybe_unused]] const AZStd::vector<AZStd::string_view>& args) {}
//! executes a Python script as a test
virtual void ExecuteByFilenameAsTest(
virtual bool ExecuteByFilenameAsTest(
[[maybe_unused]] AZStd::string_view filename,
[[maybe_unused]] AZStd::string_view testCase,
[[maybe_unused]] const AZStd::vector<AZStd::string_view>& args)
{
return false;
}
};
using EditorPythonRunnerRequestBus = AZ::EBus<EditorPythonRunnerRequests>;