birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #6714 from aws-lumberyard-dev/crash_log_bug

Browse Source 
Crash log test tools bug
This commit is contained in:
Gene Walters
2022-01-12 12:24:07 -08:00
committed by GitHub
parent 5dc442fcb0 8fd6d534b3
commit c056ae705d
6 changed files with 50 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/test_tools_fixtures.py
+1 -1
View File
@@ -413,7 +413,7 @@ def crash_log_watchdog(request, workspace):
def _crash_log_watchdog(request, workspace, raise_on_crash):
"""Separate implementation to call directly during unit tests"""
error_log = os.path.join(workspace.paths.project_log(), 'error.log')
error_log = workspace.paths.crash_log()
crash_log_watchdog = ly_test_tools.environment.watchdog.CrashLogWatchdog(
error_log, raise_on_condition=raise_on_crash)
Tools/LyTestTools/ly_test_tools/o3de/editor_test.py
+13 -4
View File
@@ -762,7 +762,7 @@ class EditorTestSuite():
cmdline = [
"--runpythontest", test_filename,
"-logfile", f"@log@/{log_name}",
"-project-log-path", ly_test_tools._internal.pytest_plugin.output_path] + test_cmdline_args
"-project-log-path", editor_utils.retrieve_log_path(run_id, workspace)] + test_cmdline_args
editor.args.extend(cmdline)
editor.start(backupFiles = False, launch_ap = False, configure_settings=False)
@@ -771,7 +771,8 @@ class EditorTestSuite():
output = editor.get_output()
return_code = editor.get_returncode()
editor_log_content = editor_utils.retrieve_editor_log_content(run_id, log_name, workspace)
# Save the editor log
workspace.artifact_manager.save_artifact(os.path.join(editor_utils.retrieve_log_path(run_id, workspace), log_name))
if return_code == 0:
test_result = Result.Pass.create(test_spec, output, editor_log_content)
else:
@@ -779,6 +780,9 @@ class EditorTestSuite():
if has_crashed:
test_result = Result.Crash.create(test_spec, output, return_code, editor_utils.retrieve_crash_output
(run_id, workspace, self._TIMEOUT_CRASH_LOG), None)
# Save the crash log
crash_file_name = os.path.basename(workspace.paths.crash_log())
workspace.artifact_manager.save_artifact(os.path.join(editor_utils.retrieve_log_path(run_id, workspace), crash_file_name))
editor_utils.cycle_crash_report(run_id, workspace)
else:
test_result = Result.Fail.create(test_spec, output, editor_log_content)
@@ -830,7 +834,7 @@ class EditorTestSuite():
cmdline = [
"--runpythontest", test_filenames_str,
"-logfile", f"@log@/{log_name}",
"-project-log-path", ly_test_tools._internal.pytest_plugin.output_path] + test_cmdline_args
"-project-log-path", editor_utils.retrieve_log_path(run_id, workspace)] + test_cmdline_args
editor.args.extend(cmdline)
editor.start(backupFiles = False, launch_ap = False, configure_settings=False)
@@ -842,7 +846,8 @@ class EditorTestSuite():
output = editor.get_output()
return_code = editor.get_returncode()
editor_log_content = editor_utils.retrieve_editor_log_content(run_id, log_name, workspace)
# Save the editor log
workspace.artifact_manager.save_artifact(os.path.join(editor_utils.retrieve_log_path(run_id, workspace), log_name))
if return_code == 0:
# No need to scrap the output, as all the tests have passed
for test_spec in test_spec_list:
@@ -863,6 +868,10 @@ class EditorTestSuite():
# The first test with "Unknown" result (no data in output) is likely the one that crashed
crash_error = editor_utils.retrieve_crash_output(run_id, workspace,
self._TIMEOUT_CRASH_LOG)
# Save the crash log
crash_file_name = os.path.basename(workspace.paths.crash_log())
workspace.artifact_manager.save_artifact(
os.path.join(editor_utils.retrieve_log_path(run_id, workspace), crash_file_name))
editor_utils.cycle_crash_report(run_id, workspace)
results[test_spec_name] = Result.Crash.create(result.test_spec, output, return_code,
crash_error, result.editor_log)
Tools/LyTestTools/ly_test_tools/o3de/editor_test_utils.py
+4 -2
View File
@@ -71,7 +71,9 @@ def retrieve_crash_output(run_id: int, workspace: AbstractWorkspaceManager, time
:return str: The contents of the editor crash file (error.log)
"""
crash_info = "-- No crash log available --"
crash_log = os.path.join(retrieve_log_path(run_id, workspace), 'error.log')
# Grab the file name of the crash log which can be different depending on platform
crash_file_name = os.path.basename(workspace.paths.crash_log())
crash_log = os.path.join(retrieve_log_path(run_id, workspace), crash_file_name)
try:
waiter.wait_for(lambda: os.path.exists(crash_log), timeout=timeout)
except AssertionError:
@@ -92,7 +94,7 @@ def cycle_crash_report(run_id: int, workspace: AbstractWorkspaceManager) -> None
:param workspace: Workspace fixture
"""
log_path = retrieve_log_path(run_id, workspace)
files_to_cycle = ['error.log', 'error.dmp']
files_to_cycle = ['crash.log', 'error.log', 'error.dmp']
for filename in files_to_cycle:
filepath = os.path.join(log_path, filename)
name, ext = os.path.splitext(filename)
Tools/LyTestTools/tests/unit/test_editor_test_utils.py
+14 -5
View File
@@ -60,22 +60,31 @@ class TestEditorTestUtils(unittest.TestCase):
assert expected == editor_test_utils.retrieve_log_path(0, mock_workspace)
@mock.patch('os.listdir')
@mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
@mock.patch('os.path.join', mock.MagicMock())
@mock.patch('os.path.basename', mock.MagicMock())
@mock.patch('os.path.isfile', mock.MagicMock())
@mock.patch('ly_test_tools.environment.waiter.wait_for', mock.MagicMock())
def test_RetrieveCrashOutput_CrashLogExists_ReturnsLogInfo(self, mock_retrieve_log_path):
mock_retrieve_log_path.return_value = 'mock_log_path'
def test_RetrieveCrashOutput_CrashLogExists_ReturnsLogInfo(self, mock_retrieve_log_path, mock_listdir):
mock_retrieve_log_path.return_value = 'mock_path'
mock_workspace = mock.MagicMock()
mock_log = 'mock crash info'
mock_listdir.return_value = ['mock_error_log.log']
with mock.patch('builtins.open', mock.mock_open(read_data=mock_log)) as mock_file:
assert mock_log == editor_test_utils.retrieve_crash_output(0, mock_workspace, 0)
@mock.patch('os.listdir')
@mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
@mock.patch('os.path.isfile', mock.MagicMock())
@mock.patch('ly_test_tools.environment.waiter.wait_for', mock.MagicMock())
def test_RetrieveCrashOutput_CrashLogNotExists_ReturnsError(self, mock_retrieve_log_path):
def test_RetrieveCrashOutput_CrashLogNotExists_ReturnsError(self, mock_retrieve_log_path, mock_listdir):
mock_retrieve_log_path.return_value = 'mock_log_path'
mock_workspace = mock.MagicMock()
mock_workspace.paths.crash_log.return_value = 'mock_file.log'
error_message = "No crash log available"
mock_listdir.return_value = ['mock_file.log']
assert error_message in editor_test_utils.retrieve_crash_output(0, mock_workspace, 0)
@@ -86,7 +95,7 @@ class TestEditorTestUtils(unittest.TestCase):
@mock.patch('os.path.exists')
def test_CycleCrashReport_DmpExists_NamedCorrectly(self, mock_exists, mock_retrieve_log_path, mock_strftime,
mock_rename):
mock_exists.side_effect = [False, True]
mock_exists.side_effect = [False, False, True]
mock_retrieve_log_path.return_value = 'mock_log_path'
mock_workspace = mock.MagicMock()
mock_strftime.return_value = 'mock_strftime'
@@ -102,7 +111,7 @@ class TestEditorTestUtils(unittest.TestCase):
@mock.patch('os.path.exists')
def test_CycleCrashReport_LogExists_NamedCorrectly(self, mock_exists, mock_retrieve_log_path, mock_strftime,
mock_rename):
mock_exists.side_effect = [True, False]
mock_exists.side_effect = [False, True, False]
mock_retrieve_log_path.return_value = 'mock_log_path'
mock_workspace = mock.MagicMock()
mock_strftime.return_value = 'mock_strftime'
Tools/LyTestTools/tests/unit/test_fixtures.py
+5 -6
View File
@@ -323,20 +323,19 @@ class TestFixtures(object):
@mock.patch('ly_test_tools.environment.watchdog.CrashLogWatchdog')
def test_CrashLogWatchdog_Instantiates_CreatesWatchdog(self, under_test):
mock_workspace = mock.MagicMock()
mock_path = 'C:/foo'
mock_workspace.paths.project_log.return_value = mock_path
mock_workspace.paths.crash_log.return_value = mock.MagicMock()
mock_request = mock.MagicMock()
mock_request.addfinalizer = mock.MagicMock()
mock_raise_on_crash = mock.MagicMock()
mock_watchdog = test_tools_fixtures._crash_log_watchdog(mock_request, mock_workspace, mock_raise_on_crash)
under_test.assert_called_once_with(os.path.join(mock_path, 'error.log'), raise_on_condition=mock_raise_on_crash)
under_test.assert_called_once_with(mock_workspace.paths.crash_log.return_value, raise_on_condition=mock_raise_on_crash)
@mock.patch('ly_test_tools.environment.watchdog.CrashLogWatchdog.start')
def test_CrashLogWatchdog_Instantiates_StartsThread(self, under_test):
mock_workspace = mock.MagicMock()
mock_path = 'C:/foo'
mock_workspace.paths.project_log.return_value = mock_path
mock_workspace.paths.crash_log.return_value = mock_path
mock_request = mock.MagicMock()
mock_request.addfinalizer = mock.MagicMock()
mock_raise_on_crash = mock.MagicMock()
@@ -348,7 +347,7 @@ class TestFixtures(object):
def test_CrashLogWatchdog_Instantiates_AddsTeardown(self):
mock_workspace = mock.MagicMock()
mock_path = 'C:/foo'
mock_workspace.paths.project_log.return_value = mock_path
mock_workspace.paths.crash_log.return_value = mock_path
mock_request = mock.MagicMock()
mock_request.addfinalizer = mock.MagicMock()
mock_raise_on_crash = mock.MagicMock()
@@ -361,7 +360,7 @@ class TestFixtures(object):
def test_CrashLogWatchdog_Teardown_CallsStop(self, mock_stop):
mock_workspace = mock.MagicMock()
mock_path = 'C:/foo'
mock_workspace.paths.project_log.return_value = mock_path
mock_workspace.paths.crash_log.return_value = mock_path
mock_request = mock.MagicMock()
mock_request.addfinalizer = mock.MagicMock()
mock_raise_condition = mock.MagicMock()
Tools/LyTestTools/tests/unit/test_o3de_editor_test.py
+13
View File
@@ -589,6 +589,7 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
@mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
@mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
@mock.patch('os.path.join', mock.MagicMock())
def test_ExecEditorTest_TestSucceeds_ReturnsPass(self, mock_cycle_crash, mock_get_testcase_filepath,
mock_retrieve_log, mock_retrieve_editor_log,
mock_get_output_results, mock_create):
@@ -616,6 +617,7 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
@mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
@mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
@mock.patch('os.path.join', mock.MagicMock())
def test_ExecEditorTest_TestFails_ReturnsFail(self, mock_cycle_crash, mock_get_testcase_filepath,
mock_retrieve_log, mock_retrieve_editor_log,
mock_get_output_results, mock_create):
@@ -644,6 +646,8 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
@mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
@mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
@mock.patch('os.path.join', mock.MagicMock())
@mock.patch('os.path.basename', mock.MagicMock())
def test_ExecEditorTest_TestCrashes_ReturnsCrash(self, mock_cycle_crash, mock_get_testcase_filepath,
mock_retrieve_log, mock_retrieve_editor_log,
mock_get_output_results, mock_retrieve_crash, mock_create):
@@ -699,10 +703,14 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
@mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
@mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
@mock.patch('os.path.join', mock.MagicMock())
def test_ExecEditorMultitest_AllTestsPass_ReturnsPasses(self, mock_cycle_crash, mock_get_testcase_filepath,
mock_retrieve_log, mock_retrieve_editor_log, mock_create):
mock_test_suite = ly_test_tools.o3de.editor_test.EditorTestSuite()
mock_workspace = mock.MagicMock()
mock_artifact_manager = mock.MagicMock()
mock_artifact_manager.save_artifact.return_value = mock.MagicMock()
mock_workspace.artifact_manager = mock_artifact_manager
mock_workspace.paths.engine_root.return_value = ""
mock_editor = mock.MagicMock()
mock_editor.get_returncode.return_value = 0
@@ -727,6 +735,7 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
@mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
@mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
@mock.patch('os.path.join', mock.MagicMock())
def test_ExecEditorMultitest_OneFailure_CallsCorrectFunc(self, mock_cycle_crash, mock_get_testcase_filepath,
mock_retrieve_log, mock_retrieve_editor_log, mock_get_results):
mock_test_suite = ly_test_tools.o3de.editor_test.EditorTestSuite()
@@ -752,6 +761,8 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
@mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
@mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
@mock.patch('os.path.join', mock.MagicMock())
@mock.patch('os.path.basename', mock.MagicMock())
def test_ExecEditorMultitest_OneCrash_ReportsOnUnknownResult(self, mock_cycle_crash, mock_get_testcase_filepath,
mock_retrieve_log, mock_retrieve_editor_log,
mock_get_results, mock_retrieve_crash, mock_create):
@@ -787,6 +798,8 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
@mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
@mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
@mock.patch('os.path.join', mock.MagicMock())
@mock.patch('os.path.basename', mock.MagicMock())
def test_ExecEditorMultitest_ManyUnknown_ReportsUnknownResults(self, mock_cycle_crash, mock_get_testcase_filepath,
mock_retrieve_log, mock_retrieve_editor_log,
mock_get_results, mock_retrieve_crash, mock_create):