birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Collects failed assets on LyTestTools test failures (#7368)

Browse Source 
* Collects failed assets on LyTestTools test failures

Signed-off-by: evanchia <evanchia@amazon.com>

* Changed query to all lines because of current asset logging bug

Signed-off-by: evanchia <evanchia@amazon.com>

* fixes per pr feedback

Signed-off-by: evanchia <evanchia@amazon.com>

* testing removing change for AR failure

Signed-off-by: evanchia <evanchia@amazon.com>

* Moved asset log artifact collection to after the results have been collected

Signed-off-by: evanchia <evanchia@amazon.com>

* Adding debug line to help debug AR failure

Signed-off-by: evanchia <evanchia@amazon.com>

* Made asset log collection non failing

Signed-off-by: evanchia <evanchia@amazon.com>

* moved asset log collection after test result collection

Signed-off-by: evanchia <evanchia@amazon.com>

* changed asset saving to non failing

Signed-off-by: evanchia <evanchia@amazon.com>

* improved logging and comments

Signed-off-by: evanchia <evanchia@amazon.com>
This commit is contained in:
evanchia-ly-sdets
2022-02-09 18:13:01 -08:00
committed by GitHub
parent cdedd38767
commit 5ef2b12dca
5 changed files with 132 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tools/LyTestTools/tests/unit/test_editor_test_utils.py
+56
View File
@@ -170,3 +170,59 @@ class TestEditorTestUtils(unittest.TestCase):
mock_test_list.append(mock_test)
assert 0 == editor_test_utils.retrieve_last_run_test_index_from_output(mock_test_list, mock_editor_output)
@mock.patch('ly_test_tools.o3de.editor_test_utils._check_log_errors_warnings')
@mock.patch('os.walk')
def test_SaveFailedAssetJoblogs_ManyValidLogs_SavesCorrectly(self, mock_walk, mock_check_log):
mock_workspace = mock.MagicMock()
mock_walk.return_value = [['MockDirectory', None, ['mock_log.log']],
['MockDirectory2', None, ['mock_log2.log']]]
mock_check_log.return_value = True
editor_test_utils.save_failed_asset_joblogs(mock_workspace)
assert mock_workspace.artifact_manager.save_artifact.call_count == 2
@mock.patch('ly_test_tools.o3de.editor_test_utils._check_log_errors_warnings')
@mock.patch('os.walk')
def test_SaveFailedAssetJoblogs_ManyInvalidLogs_NoSaves(self, mock_walk, mock_check_log):
mock_workspace = mock.MagicMock()
mock_walk.return_value = [['MockDirectory', None, ['mock_log.log']],
['MockDirectory2', None, ['mock_log2.log']]]
mock_check_log.return_value = False
editor_test_utils.save_failed_asset_joblogs(mock_workspace)
assert mock_workspace.artifact_manager.save_artifact.call_count == 0
def test_CheckLogErrorWarnings_ValidLine_ReturnsTrue(self):
mock_log = '~~1643759303647~~1~~00000000000009E0~~AssetBuilder~~S: 0 errors, 1 warnings'
mock_log_path = mock.MagicMock()
with mock.patch('builtins.open', mock.mock_open(read_data=mock_log)) as mock_file:
expected = editor_test_utils._check_log_errors_warnings(mock_log_path)
assert expected
def test_CheckLogErrorWarnings_MultipleValidLine_ReturnsTrue(self):
mock_log = 'foo\nfoo\n~~1643759303647~~1~~00000000000009E0~~AssetBuilder~~S: 1 errors, 1 warnings'
mock_log_path = mock.MagicMock()
with mock.patch('builtins.open', mock.mock_open(read_data=mock_log)) as mock_file:
expected = editor_test_utils._check_log_errors_warnings(mock_log_path)
assert expected
def test_CheckLogErrorWarnings_InvalidLine_ReturnsFalse(self):
mock_log = 'foo\n~~1643759303647~~1~~00000000000009E0~~AssetBuilder~~S: 0 errors, 0 warnings'
mock_log_path = mock.MagicMock()
with mock.patch('builtins.open', mock.mock_open(read_data=mock_log)) as mock_file:
expected = editor_test_utils._check_log_errors_warnings(mock_log_path)
assert not expected
def test_CheckLogErrorWarnings_InvalidRegex_ReturnsTrue(self):
mock_log = 'Invalid last line'
mock_log_path = mock.MagicMock()
with mock.patch('builtins.open', mock.mock_open(read_data=mock_log)) as mock_file:
expected = editor_test_utils._check_log_errors_warnings(mock_log_path)
assert expected
Tools/LyTestTools/tests/unit/test_o3de_editor_test.py
+7
View File
@@ -871,6 +871,7 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._report_result')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._exec_editor_test')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._setup_editor_test')
@mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
def test_RunSingleTest_ValidTest_ReportsResults(self, mock_setup_test, mock_exec_editor_test, mock_report_result):
mock_test_suite = ly_test_tools.o3de.editor_test.EditorTestSuite()
mock_test_data = mock.MagicMock()
@@ -903,6 +904,7 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('threading.Thread')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._get_number_parallel_editors')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._setup_editor_test')
@mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
def test_RunParallelTests_TwoTestsAndEditors_TwoThreads(self, mock_setup_test, mock_get_num_editors, mock_thread):
mock_test_suite = ly_test_tools.o3de.editor_test.EditorTestSuite()
mock_get_num_editors.return_value = 2
@@ -919,6 +921,7 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('threading.Thread')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._get_number_parallel_editors')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._setup_editor_test')
@mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
def test_RunParallelTests_TenTestsAndTwoEditors_TenThreads(self, mock_setup_test, mock_get_num_editors, mock_thread):
mock_test_suite = ly_test_tools.o3de.editor_test.EditorTestSuite()
mock_get_num_editors.return_value = 2
@@ -937,6 +940,7 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('threading.Thread')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._get_number_parallel_editors')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._setup_editor_test')
@mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
def test_RunParallelTests_TenTestsAndThreeEditors_TenThreads(self, mock_setup_test, mock_get_num_editors,
mock_thread):
mock_test_suite = ly_test_tools.o3de.editor_test.EditorTestSuite()
@@ -956,6 +960,7 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('threading.Thread')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._get_number_parallel_editors')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._setup_editor_test')
@mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
def test_RunParallelBatchedTests_TwoTestsAndEditors_TwoThreads(self, mock_setup_test, mock_get_num_editors,
mock_thread):
mock_test_suite = ly_test_tools.o3de.editor_test.EditorTestSuite()
@@ -973,6 +978,7 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('threading.Thread')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._get_number_parallel_editors')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._setup_editor_test')
@mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
def test_RunParallelBatchedTests_TenTestsAndTwoEditors_2Threads(self, mock_setup_test, mock_get_num_editors,
mock_thread):
mock_test_suite = ly_test_tools.o3de.editor_test.EditorTestSuite()
@@ -992,6 +998,7 @@ class TestRunningTests(unittest.TestCase):
@mock.patch('threading.Thread')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._get_number_parallel_editors')
@mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite._setup_editor_test')
@mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
def test_RunParallelBatchedTests_TenTestsAndThreeEditors_ThreeThreads(self, mock_setup_test, mock_get_num_editors,
mock_thread):
mock_test_suite = ly_test_tools.o3de.editor_test.EditorTestSuite()