birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Integrating up through commit 90f050496

Browse Source
This commit is contained in:
alexpete
2021-04-07 14:03:29 -07:00
parent 8f2ed080a9
commit c2cbd430fe
2694 changed files with 285622 additions and 176874 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tools/LyTestTools/ly_test_tools/_internal/managers/abstract_resource_locator.py
+1 -1
View File
@@ -118,7 +118,7 @@ class AbstractResourceLocator(object):
def project(self):
"""
Return path to the project directory
ex. engine_root/dev/SamplesProject
ex. engine_root/dev/AutomatedTesting
:return: path to <engine_root>/dev/Project
"""
return os.path.join(self.dev(), self._project)
Tools/LyTestTools/ly_test_tools/lumberyard/asset_processor.py
+65 -30
View File
@@ -78,6 +78,8 @@ class AssetProcessor(object):
self._control_connection = None
self._function_name = None
self._failed_log_root = None
self._temp_log_directory = None
self._temp_log_root = None
self._disable_all_platforms = False
self._enabled_platform_overrides = dict()
@@ -96,7 +98,7 @@ class AssetProcessor(object):
"""
self.send_message("waitforidle")
result = self.read_message(read_timeout=timeout)
assert result == "idle", "Couldn't get idle state from AP"
assert result == "idle" or not self.process_exists(), f"Couldn't get idle state from AP, message {result}"
return True
def next_idle(self):
@@ -109,7 +111,7 @@ class AssetProcessor(object):
"""
self.send_message("signalidle")
result = self.read_message()
assert result == "idle", "Couldn't get idle state from AP"
assert result == "idle" or not self.process_exists(), f"Couldn't get idle state from AP, message {result}"
def send_quit(self):
"""
@@ -143,17 +145,17 @@ class AssetProcessor(object):
def read_message(self, read_timeout=DEFAULT_TIMEOUT_SECONDS):
"""
Read the next message from the AP Contorl socket. Must be running through gui_process/start method
Read the next message from the AP Control socket. Must be running through gui_process/start method
"""
if not self._ap_proc:
logger.warning("Attempted to read message to AP but not currently running")
return
return "no_process"
if not self._control_connection:
self.connect_control()
if not self._control_connection:
return
return "no_connection"
self._control_connection.settimeout(read_timeout)
try:
@@ -163,6 +165,8 @@ class AssetProcessor(object):
return result_message
except IOError as e:
logger.warning(f"Failed to read message from with error {e}")
return f"error_{e}"
def read_control_port(self):
@@ -178,15 +182,18 @@ class AssetProcessor(object):
start_time = time.time()
read_port_timeout = 10
while (time.time() - start_time) < read_port_timeout:
log = APLogParser(self._workspace.paths.ap_gui_log())
if len(log.runs):
try:
port = log.runs[-1][port_type]
if port:
logger.info(f"Read port type {port_type} : {port}")
return port
except:
pass
if not os.path.exists(self._workspace.paths.ap_gui_log()):
logger.debug(f"Log at {self._workspace.paths.ap_gui_log()} doesn't exist, sleeping")
else:
log = APLogParser(self._workspace.paths.ap_gui_log())
if len(log.runs):
try:
port = log.runs[-1][port_type]
if port:
logger.info(f"Read port type {port_type} : {port}")
return port
except:
pass
time.sleep(1)
logger.warning(f"Failed to read port type {port_type}")
return 0
@@ -201,8 +208,13 @@ class AssetProcessor(object):
"""
if not self._control_connection:
control_timeout = 60
return self.connect_socket("Control Connection", self.read_control_port,
try:
return self.connect_socket("Control Connection", self.read_control_port,
set_port_method=self.set_control_connection, timeout=control_timeout)
except AssetProcessorError as e:
# We dont want a failure of our test socket connection to fail the entire test automatically.
logger.error(f"Failed to connect control socket with error {e}")
pass
return True, None
def using_temp_workspace(self):
@@ -273,13 +285,13 @@ class AssetProcessor(object):
wait_timeout = timeout
try:
waiter.wait_for(lambda: not psutil.pid_exists(self.get_pid()), exc=AssetProcessorError, timeout=wait_timeout)
waiter.wait_for(lambda: not self.process_exists(), exc=AssetProcessorError, timeout=wait_timeout)
except AssetProcessorError:
logger.warning(f"Timeout attempting to quit asset processor after {wait_timeout} seconds, using terminate")
self.terminate()
raise
pass
if psutil.pid_exists(self.get_pid()):
if self.process_exists():
logger.warning(f"Failed to stop process {self.get_pid()} after {wait_timeout} seconds, using terminate")
self.terminate()
self._ap_proc = None
@@ -323,9 +335,9 @@ class AssetProcessor(object):
"""
Returns pid of asset processor proc currently executing the start command (Ap Gui)
:return: pid if exists, else 0
:return: pid if exists, else -1
"""
return self._ap_proc.pid if self._ap_proc else 0
return self._ap_proc.pid if self._ap_proc else -1
def get_process_list(self, name_filter: str = None):
"""
@@ -334,12 +346,13 @@ class AssetProcessor(object):
:param name_filter: Name to match against, such as AssetBuilder
:return: List of processes
"""
if not self._ap_proc or not self.get_pid():
my_pid = self.get_pid()
if my_pid is -1:
return []
return_list = []
try:
return_list = [psutil.Process(self.get_pid())]
return_list.extend(utils.child_process_list(self.get_pid()))
return_list = [psutil.Process(my_pid)]
return_list.extend(utils.child_process_list(my_pid))
except psutil.NoSuchProcess:
logger.warning("Process already finished calling get_process_list")
return return_list
@@ -400,9 +413,9 @@ class AssetProcessor(object):
def process_exists(self):
try:
my_pid = self.get_pid()
if my_pid == 0:
if my_pid == -1:
return False
return psutil.pid_exists(self.get_pid())
return psutil.pid_exists(my_pid)
except psutil.NoSuchProcess:
pass
return False
@@ -410,6 +423,7 @@ class AssetProcessor(object):
def batch_process(self, timeout=DEFAULT_TIMEOUT_SECONDS, fastscan=True, capture_output=False, platforms=None,
extra_params=None, add_gem_scan_folders=None, add_config_scan_folders=None, decode=True,
expect_failure=False, scan_folder_pattern=None):
self.create_temp_log_root()
ap_path = self._workspace.paths.asset_processor_batch()
command = self.build_ap_command(ap_path=ap_path, fastscan=fastscan, platforms=platforms,
extra_params=extra_params, add_gem_scan_folders=add_gem_scan_folders,
@@ -454,6 +468,7 @@ class AssetProcessor(object):
else:
extra_gui_params.append(extra_params)
self.create_temp_log_root()
command = self.build_ap_command(ap_path=ap_path, fastscan=fastscan, platforms=platforms,
extra_params=extra_gui_params, add_gem_scan_folders=add_gem_scan_folders,
add_config_scan_folders=add_config_scan_folders,
@@ -480,7 +495,7 @@ class AssetProcessor(object):
self.connect_listen()
if quitonidle:
waiter.wait_for(lambda: not psutil.pid_exists(self.get_pid()), timeout=timeout)
waiter.wait_for(lambda: not self.process_exists(), timeout=timeout)
elif run_until_idle and accept_input:
if not self.wait_for_idle():
return False, None
@@ -519,15 +534,17 @@ class AssetProcessor(object):
command.append(f'--regset="/Amazon/AzCore/Bootstrap/project_path={self._project_path}"')
# When using a scratch workspace we will set several options. The asset root controls where our
# cache lives, gives us a unique directory to place our logs in, and indicates we wish to randomize our
# cache lives.
# The logDir gives us a unique directory to place our logs in, and indicates we wish to randomize our
# listening port to avoid collisions. The port will be written to the logs which we'll retrieve it from
if self._temp_asset_root:
command.append("--assetroot")
command.append(f"{self._temp_asset_root}")
command.append("--logDir")
command.append(f"{self._temp_asset_root}")
command.append(f'--regset="/Amazon/AzCore/Bootstrap/project_cache_path={self.temp_project_cache_path()}"')
command.append("--randomListeningPort")
if self._temp_log_root:
command.append("--logDir")
command.append(f"{self._temp_log_root}")
if self._override_scan_folders:
command.append("--scanfolders")
command.append(f"{','.join(self._override_scan_folders)}")
@@ -699,11 +716,29 @@ class AssetProcessor(object):
self._temp_asset_directory = tempfile.TemporaryDirectory()
self._temp_asset_root = self._temp_asset_directory.name
self._copy_asset_root_files()
self._workspace.paths.set_ap_log_root(self._temp_asset_root)
self._project_path = os.path.join(self._temp_asset_root, self._workspace.project)
if project_scan_folder:
self.add_scan_folder(self._project_path)
def log_root(self):
"""
Return the temp log root
"""
return self._temp_log_root
def create_temp_log_root(self):
"""
Create a temp folder for logs. We want a unique temp folder for logs for each test using the AssetProcessor
test class rather than using the default logs folder which could contain any old data or be used by other
runs of asset processor.
"""
if self._temp_log_root:
logger.info(f'Cleaning up old log root at {self._temp_log_root}')
shutil.rmtree(self._temp_log_root, True)
self._temp_log_directory = tempfile.TemporaryDirectory()
self._temp_log_root = self._temp_log_directory.name
self._workspace.paths.set_ap_log_root(self._temp_log_root)
def add_scan_folder(self, folder_name) -> str:
"""
Add a new folder as one of the default folders which AP Batch should scan on the next run
Tools/LyTestTools/tests/example/test_system_example.py
+3 -3
View File
@@ -51,7 +51,7 @@ def remote_console(request):
# Shared parameters & fixtures for all test methods inside the TestSystemExample class.
@pytest.mark.usefixtures("automatic_process_killer")
@pytest.mark.parametrize('project', ['SamplesProject'])
@pytest.mark.parametrize('project', ['AutomatedTesting'])
class TestSystemExample(object):
"""
Example test case class to hold a set of test case methods.
@@ -68,11 +68,11 @@ class TestSystemExample(object):
@pytest.mark.parametrize('level', ['simple_jacklocomotion'])
@pytest.mark.parametrize('load_wait', [120])
@pytest.mark.test_case_id('C16806863')
def test_SystemTestExample_AllSupportedPlatforms_LaunchSamplesProject(
def test_SystemTestExample_AllSupportedPlatforms_LaunchAutomatedTesting(
# launcher_platform, asset_processor_platform, # Re-add these here if you plan to use them.
self, launcher, remote_console, level, load_wait):
"""
Tests launching the SamplesProject then launches the Lumberyard client &
Tests launching the AutomatedTesting then launches the Lumberyard client &
loads the "simple_jacklocomotion" level using the remote console.
Assumes the user already setup & built their machine for the test.
"""
Tools/LyTestTools/tests/unit/test_artifact_manager.py
-1
View File
@@ -22,7 +22,6 @@ import pytest
import ly_test_tools._internal.managers.artifact_manager
pytestmark = pytest.mark.SUITE_smoke
TIMESTAMP_FORMAT = '%Y-%m-%dT%H-%M-%S-%f'
DATE = datetime.datetime.now().strftime(TIMESTAMP_FORMAT)
Tools/LyTestTools/tests/unit/test_asset_processor.py
+14 -10
View File
@@ -61,6 +61,7 @@ class TestAssetProcessor(object):
assert under_test._ap_proc is not None
mock_popen.assert_called_once_with([mock_ap_path, '--zeroAnalysisMode', '--regset="/Amazon/AzCore/Bootstrap/project_path=AutomatedTesting"',
'--logDir', under_test.log_root(),
'--acceptInput', '--platforms', 'bar'], cwd=os.path.dirname(mock_ap_path))
mock_connect.assert_called()
@@ -68,7 +69,8 @@ class TestAssetProcessor(object):
@mock.patch('subprocess.Popen')
@mock.patch('os.path.basename', mock.MagicMock(return_value=""))
@mock.patch('os.path.dirname', mock.MagicMock(return_value=""))
@mock.patch('ly_test_tools.environment.process_utils.kill_processes_with_name_not_started_from', mock.MagicMock(return_value=None))
@mock.patch('ly_test_tools.environment.process_utils.kill_processes_with_name_not_started_from',
mock.MagicMock(return_value=None))
@mock.patch('ly_test_tools.environment.process_utils.process_exists', mock.MagicMock(return_value=True))
@mock.patch('socket.socket.connect')
def test_Start_ProcAlreadyRunning_ProcNotChanged(self, mock_connect, mock_popen, mock_workspace):
@@ -101,7 +103,6 @@ class TestAssetProcessor(object):
mock_waiter.assert_called_once()
assert under_test._ap_proc is None
@mock.patch('ly_test_tools._internal.managers.workspace.AbstractWorkspaceManager')
@mock.patch('subprocess.run')
def test_BatchProcess_NoFastscanBatchCompletes_Success(self, mock_run, mock_workspace):
@@ -112,7 +113,8 @@ class TestAssetProcessor(object):
result, _ = under_test.batch_process(1, False)
assert result
mock_run.assert_called_once_with([apb_path], close_fds=True, capture_output=False,
mock_run.assert_called_once_with([apb_path, '--logDir', under_test.log_root()],
close_fds=True, capture_output=False,
timeout=1)
@mock.patch('ly_test_tools._internal.managers.workspace.AbstractWorkspaceManager')
@@ -126,9 +128,13 @@ class TestAssetProcessor(object):
result = under_test.batch_process(1, True)
assert result
mock_run.assert_called_once_with([apb_path, '--zeroAnalysisMode', '--regset="/Amazon/AzCore/Bootstrap/project_path=AutomatedTesting"'],
close_fds=True, capture_output=False,
timeout=1)
mock_run.assert_called_once_with(
[apb_path, '--zeroAnalysisMode', '--regset="/Amazon/AzCore/Bootstrap/project_path=AutomatedTesting"',
'--logDir',
under_test.log_root()],
close_fds=True, capture_output=False,
timeout=1)
@mock.patch('ly_test_tools._internal.managers.workspace.AbstractWorkspaceManager')
@mock.patch('subprocess.run')
@@ -141,8 +147,8 @@ class TestAssetProcessor(object):
result, _ = under_test.batch_process(None, False)
assert not result
mock_run.assert_called_once_with([apb_path], close_fds=True, capture_output=False, timeout=28800.0)
mock_run.assert_called_once_with([apb_path, '--logDir', under_test.log_root()],
close_fds=True, capture_output=False, timeout=28800.0)
@mock.patch('ly_test_tools._internal.managers.workspace.AbstractWorkspaceManager')
def test_EnableAssetProcessorPlatform_AssetProcessorObject_Updated(self, mock_workspace):
@@ -181,5 +187,3 @@ class TestAssetProcessor(object):
mock_stop.assert_called()
mock_restore_ap.assert_called()
Tools/LyTestTools/tests/unit/test_fixtures.py
+4 -4
View File
@@ -138,9 +138,9 @@ class TestFixtures(object):
@mock.patch("ly_test_tools.builtin.helpers.create_builtin_workspace")
def test_Workspace_MockFixturesAndExecTeardown_ReturnWorkspaceRegisterTeardown(self, mock_create, mock_setup):
test_module = 'example.tests.test_system_example'
test_class = ('TestSystemExample.test_SystemTestExample_AllSupportedPlatforms_LaunchSamplesProject'
'[120-simple_jacklocomotion-SamplesProject-all-profile-win_x64_vs2017]')
test_method = 'test_SystemTestExample_AllSupportedPlatforms_LaunchSamplesProject'
test_class = ('TestSystemExample.test_SystemTestExample_AllSupportedPlatforms_LaunchAutomatedTesting'
'[120-simple_jacklocomotion-AutomatedTesting-all-profile-win_x64_vs2017]')
test_method = 'test_SystemTestExample_AllSupportedPlatforms_LaunchAutomatedTesting'
artifact_folder_name = 'TheArtifactFolder'
artifact_path = "PathToArtifacts"
@@ -183,7 +183,7 @@ class TestFixtures(object):
mock_workspace.artifact_manager.generate_folder_name.assert_called_with(
test_module.split('.')[-1], # 'example.tests.test_system_example' -> 'test_system_example'
test_class.split('.')[0], # 'TestSystemExample.test_SystemTestExample_...' -> 'TestSystemExample'
test_method # 'test_SystemTestExample_AllSupportedPlatforms_LaunchSamplesProject'
test_method # 'test_SystemTestExample_AllSupportedPlatforms_LaunchAutomatedTesting'
)
@mock.patch('os.path.exists', mock.MagicMock(return_value=True))