Change test module default timeouts to 25 minutes to unblock Periodic and Sandbox CI builds, without incurring the costs involved with increased log verbosity.

2021-04-26 12:38:00 -07:00
parent fb26308a26
commit d8afec36cd
4 changed files with 23 additions and 21 deletions
@@ -25,7 +25,7 @@ add_subdirectory(assetpipeline)
 #        TEST_SUITE main
 #        TEST_SERIAL
 #        PATH ${CMAKE_CURRENT_LIST_DIR}/physics/TestSuite_Active.py
-#        TIMEOUT 3600
+#        TIMEOUT 1500
 #        RUNTIME_DEPENDENCIES
 #            Legacy::Editor
 #            Legacy::CryRenderNULL
@@ -39,7 +39,7 @@ add_subdirectory(assetpipeline)
 #        TEST_SUITE sandbox
 #        TEST_SERIAL
 #        PATH ${CMAKE_CURRENT_LIST_DIR}/physics/TestSuite_Sandbox.py
-#        TIMEOUT 3600
+#        TIMEOUT 1500
 #        RUNTIME_DEPENDENCIES
 #            Legacy::Editor
 #            Legacy::CryRenderNULL
@@ -57,7 +57,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE periodic
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/scripting/TestSuite_Active.py
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@@ -71,7 +71,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE sandbox
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/scripting/TestSuite_Sandbox.py
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@@ -88,7 +88,7 @@ endif()
 #        TEST_SUITE main
 #        TEST_SERIAL
 #        PATH ${CMAKE_CURRENT_LIST_DIR}/WhiteBox/TestSuite_Active.py
-#        TIMEOUT 3600
+#        TIMEOUT 1500
 #        RUNTIME_DEPENDENCIES
 #            Legacy::Editor
 #            Legacy::CryRenderNULL
@@ -107,7 +107,7 @@ endif()
 #        TEST_SUITE main
 #        TEST_SERIAL
 #        PATH ${CMAKE_CURRENT_LIST_DIR}/NvCloth/TestSuite_Active.py
-#        TIMEOUT 3600
+#        TIMEOUT 1500
 #        RUNTIME_DEPENDENCIES
 #            Legacy::Editor
 #            Legacy::CryRenderNULL
@@ -123,7 +123,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE sandbox
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/EditorPythonBindings
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@@ -141,7 +141,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE periodic
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/PythonAssetBuilder
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@@ -160,7 +160,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE periodic
        TEST_SERIAL TRUE
        PATH ${CMAKE_CURRENT_LIST_DIR}/Blast/TestSuite_Active.py
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            AZ::AssetProcessor
@@ -183,7 +183,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE main
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/dyn_veg
        PYTEST_MARKS "not SUITE_sandbox and not SUITE_periodic and not SUITE_benchmark"
-        TIMEOUT 36000
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessor
            Legacy::Editor
@@ -200,7 +200,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE sandbox
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/dyn_veg
        PYTEST_MARKS "SUITE_sandbox"
-        TIMEOUT 36000
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessor
            Legacy::Editor
@@ -217,7 +217,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE periodic
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/dyn_veg
        PYTEST_MARKS "SUITE_periodic"
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessor
            Legacy::Editor
@@ -234,7 +234,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE main
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/landscape_canvas
        PYTEST_MARKS "not SUITE_sandbox and not SUITE_periodic and not SUITE_benchmark"
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
           AZ::AssetProcessor
           Legacy::Editor
@@ -250,7 +250,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE periodic
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/landscape_canvas
        PYTEST_MARKS "SUITE_periodic"
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
           AZ::AssetProcessor
           Legacy::Editor
@@ -266,7 +266,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SERIAL
        TEST_SUITE periodic
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/gradient_signal
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessor
            Legacy::Editor
@@ -284,7 +284,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE periodic
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/editor
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@@ -100,7 +100,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        PATH ${CMAKE_CURRENT_LIST_DIR}/asset_bundler_batch_tests.py
        EXCLUDE_TEST_RUN_TARGET_FROM_IDE
        TEST_SERIAL 
-        TIMEOUT 3600
+        TIMEOUT 1500
        TEST_SUITE periodic
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessor
@@ -123,7 +123,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        PATH ${CMAKE_CURRENT_LIST_DIR}/missing_dependency_tests.py
        EXCLUDE_TEST_RUN_TARGET_FROM_IDE
        TEST_SERIAL 
-        TIMEOUT 3600
+        TIMEOUT 1500
        TEST_SUITE periodic
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessorBatch
@@ -166,7 +166,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
    ly_add_googlebenchmark(
        NAME Gem::PhysX.Benchmarks
        TARGET Gem::PhysX.Tests
-        TIMEOUT 2400 #40mins
+        TIMEOUT 1500 #25mins
    )

    list(APPEND testTargets PhysX.Tests)
@@ -24,8 +24,8 @@ ly_set(LY_PYTEST_EXECUTABLE ${LY_PYTHON_CMD} -B -m pytest -v --tb=short --show-c
 ly_set(LY_TEST_GLOBAL_KNOWN_SUITE_NAMES "smoke" "main" "periodic" "benchmark" "sandbox")
 ly_set(LY_TEST_GLOBAL_KNOWN_REQUIREMENTS "gpu")

-# Set default to 20 minutes
-ly_set(LY_TEST_DEFAULT_TIMEOUT 1200)
+# Set default test aborts to 25 minutes, avoids hitting the CI pipeline inactivity timeout usually set to 30 minutes
+ly_set(LY_TEST_DEFAULT_TIMEOUT 1500)

 # Add the CMake Test targets for each suite if testing is supported
 if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
@@ -115,6 +115,8 @@ function(ly_add_test)
    # Set default test module timeout
    if(NOT ly_add_test_TIMEOUT)
        set(ly_add_test_TIMEOUT ${LY_TEST_DEFAULT_TIMEOUT})
+    elseif(ly_add_test_TIMEOUT GREATER LY_TEST_DEFAULT_TIMEOUT)
+        message(WARNING "TIMEOUT for test ${ly_add_test_NAME} set at ${ly_add_test_TIMEOUT} seconds which is longer than the default of ${LY_TEST_DEFAULT_TIMEOUT}. Allowing a single module to run exceedingly long creates problems in a CI pipeline.")
    endif()

    if(NOT ly_add_test_TEST_COMMAND)