Merge pull request #321 from aws-lumberyard-dev/modify_test_timeout

Modify test timeouts
5 years ago · dcb253e08c
parent 292e94fae8 924fee57a5
commit dcb253e08c
4 changed files with 23 additions and 21 deletions
--- a/AutomatedTesting/Gem/PythonTests/CMakeLists.txt
+++ b/AutomatedTesting/Gem/PythonTests/CMakeLists.txt
@ -25,7 +25,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE main
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/physics/TestSuite_Main.py
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@ -39,7 +39,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE periodic
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/physics/TestSuite_Periodic.py
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@ -71,7 +71,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE periodic
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/scripting/TestSuite_Active.py
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@ -85,7 +85,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE sandbox
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/scripting/TestSuite_Sandbox.py
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@ -101,7 +101,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE main
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/WhiteBox/TestSuite_Active.py
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@ -120,7 +120,7 @@ endif()
 #        TEST_SUITE main
 #        TEST_SERIAL
 #        PATH ${CMAKE_CURRENT_LIST_DIR}/NvCloth/TestSuite_Active.py
-#        TIMEOUT 3600
+#        TIMEOUT 1500
 #        RUNTIME_DEPENDENCIES
 #            Legacy::Editor
 #            Legacy::CryRenderNULL
@ -136,7 +136,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE sandbox
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/EditorPythonBindings
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@ -154,7 +154,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE periodic
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/PythonAssetBuilder
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
@ -173,7 +173,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        TEST_SUITE main
        TEST_SERIAL TRUE
        PATH ${CMAKE_CURRENT_LIST_DIR}/Blast/TestSuite_Active.py
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            AZ::AssetProcessor
@ -196,7 +196,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE main
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/dyn_veg
        PYTEST_MARKS "not SUITE_sandbox and not SUITE_periodic and not SUITE_benchmark"
-        TIMEOUT 36000
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessor
            Legacy::Editor
@ -212,7 +212,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE sandbox
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/dyn_veg
        PYTEST_MARKS "SUITE_sandbox"
-        TIMEOUT 36000
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessor
            Legacy::Editor
@ -228,7 +228,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE periodic
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/dyn_veg
        PYTEST_MARKS "SUITE_periodic"
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessor
            Legacy::Editor
@ -244,7 +244,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE main
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/landscape_canvas
        PYTEST_MARKS "not SUITE_sandbox and not SUITE_periodic and not SUITE_benchmark"
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
           AZ::AssetProcessor
           Legacy::Editor
@ -259,7 +259,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE periodic
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/landscape_canvas
        PYTEST_MARKS "SUITE_periodic"
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
           AZ::AssetProcessor
           Legacy::Editor
@ -274,7 +274,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SERIAL
        TEST_SUITE periodic
        PATH ${CMAKE_CURRENT_LIST_DIR}/largeworlds/gradient_signal
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessor
            Legacy::Editor
@ -292,7 +292,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
        TEST_SUITE periodic
        TEST_SERIAL
        PATH ${CMAKE_CURRENT_LIST_DIR}/editor
-        TIMEOUT 3600
+        TIMEOUT 1500
        RUNTIME_DEPENDENCIES
            Legacy::Editor
            Legacy::CryRenderNULL
--- a/AutomatedTesting/Gem/PythonTests/assetpipeline/asset_processor_tests/CMakeLists.txt
+++ b/AutomatedTesting/Gem/PythonTests/assetpipeline/asset_processor_tests/CMakeLists.txt
@ -100,7 +100,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        PATH ${CMAKE_CURRENT_LIST_DIR}/asset_bundler_batch_tests.py
        EXCLUDE_TEST_RUN_TARGET_FROM_IDE
        TEST_SERIAL 
-        TIMEOUT 3600
+        TIMEOUT 1500
        TEST_SUITE periodic
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessor
@ -123,7 +123,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
        PATH ${CMAKE_CURRENT_LIST_DIR}/missing_dependency_tests.py
        EXCLUDE_TEST_RUN_TARGET_FROM_IDE
        TEST_SERIAL 
-        TIMEOUT 3600
+        TIMEOUT 1500
        TEST_SUITE periodic
        RUNTIME_DEPENDENCIES
            AZ::AssetProcessorBatch
--- a/Gems/PhysX/Code/CMakeLists.txt
+++ b/Gems/PhysX/Code/CMakeLists.txt
@ -168,7 +168,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
    ly_add_googlebenchmark(
        NAME Gem::PhysX.Benchmarks
        TARGET Gem::PhysX.Tests
-        TIMEOUT 2400 #40mins
+        TIMEOUT 1500 #25mins
    )

    list(APPEND testTargets PhysX.Tests)
--- a/cmake/LYTestWrappers.cmake
+++ b/cmake/LYTestWrappers.cmake
@ -24,8 +24,8 @@ ly_set(LY_PYTEST_EXECUTABLE ${LY_PYTHON_CMD} -B -m pytest -v --tb=short --show-c
 ly_set(LY_TEST_GLOBAL_KNOWN_SUITE_NAMES "smoke" "main" "periodic" "benchmark" "sandbox")
 ly_set(LY_TEST_GLOBAL_KNOWN_REQUIREMENTS "gpu")

-# Set default to 20 minutes
-ly_set(LY_TEST_DEFAULT_TIMEOUT 1200)
+# Set default test aborts to 25 minutes, avoids hitting the CI pipeline inactivity timeout usually set to 30 minutes
+ly_set(LY_TEST_DEFAULT_TIMEOUT 1500)

 # Add the CMake Test targets for each suite if testing is supported
 if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
@ -115,6 +115,8 @@ function(ly_add_test)
    # Set default test module timeout
    if(NOT ly_add_test_TIMEOUT)
        set(ly_add_test_TIMEOUT ${LY_TEST_DEFAULT_TIMEOUT})
+    elseif(ly_add_test_TIMEOUT GREATER LY_TEST_DEFAULT_TIMEOUT)
+        message(WARNING "TIMEOUT for test ${ly_add_test_NAME} set at ${ly_add_test_TIMEOUT} seconds which is longer than the default of ${LY_TEST_DEFAULT_TIMEOUT}. Allowing a single module to run exceedingly long creates problems in a CI pipeline.")
    endif()

    if(NOT ly_add_test_TEST_COMMAND)