Initial commit

ctest_scripts/CMakeLists.txt

@@ -0,0 +1,80 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+# Currently a sanity test is being registered here to validate that the ly_add_pytest function works
+# don't change the names of the tests, they are used in a self test.
+
+if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
+    return()
+endif()
+
+################################################################################
+# Asset Processing Target
+# i.e. Tests depend on AutomatedTesting.Assets
+################################################################################
+
+if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
+    foreach(project ${LY_PROJECTS})
+        add_custom_target(${project}.Assets
+            COMMENT "Processing ${project} assets..."
+            COMMAND $<TARGET_FILE:AZ::AssetProcessorBatch> --zeroAnalysisMode --gamefolder=${project} --platforms=${LY_ASSET_DEPLOY_ASSET_TYPE}
+        )
+        set_target_properties(${project}.Assets
+            PROPERTIES 
+                EXCLUDE_FROM_ALL TRUE
+                FOLDER ${project}
+        )
+    endforeach()
+endif()
+
+################################################################################
+# Tests
+################################################################################
+
+foreach(suite_name ${LY_TEST_GLOBAL_KNOWN_SUITE_NAMES})
+    ly_add_pytest(
+        NAME pytest_sanity_${suite_name}_no_gpu
+        PATH ${CMAKE_CURRENT_LIST_DIR}/sanity_test.py
+        TEST_SUITE ${suite_name}
+    )
+
+    ly_add_pytest(
+        NAME pytest_sanity_${suite_name}_requires_gpu
+        PATH ${CMAKE_CURRENT_LIST_DIR}/sanity_test.py
+        TEST_SUITE ${suite_name}
+        TEST_REQUIRES gpu
+    )
+endforeach()
+
+
+# EPB Sanity test is being registered here to validate that the ly_add_editor_python_test function works.
+if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedTesting IN_LIST LY_PROJECTS)
+    ly_add_editor_python_test(
+        NAME epb_sanity_smoke_no_gpu
+        TEST_PROJECT AutomatedTesting
+        PATH ${CMAKE_CURRENT_LIST_DIR}/epb_sanity_test.py
+        TEST_SUITE smoke
+        TEST_SERIAL TRUE
+        RUNTIME_DEPENDENCIES
+            AutomatedTesting.Assets
+    )
+endif()
+
+# add a custom test which makes sure that the test filtering works!
+
+ly_add_test(
+    NAME cli_test_driver
+    EXCLUDE_TEST_RUN_TARGET_FROM_IDE
+    TEST_COMMAND ${LY_PYTHON_CMD} ${CMAKE_CURRENT_LIST_DIR}/ctest_driver_test.py
+        -x ${CMAKE_CTEST_COMMAND} 
+        --build-path ${CMAKE_BINARY_DIR}
+)
+

ctest_scripts/ctest_driver.py

@@ -0,0 +1,210 @@
+"""
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+A wrapper to simplify invoking CTest with common parameters and sub-filter to specific suites
+"""
+import argparse
+import multiprocessing
+import os
+import subprocess
+import sys
+import shutil
+
+SUITES_AND_DESCRIPTIONS = {
+    "smoke": "Quick across-the-board set of tests designed to check if something is fundamentally broken",
+    "main": "The default set of tests, covers most of all testing.",
+    "periodic": "Tests which can take a long time and should be done periodially instead of every commit - these should not block code submission",
+    "benchmark": "Benchmarks - instead of pass/fail, these collect data for comparison against historic data",
+    "sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty"
+}
+
+BUILD_CONFIGURATIONS = [
+    "profile",
+    "debug",
+    "release",
+]
+
+def _regex_matching_any(words):
+    """
+    :param words: iterable of strings to match
+    :return: a regex with groups to match each string
+    """
+    return "^(" + "|".join(words) + ")$"
+
+
+def run_single_test_suite(suite, ctest_path, cmake_build_path, build_config, disable_gpu, only_gpu, generate_xml, extra_args):
+    """
+    Starts CTest to filter down to a specific suite
+    :param suite: subset of tests to run, see SUITES_AND_DESCRIPTIONS
+    :param ctest_path: path to ctest.exe
+    :param cmake_build_path: path to build output
+    :param build_config: cmake build variant to select
+    :param disable_gpu: run only non-gpu tests
+    :param only_gpu: run only gpu-required tests
+    :param generate_xml: enable to produce the CTest xml file
+    :param extrargs: forward args to ctest
+    :return: CTest exit code
+    """
+    ctest_command = [
+        ctest_path,
+        "--build-config", build_config,
+        "--output-on-failure",
+        "--parallel", str(multiprocessing.cpu_count()),  # leave serial vs parallel scheduling to CTest via set_tests_properties()
+        "--no-tests=error",
+    ]
+
+    label_excludes = []
+    label_includes = []
+
+    # ctest can't actually do "AND" queries in label include and name-include, if any
+    # match, it will accept them.  In addition, the regex language it uses does 
+    # not include positive lookahead to use workarounds...
+    # So if someone is asking for  "main" AND "requires_gpu"
+    # the only way to do this is to exclude all OTHER suites
+    # to solve this problem generally, we will always exclude all other suites than
+    # the one being tested.
+
+    for label_name in SUITES_AND_DESCRIPTIONS.keys():
+        if label_name != suite:
+            label_excludes.append(f"SUITE_{label_name}")
+    
+    # only one of these can be true, or neither.  If neither, we apply no REQUIRES_* filter.
+    if only_gpu:
+        label_includes.append("REQUIRES_gpu")
+    elif disable_gpu:
+        label_excludes.append("REQUIRES_gpu")
+
+    union_regex = _regex_matching_any(label_includes) if label_includes else None
+    difference_regex = _regex_matching_any(label_excludes) if label_excludes else None
+
+    if union_regex:
+        ctest_command.append("--label-regex")
+        ctest_command.append(union_regex)
+    if difference_regex:
+        ctest_command.append("--label-exclude")
+        ctest_command.append(difference_regex)
+    if generate_xml:
+        ctest_command.append('-T')
+        ctest_command.append('Test')
+
+    for extra_arg in extra_args:
+        ctest_command.append(extra_arg)
+
+    ctest_command_string = ' '.join(ctest_command) # ONLY used for display
+    print("Executing CTest with command:\n"
+          f"  {ctest_command_string}\n"
+          "in working directory:\n"
+          f"  {cmake_build_path}\n")
+
+    result = subprocess.run(ctest_command, shell=False, cwd=cmake_build_path, stdout=sys.stdout, stderr=sys.stderr)
+    
+    return result.returncode
+
+
+def main():
+    # establish defaults
+    ctest_version = "3.17.0"
+    if sys.platform == "win32":
+        ctest_build = "Windows"
+        ctest_relpath = "bin"
+        ctest_exe = "ctest.exe"
+    elif sys.platform.startswith("linux"):
+        ctest_build = "Linux"
+        ctest_relpath = "bin"
+        ctest_exe = "ctest"
+    elif sys.platform.startswith('darwin'):
+        ctest_build = "Mac"
+        ctest_relpath = "CMake.app/Contents/bin"
+        ctest_exe = "ctest"
+    else:
+        raise NotImplementedError(f"CTest is not currently configured for platform '{sys.platform}'")
+
+    current_script_path = os.path.dirname(__file__)
+    dev_default = os.path.dirname(current_script_path)
+    thirdparty_default = os.path.join(os.path.dirname(dev_default), "3rdParty")
+
+    # if a specific known location contains cmake, we'll use it
+    ctest_default = os.path.join(thirdparty_default, "CMake", ctest_version, ctest_build, ctest_relpath, ctest_exe)
+    
+    # parse args, with defaults
+    parser = argparse.ArgumentParser(
+        description="CTest CLI driver: simplifies providing common arguments to CTest",
+        # extra wide help messages to avoid newlines appearing in path defaults, which break copy-paste of paths
+        formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=4096),
+        epilog="(Unrecognised parameters will be sent to ctest directly)"
+    )
+
+    parser.add_argument('-x', '--ctest-executable', 
+                        help="Override path to the CTest executable (will use PATH env otherwise)")
+    parser.add_argument('-B', '--build-path', # -B to match cmake's syntax for same thing.
+                        help="Path to a CMake build folder (generated by running cmake).",
+                        required=True)
+    parser.add_argument('--config', choices=BUILD_CONFIGURATIONS, default="debug", # --config to match cmake
+                        help="CMake variant build configuration to target (debug/profile/release)")
+    parser.add_argument('-s', '--suite', choices=SUITES_AND_DESCRIPTIONS.keys(), 
+                        default="main",
+                        help="Which subset of tests to execute")
+    parser.add_argument('--generate-xml', action='store_true', 
+                        help='Enable this option to produce the CTest xml file.')
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--no-gpu', action='store_true',
+                        help="Disable tests that require a GPU")
+    group.add_argument('--only-gpu', action='store_true',
+                        help="Run only tests that require a GPU")
+
+    args, unknown_args = parser.parse_known_args()
+
+    # handle the CTEST executable.
+    # we always obey command line, and its an error if the command line has
+    # a bad executable
+    # if no command line is specified, it will fallback to a known good location
+    # and then finally, use the PATH.
+    if args.ctest_executable and not os.path.exists(args.ctest_executable):
+        print(f"Error: Invalid ctest executable specified - not found: {args.ctest_executable}")
+        return 1
+    if not args.ctest_executable:
+        # try the default
+        if os.path.exists(ctest_default):
+            print(f"Using default CTest executable: {ctest_default}")
+            args.ctest_executable = ctest_default
+        else: # try the PATH env var:
+            found_ctest = shutil.which(ctest_exe)
+            if found_ctest:
+                print(f"Using CTest executable from PATH: {found_ctest}")
+                args.ctest_executable = found_ctest
+            else:
+                print(f"Could not find CTest Executable ('{ctest_exe}')on PATH or in a pre-set location.")
+                return 1
+    
+    # handle the build path.  You must specify a build path, and it must contain CTestTestfile.cmake
+    if not os.path.exists(args.build_path):
+        print(f"Error: specified folder does not exist: {args.build_path}")
+        return 1
+    ctest_testfile = os.path.join(args.build_path, "CTestTestfile.cmake")
+    if not os.path.exists(ctest_testfile):
+        print(f"Error: '{ctest_testfile}' missing, run CMake configure+generate on the folder first.")
+        return 1
+
+    print(f"Starting '{args.suite}' suite: {SUITES_AND_DESCRIPTIONS[args.suite]}")
+
+    # execute
+    return run_single_test_suite(
+        suite=args.suite,
+        ctest_path=args.ctest_executable,
+        cmake_build_path=args.build_path,
+        build_config=args.config,
+        disable_gpu=args.no_gpu,
+        only_gpu=args.only_gpu,
+        generate_xml=args.generate_xml,
+        extra_args=unknown_args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ctest_scripts/ctest_driver_test.py

@@ -0,0 +1,87 @@
+"""
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+Self-tests for ctest_driver.py.
+
+Expects you to have generated a folder using cmake and to pass in that folder name
+as the only param
+"""
+import os
+import subprocess
+import sys
+import argparse
+from ctest_driver import SUITES_AND_DESCRIPTIONS
+
+def main(build_path, ctest_executable):
+    script_folder = os.path.dirname(__file__)
+    # -N prevents tests from running, just lists them:
+    base_args = [sys.executable, os.path.join(script_folder,'ctest_driver.py'), "--build-path", build_path, '-N']
+    if ctest_executable:
+        base_args.append("--ctest-executable")
+        base_args.append(ctest_executable)
+
+    base_args.append("-R") # limit it to only the sanity tests, faster
+    base_args.append("pytest_sanity_.*")
+
+    for suite in SUITES_AND_DESCRIPTIONS:
+        for gpu_option in [None, True, False]:
+            args_to_send = base_args.copy()
+            args_to_send.append("--suite")
+            args_to_send.append(suite)
+            print(f"----- Test case: [suite = {suite}, gpu = {gpu_option}] ----")
+            if gpu_option is not None:
+                if gpu_option:
+                    args_to_send.append("--only-gpu")
+                else:
+                    args_to_send.append("--no-gpu")
+            result = subprocess.check_output(args_to_send, shell=False, cwd=build_path, stderr=sys.stderr)
+            output = result.decode('utf-8')
+
+            # ensure that the appropriate suites are filtered in and out
+            for suite_name in SUITES_AND_DESCRIPTIONS.keys():
+                if (f"pytest_sanity_{suite_name}_" in output and suite != suite_name):
+                    print(output)
+                    print("Test failed - executed a test not in the currently selected suite.")
+                    return -1
+
+            gpu_test_present = "_requires_gpu" in output
+            non_gpu_test_present = "_no_gpu" in output
+
+            if gpu_option is not None:
+                if gpu_option and non_gpu_test_present:
+                    print("Test failed - required gpu only, and a non-gpu tes was run")
+                    print(output)
+                    return -1
+
+                if not gpu_option and gpu_test_present:
+                    print("Test failed - executed a gpu test when forbidden")
+                    print(output)
+                    return -1
+            else:
+                if not (gpu_test_present and non_gpu_test_present):
+                    print("Test failed - expected both kinds of tests (gpu and non gpu)")
+                    print(output)
+                    return -1
+
+    print("All passed.")
+    return 0
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="CTest CLI driver self-tests.")
+    parser.add_argument('-x', '--ctest-executable', 
+                        help="Override path to the CTest executable (will use PATH env otherwise)")
+    parser.add_argument('-b', '--build-path', 
+                        required=True,
+                        help="Path to a CMake build folder (generated by running cmake)")
+    args = parser.parse_args()
+    sys.exit(main(args.build_path, args.ctest_executable))
+
+

ctest_scripts/ctest_entrypoint.cmd

@@ -0,0 +1,22 @@
+@ECHO OFF
+REM
+REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+REM its licensors.
+REM
+REM For complete copyright and license terms please see the LICENSE at the root of this
+REM distribution (the "License"). All use of this software is governed by the License,
+REM or, if provided, by the license below or the license accompanying this file. Do not
+REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+REM
+REM Continuous Integration CLI entrypoint script to start CTest, triggering post-build tests
+REM
+SETLOCAL
+
+SET DEV_DIR=%~dp0\..
+SET PYTHON=%DEV_DIR%\python\python.cmd
+SET CTEST_SCRIPT=%~dp0\ctest_driver.py
+
+REM pass all args to python-based CTest script
+call %PYTHON% %CTEST_SCRIPT% %*
+exit /b %ERRORLEVEL%

ctest_scripts/ctest_entrypoint.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+# Continuous Integration CLI entrypoint script to start CTest, triggering post-build tests
+#
+
+CURRENT_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE:-0}" )" >/dev/null 2>&1 && pwd )"
+DEV_DIR="$( dirname "$CURRENT_SCRIPT_DIR" )"
+PYTHON=$DEV_DIR/python/python.sh
+CTEST_SCRIPT=$CURRENT_SCRIPT_DIR/ctest_driver.py
+
+# pass all args to python-based CTest script
+echo "Invoking: $PYTHON $CTEST_SCRIPT $*"
+$PYTHON $CTEST_SCRIPT $*

ctest_scripts/epb_sanity_test.py

@@ -0,0 +1,21 @@
+"""
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+"""
+
+# Sanity test for EditorPythonBindings CTest wrapper
+
+import azlmbr.framework as framework
+
+print("EditorPythonBindings CTest Sanity Test")
+
+# A test should have logic to determine success (zero) or failure (non-zero) and 
+# return it to the caller. In this sanity test, always return success.
+return_code = 0
+framework.Terminate(return_code)

ctest_scripts/sanity_test.py

@@ -0,0 +1,45 @@
+"""
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+
+Sanity tests to verify whether suite filtering is hooked up
+"""
+import pytest
+
+def test_Sanity_Untagged_Pass(): # should happen in main pass also
+    pass
+
+@pytest.mark.SUITE_smoke
+def test_Sanity_Smoke_Pass():
+    pass
+
+@pytest.mark.SUITE_main
+def test_Sanity_Main_Pass(): # should happen in main pass
+    pass
+
+@pytest.mark.SUITE_periodic
+def test_Sanity_Periodic_Pass():
+    pass
+
+@pytest.mark.SUITE_benchmark
+def test_Sanity_Benchmark_Pass():
+    pass
+
+@pytest.mark.SUITE_sandbox
+def test_Sanity_Sandbox_Pass():
+    pass
+
+@pytest.mark.REQUIRES_gpu
+def test_Sanity_RequireGpu_Pass():
+    pass
+
+@pytest.mark.skip
+def test_Insanity():
+    raise RuntimeError