diff --git a/AutomatedTesting/Gem/CMakeLists.txt b/AutomatedTesting/Gem/CMakeLists.txt index 5d881f3f95..990fa72222 100644 --- a/AutomatedTesting/Gem/CMakeLists.txt +++ b/AutomatedTesting/Gem/CMakeLists.txt @@ -11,3 +11,4 @@ add_subdirectory(Code) add_subdirectory(PythonTests) +add_subdirectory(PythonCoverage) diff --git a/AutomatedTesting/Gem/Code/enabled_gems.cmake b/AutomatedTesting/Gem/Code/enabled_gems.cmake index d99d17b55e..4f04e61f2a 100644 --- a/AutomatedTesting/Gem/Code/enabled_gems.cmake +++ b/AutomatedTesting/Gem/Code/enabled_gems.cmake @@ -55,4 +55,5 @@ set(ENABLED_GEMS AWSCore AWSClientAuth AWSMetrics + PythonCoverage ) diff --git a/AutomatedTesting/Gem/PythonCoverage/CMakeLists.txt b/AutomatedTesting/Gem/PythonCoverage/CMakeLists.txt new file mode 100644 index 0000000000..20a680bce9 --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/CMakeLists.txt @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +add_subdirectory(Code) diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/CMakeLists.txt b/AutomatedTesting/Gem/PythonCoverage/Code/CMakeLists.txt new file mode 100644 index 0000000000..6d0242d02e --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/CMakeLists.txt @@ -0,0 +1,62 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +ly_get_list_relative_pal_filename(pal_dir ${CMAKE_CURRENT_LIST_DIR}/Platform/${PAL_PLATFORM_NAME} ${o3de_gem_restricted_path} ${o3de_gem_path} ${o3de_gem_name}) +include(${pal_dir}/PAL_${PAL_PLATFORM_NAME_LOWERCASE}.cmake) + +if(PAL_TRAIT_PYTHONCOVERAGE_SUPPORTED) + if(PAL_TRAIT_BUILD_HOST_TOOLS) + ly_add_target( + NAME PythonCoverage.Editor.Static STATIC + NAMESPACE Gem + FILES_CMAKE + pythoncoverage_editor_files.cmake + INCLUDE_DIRECTORIES + PRIVATE + Source + PUBLIC + Include + COMPILE_DEFINITIONS + PUBLIC + PYTHON_COVERAGE_EDITOR + PRIVATE + ${LY_TEST_IMPACT_CONFIG_FILE_PATH_DEFINITION} + BUILD_DEPENDENCIES + PUBLIC + AZ::AzToolsFramework + RUNTIME_DEPENDENCIES + Gem::EditorPythonBindings.Editor + ) + + ly_add_target( + NAME PythonCoverage.Editor GEM_MODULE + NAMESPACE Gem + AUTOMOC + OUTPUT_NAME Gem.PythonCoverage.Editor + FILES_CMAKE + pythoncoverage_editor_shared_files.cmake + COMPILE_DEFINITIONS + PRIVATE + PYTHON_COVERAGE_EDITOR + INCLUDE_DIRECTORIES + PRIVATE + Source + PUBLIC + Include + BUILD_DEPENDENCIES + PUBLIC + Gem::PythonCoverage.Editor.Static + ) + + ly_create_alias(NAME PythonCoverage.Tools NAMESPACE Gem TARGETS Gem::PythonCoverage.Editor) + ly_create_alias(NAME PythonCoverage.Builders NAMESPACE Gem TARGETS Gem::PythonCoverage.Editor) + endif() +endif() diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Android/PAL_android.cmake b/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Android/PAL_android.cmake new file mode 100644 index 0000000000..7385380639 --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Android/PAL_android.cmake @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(PAL_TRAIT_PYTHONCOVERAGE_SUPPORTED FALSE) diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Linux/PAL_linux.cmake b/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Linux/PAL_linux.cmake new file mode 100644 index 0000000000..7385380639 --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Linux/PAL_linux.cmake @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(PAL_TRAIT_PYTHONCOVERAGE_SUPPORTED FALSE) diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Mac/PAL_mac.cmake b/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Mac/PAL_mac.cmake new file mode 100644 index 0000000000..7385380639 --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Mac/PAL_mac.cmake @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(PAL_TRAIT_PYTHONCOVERAGE_SUPPORTED FALSE) diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Windows/PAL_windows.cmake b/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Windows/PAL_windows.cmake new file mode 100644 index 0000000000..99b5ab4780 --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/Platform/Windows/PAL_windows.cmake @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(PAL_TRAIT_PYTHONCOVERAGE_SUPPORTED TRUE) diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/Platform/iOS/PAL_ios.cmake b/AutomatedTesting/Gem/PythonCoverage/Code/Platform/iOS/PAL_ios.cmake new file mode 100644 index 0000000000..7385380639 --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/Platform/iOS/PAL_ios.cmake @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(PAL_TRAIT_PYTHONCOVERAGE_SUPPORTED FALSE) diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorModule.cpp b/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorModule.cpp new file mode 100644 index 0000000000..fd3dce4cde --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorModule.cpp @@ -0,0 +1,38 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "PythonCoverageEditorModule.h" +#include "PythonCoverageEditorSystemComponent.h" + +namespace PythonCoverage +{ + AZ_CLASS_ALLOCATOR_IMPL(PythonCoverageEditorModule, AZ::SystemAllocator, 0) + + PythonCoverageEditorModule::PythonCoverageEditorModule() + { + m_descriptors.insert( + m_descriptors.end(), + { + PythonCoverageEditorSystemComponent::CreateDescriptor() + }); + } + + PythonCoverageEditorModule::~PythonCoverageEditorModule() = default; + + AZ::ComponentTypeList PythonCoverageEditorModule::GetRequiredSystemComponents() const + { + // add required SystemComponents to the SystemEntity + return AZ::ComponentTypeList{ azrtti_typeid() }; + } +} // namespace PythonCoverage + +AZ_DECLARE_MODULE_CLASS(Gem_PythonCoverageEditor, PythonCoverage::PythonCoverageEditorModule) diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorModule.h b/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorModule.h new file mode 100644 index 0000000000..2295d20f3e --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorModule.h @@ -0,0 +1,33 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace PythonCoverage +{ + class PythonCoverageEditorModule + : public AZ::Module + { + public: + AZ_CLASS_ALLOCATOR_DECL + AZ_RTTI(PythonCoverageEditorModule, "{32C0FFEA-09A7-460F-9257-5BDEF74FCD5B}", AZ::Module); + + PythonCoverageEditorModule(); + ~PythonCoverageEditorModule(); + + // PythonCoverageModule ... + AZ::ComponentTypeList GetRequiredSystemComponents() const override; + }; +} // namespace WhiteBox diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorSystemComponent.cpp b/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorSystemComponent.cpp new file mode 100644 index 0000000000..dec9581296 --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorSystemComponent.cpp @@ -0,0 +1,246 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include +#include +#include +#include + +#include + +namespace PythonCoverage +{ + static constexpr char* const LogCallSite = "PythonCoverageEditorSystemComponent"; + + void PythonCoverageEditorSystemComponent::Reflect(AZ::ReflectContext* context) + { + if (auto serializeContext = azrtti_cast(context)) + { + serializeContext->Class()->Version(1); + } + } + + void PythonCoverageEditorSystemComponent::Activate() + { + AzToolsFramework::EditorPythonScriptNotificationsBus::Handler::BusConnect(); + AZ::EntitySystemBus::Handler::BusConnect(); + + // If no output directory discovered, coverage gathering will be disabled + if (ParseCoverageOutputDirectory() == CoverageState::Disabled) + { + return; + } + + EnumerateAllModuleComponents(); + } + + void PythonCoverageEditorSystemComponent::Deactivate() + { + AZ::EntitySystemBus::Handler::BusDisconnect(); + AzToolsFramework::EditorPythonScriptNotificationsBus::Handler::BusDisconnect(); + } + + void PythonCoverageEditorSystemComponent::OnEntityActivated(const AZ::EntityId& entityId) + { + if (m_coverageState == CoverageState::Disabled) + { + return; + } + + EnumerateComponentsForEntity(entityId); + + // There is currently no way to receive a graceful exit signal in order to properly handle the coverage end of life so + // instead we have to serialize the data on-the-fly with blocking disk writes on the main thread... if this adversely + // affects performance in a measurable way then this could potentially be put on a worker thread, although it remains to + // be seen whether the asynchronous nature of such a thread results in queued up coverage being lost due to the hard exit + if (m_coverageState == CoverageState::Gathering) + { + WriteCoverageFile(); + } + } + + PythonCoverageEditorSystemComponent::CoverageState PythonCoverageEditorSystemComponent::ParseCoverageOutputDirectory() + { + m_coverageState = CoverageState::Disabled; + const AZStd::string configFilePath = LY_TEST_IMPACT_DEFAULT_CONFIG_FILE; + + if (configFilePath.empty()) + { + AZ_Warning(LogCallSite, false, "No test impact analysis framework config file specified."); + return m_coverageState; + } + + const auto fileSize = AZ::IO::SystemFile::Length(configFilePath.c_str()); + if(!fileSize) + { + AZ_Error(LogCallSite, false, "Test impact analysis framework config file '%s' does not exist", configFilePath.c_str()); + return m_coverageState; + } + + AZStd::vector buffer(fileSize + 1); + buffer[fileSize] = '\0'; + if (!AZ::IO::SystemFile::Read(configFilePath.c_str(), buffer.data())) + { + AZ_Error(LogCallSite, false, "Could not read contents of test impact analysis framework config file '%s'", configFilePath.c_str()); + return m_coverageState; + } + + const AZStd::string configurationData = AZStd::string(buffer.begin(), buffer.end()); + rapidjson::Document configurationFile; + if (configurationFile.Parse(configurationData.c_str()).HasParseError()) + { + AZ_Error(LogCallSite, false, "Could not parse test impact analysis framework config file data, JSON has errors"); + return m_coverageState; + } + + const auto& tempConfig = configurationFile["workspace"]["temp"]; + + // Temp directory root path is absolute + const AZ::IO::Path tempWorkspaceRootDir = tempConfig["root"].GetString(); + + // Artifact directory is relative to temp directory root + const AZ::IO::Path artifactRelativeDir = tempConfig["relative_paths"]["artifact_dir"].GetString(); + m_coverageDir = tempWorkspaceRootDir / artifactRelativeDir; + + // Everything is good to go, await the first python test case + m_coverageState = CoverageState::Idle; + return m_coverageState; + } + + void PythonCoverageEditorSystemComponent::WriteCoverageFile() + { + AZStd::string contents; + + // Compile the coverage for all test cases in this script + for (const auto& [testCase, entityComponents] : m_entityComponentMap) + { + const auto coveringModules = GetParentComponentModulesForAllActivatedEntities(entityComponents); + if (coveringModules.empty()) + { + return; + } + + contents = testCase + "\n"; + for (const auto& coveringModule : coveringModules) + { + contents += AZStd::string::format(" %s\n", coveringModule.c_str()); + } + } + + AZ::IO::SystemFile file; + const AZStd::vector bytes(contents.begin(), contents.end()); + if (!file.Open( + m_coverageFile.c_str(), + AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY)) + { + AZ_Error(LogCallSite, false, "Couldn't open file '%s' for writing", m_coverageFile.c_str()); + return; + } + + if (!file.Write(bytes.data(), bytes.size())) + { + AZ_Error(LogCallSite, false, "Couldn't write contents for file '%s'", m_coverageFile.c_str()); + return; + } + } + + void PythonCoverageEditorSystemComponent::EnumerateAllModuleComponents() + { + AZ::ModuleManagerRequestBus::Broadcast( + &AZ::ModuleManagerRequestBus::Events::EnumerateModules, + [this](const AZ::ModuleData& moduleData) + { + // We can only enumerate shared libs, static libs are invisible to us + if (moduleData.GetDynamicModuleHandle()) + { + for (const auto* moduleComponentDescriptor : moduleData.GetModule()->GetComponentDescriptors()) + { + m_moduleComponents[moduleComponentDescriptor->GetUuid()] = moduleData.GetDebugName(); + } + } + + return true; + }); + } + + void PythonCoverageEditorSystemComponent::EnumerateComponentsForEntity(const AZ::EntityId& entityId) + { + AZ::Entity* entity = nullptr; + AZ::ComponentApplicationBus::BroadcastResult(entity, &AZ::ComponentApplicationBus::Events::FindEntity, AZ::EntityId(entityId)); + + if (entity) + { + auto& entityComponents = m_entityComponentMap[m_testCase]; + for (const auto& entityComponent : entity->GetComponents()) + { + const auto componentTypeId = entityComponent->GetUnderlyingComponentType(); + AZ::ComponentDescriptor* componentDescriptor = nullptr; + AZ::ComponentDescriptorBus::EventResult( + componentDescriptor, componentTypeId, &AZ::ComponentDescriptorBus::Events::GetDescriptor); + entityComponents[componentTypeId] = componentDescriptor; + } + } + } + + AZStd::unordered_set PythonCoverageEditorSystemComponent::GetParentComponentModulesForAllActivatedEntities( + const AZStd::unordered_map& entityComponents) const + { + AZStd::unordered_set coveringModuleOutputNames; + for (const auto& [uuid, componentDescriptor] : entityComponents) + { + if (const auto moduleComponent = m_moduleComponents.find(uuid); moduleComponent != m_moduleComponents.end()) + { + coveringModuleOutputNames.insert(moduleComponent->second); + } + } + + return coveringModuleOutputNames; + } + + void PythonCoverageEditorSystemComponent::OnStartExecuteByFilenameAsTest(AZStd::string_view filename, AZStd::string_view testCase, [[maybe_unused]] const AZStd::vector& args) + { + if (m_coverageState == CoverageState::Disabled) + { + return; + } + + if (m_coverageState == CoverageState::Gathering) + { + // Dump any existing coverage data to disk + WriteCoverageFile(); + m_coverageState = CoverageState::Idle; + } + + if (testCase.empty()) + { + // We need to be able to pinpoint the coverage data to the specific test case names otherwise we will not be able + // to specify which specific tests should be run in the future (filename does not necessarily equate to test case name) + AZ_Error(LogCallSite, false, "No test case specified, coverage data gathering will be disabled for this test"); + return; + } + + const AZStd::string scriptName = AZ::IO::Path(filename).Stem().Native(); + const auto coverageFile = m_coverageDir / AZStd::string::format("%s.pycoverage", scriptName.c_str()); + + // If this is a different python script we clear the existing entity components and start afresh + if (m_coverageFile != coverageFile) + { + m_entityComponentMap.clear(); + m_coverageFile = coverageFile; + } + + m_testCase = testCase; + m_coverageState = CoverageState::Gathering; + } +} // namespace PythonCoverage diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorSystemComponent.h b/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorSystemComponent.h new file mode 100644 index 0000000000..c2d6950ade --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/Source/PythonCoverageEditorSystemComponent.h @@ -0,0 +1,88 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace AZ +{ + class ComponentDescriptor; +} + +namespace PythonCoverage +{ + //! System component for PythonCoverage editor. + class PythonCoverageEditorSystemComponent + : public AZ::Component + , private AZ::EntitySystemBus::Handler + , private AzToolsFramework::EditorPythonScriptNotificationsBus::Handler + { + public: + AZ_COMPONENT(PythonCoverageEditorSystemComponent, "{33370075-3aea-49c4-823d-476f8ac95b6f}"); + static void Reflect(AZ::ReflectContext* context); + + PythonCoverageEditorSystemComponent() = default; + + private: + //! The coverage state for Python tests. + enum class CoverageState : AZ::u8 + { + Disabled, //!< Python coverage is disabled. + Idle, //!< Python coverage is enabled but not actively gathering coverage data. + Gathering //!< Python coverage is enabled and actively gathering coverage data. + }; + + // AZ::Component overrides... + void Activate() override; + void Deactivate() override; + + // AZ::EntitySystemBus overrides... + void OnEntityActivated(const AZ::EntityId& entityId) override; + + // AZ::EditorPythonScriptNotificationsBus ... + void OnStartExecuteByFilenameAsTest(AZStd::string_view filename, AZStd::string_view testCase, const AZStd::vector& args) override; + + //! Enumerates all of the loaded shared library modules and the component descriptors that belong to them. + void EnumerateAllModuleComponents(); + + //! Enumerates all of the component descriptors for the specified entity. + void EnumerateComponentsForEntity(const AZ::EntityId& entityId); + + //! Attempts to parse the test impact analysis framework configuration file. + //! If either the test impact analysis framework is disabled or the configuration file cannot be parsed, python coverage is disabled. + //! @returns The coverage state after the parsing attempt. + CoverageState ParseCoverageOutputDirectory(); + + //! Returns all of the shared library modules that parent the component descriptors of the specified set of activated entities. + //! @note Entity component descriptors are still retrieved even if the entity in question has since been deactivated. + //! @param entityComponents The set of activated entities and their component descriptors to get the parent modules for. + AZStd::unordered_set GetParentComponentModulesForAllActivatedEntities( + const AZStd::unordered_map& entityComponents) const; + + //! Writes the current coverage data snapshot to disk. + void WriteCoverageFile(); + + CoverageState m_coverageState = CoverageState::Disabled; //!< Current coverage state. + AZStd::unordered_map> m_entityComponentMap; //!< Map of + //!< component IDs to component descriptors for all activated entities, organized by test cases. + AZStd::unordered_map m_moduleComponents; //!< Map of component IDs to module names for all modules. + AZ::IO::Path m_coverageDir; //!< Directory to write coverage data to. + AZ::IO::Path m_coverageFile; //!< Full file path to write coverage data to. + AZStd::string m_testCase; //!< Name of current test case that coverage data is being gathered for. + }; +} // namespace PythonCoverage diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/pythoncoverage_editor_files.cmake b/AutomatedTesting/Gem/PythonCoverage/Code/pythoncoverage_editor_files.cmake new file mode 100644 index 0000000000..ffdc071eb8 --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/pythoncoverage_editor_files.cmake @@ -0,0 +1,15 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + Source/PythonCoverageEditorSystemComponent.cpp + Source/PythonCoverageEditorSystemComponent.h +) diff --git a/AutomatedTesting/Gem/PythonCoverage/Code/pythoncoverage_editor_shared_files.cmake b/AutomatedTesting/Gem/PythonCoverage/Code/pythoncoverage_editor_shared_files.cmake new file mode 100644 index 0000000000..d1bb21b43e --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/Code/pythoncoverage_editor_shared_files.cmake @@ -0,0 +1,15 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + Source/PythonCoverageEditorModule.cpp + Source/PythonCoverageEditorModule.h +) diff --git a/AutomatedTesting/Gem/PythonCoverage/gem.json b/AutomatedTesting/Gem/PythonCoverage/gem.json new file mode 100644 index 0000000000..a31ee8adcb --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/gem.json @@ -0,0 +1,13 @@ +{ + "gem_name": "PythonCoverage", + "display_name": "PythonCoverage", + "summary": "A tool for generating gem coverage for Python tests.", + "canonical_tags": [ + "Gem" + ], + "user_tags": [ + "PythonCoverage" + ], + "icon_path": "preview.png", + "restricted_name": "gems" +} \ No newline at end of file diff --git a/AutomatedTesting/Gem/PythonCoverage/preview.png b/AutomatedTesting/Gem/PythonCoverage/preview.png new file mode 100644 index 0000000000..2f1ed47754 --- /dev/null +++ b/AutomatedTesting/Gem/PythonCoverage/preview.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d6204c6730e5675791765ca194e9b1cbec282208e280507de830afc2805e5fa +size 41127 diff --git a/AutomatedTesting/Gem/PythonTests/EditorPythonTestTools/editor_python_test_tools/hydra_test_utils.py b/AutomatedTesting/Gem/PythonTests/EditorPythonTestTools/editor_python_test_tools/hydra_test_utils.py index 9f498bbf50..4ea7362083 100644 --- a/AutomatedTesting/Gem/PythonTests/EditorPythonTestTools/editor_python_test_tools/hydra_test_utils.py +++ b/AutomatedTesting/Gem/PythonTests/EditorPythonTestTools/editor_python_test_tools/hydra_test_utils.py @@ -54,7 +54,7 @@ def launch_and_validate_results(request, test_directory, editor, editor_script, logger.debug("Running automated test: {}".format(editor_script)) editor.args.extend(["--skipWelcomeScreenDialog", "--regset=/Amazon/Settings/EnableSourceControl=false", "--regset=/Amazon/Preferences/EnablePrefabSystem=false", run_python, test_case, - "--runpythonargs", " ".join(cfg_args)]) + f"--pythontestcase={request.node.originalname}", "--runpythonargs", " ".join(cfg_args)]) if auto_test_mode: editor.args.extend(["--autotest_mode"]) if null_renderer: diff --git a/AutomatedTesting/Gem/PythonTests/automatedtesting_shared/base.py b/AutomatedTesting/Gem/PythonTests/automatedtesting_shared/base.py index f00227c47f..9d183ea326 100755 --- a/AutomatedTesting/Gem/PythonTests/automatedtesting_shared/base.py +++ b/AutomatedTesting/Gem/PythonTests/automatedtesting_shared/base.py @@ -94,7 +94,7 @@ class TestAutomationBase: editor_starttime = time.time() self.logger.debug("Running automated test") testcase_module_filepath = self._get_testcase_module_filepath(testcase_module) - pycmd = ["--runpythontest", testcase_module_filepath, "-BatchMode", "-autotest_mode", "-rhi=null"] + extra_cmdline_args + pycmd = ["--runpythontest", testcase_module_filepath, "-BatchMode", "-autotest_mode", "-rhi=null", f"-pythontestcase={request.node.originalname}"] + extra_cmdline_args editor.args.extend(pycmd) # args are added to the WinLauncher start command editor.start(backupFiles = False, launch_ap = False) try: diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorPythonRunnerRequestsBus.h b/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorPythonRunnerRequestsBus.h index 6a2553f90b..1b423621e1 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorPythonRunnerRequestsBus.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorPythonRunnerRequestsBus.h @@ -29,16 +29,22 @@ namespace AzToolsFramework ////////////////////////////////////////////////////////////////////////// //! executes a Python script using a string, prints the result if printResult is true and script is an expression - virtual void ExecuteByString(AZStd::string_view script, bool printResult) { AZ_UNUSED(script); AZ_UNUSED(printResult); } + virtual void ExecuteByString([[maybe_unused]] AZStd::string_view script, [[maybe_unused]] bool printResult) {} //! executes a Python script using a filename - virtual void ExecuteByFilename(AZStd::string_view filename) { AZ_UNUSED(filename); } + virtual void ExecuteByFilename([[maybe_unused]] AZStd::string_view filename) {} //! executes a Python script using a filename and args - virtual void ExecuteByFilenameWithArgs(AZStd::string_view filename, const AZStd::vector& args) { AZ_UNUSED(filename); AZ_UNUSED(args); } + virtual void ExecuteByFilenameWithArgs( + [[maybe_unused]] AZStd::string_view filename, [[maybe_unused]] const AZStd::vector& args) {} //! executes a Python script as a test - virtual void ExecuteByFilenameAsTest(AZStd::string_view filename, const AZStd::vector& args) { AZ_UNUSED(filename); AZ_UNUSED(args); } + virtual void ExecuteByFilenameAsTest( + [[maybe_unused]] AZStd::string_view filename, + [[maybe_unused]] AZStd::string_view testCase, + [[maybe_unused]] const AZStd::vector& args) + { + } }; using EditorPythonRunnerRequestBus = AZ::EBus; diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorPythonScriptNotificationsBus.h b/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorPythonScriptNotificationsBus.h new file mode 100644 index 0000000000..7586293a38 --- /dev/null +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorPythonScriptNotificationsBus.h @@ -0,0 +1,45 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ +#pragma once + +#include +#include + +namespace AzToolsFramework +{ + //! Provides a bus to notify when Python scripts are about to run. + class EditorPythonScriptNotifications + : public AZ::EBusTraits + { + public: + ////////////////////////////////////////////////////////////////////////// + // EBusTraits overrides + static const AZ::EBusHandlerPolicy HandlerPolicy = AZ::EBusHandlerPolicy::Multiple; + static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::Single; + ////////////////////////////////////////////////////////////////////////// + + //! Notifies the start of execution of a Python script using a string. + virtual void OnStartExecuteByString([[maybe_unused]] AZStd::string_view script) {} + + //! Notifies the start of execution of a Python script using a filename. + virtual void OnStartExecuteByFilename([[maybe_unused]] AZStd::string_view filename) {} + + //! Notifies the start of execution of a Python script using a filename and args. + virtual void OnStartExecuteByFilenameWithArgs( + [[maybe_unused]] AZStd::string_view filename, [[maybe_unused]] const AZStd::vector& args) {} + + //! Notifies the start of execution of a Python script as a test. + virtual void OnStartExecuteByFilenameAsTest( + [[maybe_unused]] AZStd::string_view filename, [[maybe_unused]] AZStd::string_view testCase, [[maybe_unused]] const AZStd::vector& args) {} + }; + using EditorPythonScriptNotificationsBus = AZ::EBus; +} diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/aztoolsframework_files.cmake b/Code/Framework/AzToolsFramework/AzToolsFramework/aztoolsframework_files.cmake index 9ec0deaed6..bdb1ab6f0a 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/aztoolsframework_files.cmake +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/aztoolsframework_files.cmake @@ -41,6 +41,7 @@ set(FILES API/EditorVegetationRequestsBus.h API/EditorPythonConsoleBus.h API/EditorPythonRunnerRequestsBus.h + API/EditorPythonScriptNotificationsBus.h API/EntityPropertyEditorRequestsBus.h API/EditorWindowRequestBus.h API/EntityCompositionRequestBus.h diff --git a/Code/Sandbox/Editor/CryEdit.cpp b/Code/Sandbox/Editor/CryEdit.cpp index 792f817362..ae93ef04a4 100644 --- a/Code/Sandbox/Editor/CryEdit.cpp +++ b/Code/Sandbox/Editor/CryEdit.cpp @@ -542,6 +542,7 @@ public: QString m_appRoot; QString m_logFile; QString m_pythonArgs; + QString m_pythontTestCase; QString m_execFile; QString m_execLineCmd; @@ -589,6 +590,7 @@ public: const std::vector > stringOptions = { {{"logfile", "File name of the log file to write out to.", "logfile"}, m_logFile}, {{"runpythonargs", "Command-line argument string to pass to the python script if --runpython or --runpythontest was used.", "runpythonargs"}, m_pythonArgs}, + {{"pythontestcase", "Test case name of python test script if --runpythontest was used.", "pythontestcase"}, m_pythontTestCase}, {{"exec", "cfg file to run on startup, used for systems like automation", "exec"}, m_execFile}, {{"rhi", "Command-line argument to force which rhi to use", "dummyString"}, dummyString }, {{"rhi-device-validation", "Command-line argument to configure rhi validation", "dummyString"}, dummyString }, @@ -1527,7 +1529,13 @@ void CCryEditApp::RunInitPythonScript(CEditCommandLineInfo& cmdInfo) std::transform(tokens.begin(), tokens.end(), std::back_inserter(pythonArgs), [](auto& tokenData) { return tokenData.c_str(); }); if (cmdInfo.m_bRunPythonTestScript) { - EditorPythonRunnerRequestBus::Broadcast(&EditorPythonRunnerRequestBus::Events::ExecuteByFilenameAsTest, cmdInfo.m_strFileName.toUtf8().constData(), pythonArgs); + AZStd::string pythonTestCase; + if (!cmdInfo.m_pythontTestCase.isEmpty()) + { + pythonTestCase = cmdInfo.m_pythontTestCase.toUtf8().constData(); + } + + EditorPythonRunnerRequestBus::Broadcast(&EditorPythonRunnerRequestBus::Events::ExecuteByFilenameAsTest, cmdInfo.m_strFileName.toUtf8().constData(), pythonTestCase, pythonArgs); // Close the editor gracefully as the test has completed GetIEditor()->GetDocument()->SetModifiedFlag(false); diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt index a26814fa51..bbc7dbde19 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt @@ -19,6 +19,9 @@ ly_add_target( Include PRIVATE Source + COMPILE_DEFINITIONS + PRIVATE + ${LY_TEST_IMPACT_CONFIG_FILE_PATH_DEFINITION} BUILD_DEPENDENCIES PUBLIC AZ::TestImpact.Runtime.Static diff --git a/Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.cpp b/Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.cpp index 00dbca12c9..e390578818 100644 --- a/Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.cpp +++ b/Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.cpp @@ -39,6 +39,7 @@ #include #include +#include namespace Platform { @@ -579,6 +580,9 @@ namespace EditorPythonBindings if (!script.empty()) { + AzToolsFramework::EditorPythonScriptNotificationsBus::Broadcast( + &AzToolsFramework::EditorPythonScriptNotificationsBus::Events::OnStartExecuteByString, script); + // Acquire GIL before calling Python code AZStd::lock_guard lock(m_lock); pybind11::gil_scoped_acquire acquire; @@ -639,11 +643,15 @@ namespace EditorPythonBindings void PythonSystemComponent::ExecuteByFilename(AZStd::string_view filename) { AZStd::vector args; + AzToolsFramework::EditorPythonScriptNotificationsBus::Broadcast( + &AzToolsFramework::EditorPythonScriptNotificationsBus::Events::OnStartExecuteByFilename, filename); ExecuteByFilenameWithArgs(filename, args); } - void PythonSystemComponent::ExecuteByFilenameAsTest(AZStd::string_view filename, const AZStd::vector& args) + void PythonSystemComponent::ExecuteByFilenameAsTest(AZStd::string_view filename, AZStd::string_view testCase, const AZStd::vector& args) { + AzToolsFramework::EditorPythonScriptNotificationsBus::Broadcast( + &AzToolsFramework::EditorPythonScriptNotificationsBus::Events::OnStartExecuteByFilenameAsTest, filename, testCase, args); const Result evalResult = EvaluateFile(filename, args); if (evalResult == Result::Okay) { @@ -659,6 +667,8 @@ namespace EditorPythonBindings void PythonSystemComponent::ExecuteByFilenameWithArgs(AZStd::string_view filename, const AZStd::vector& args) { + AzToolsFramework::EditorPythonScriptNotificationsBus::Broadcast( + &AzToolsFramework::EditorPythonScriptNotificationsBus::Events::OnStartExecuteByFilenameWithArgs, filename, args); EvaluateFile(filename, args); } diff --git a/Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.h b/Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.h index d5d1634c9d..69882e7d57 100644 --- a/Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.h +++ b/Gems/EditorPythonBindings/Code/Source/PythonSystemComponent.h @@ -58,7 +58,7 @@ namespace EditorPythonBindings void ExecuteByString(AZStd::string_view script, bool printResult) override; void ExecuteByFilename(AZStd::string_view filename) override; void ExecuteByFilenameWithArgs(AZStd::string_view filename, const AZStd::vector& args) override; - void ExecuteByFilenameAsTest(AZStd::string_view filename, const AZStd::vector& args) override; + void ExecuteByFilenameAsTest(AZStd::string_view filename, AZStd::string_view testCase, const AZStd::vector& args) override; //////////////////////////////////////////////////////////////////////// private: diff --git a/cmake/TestImpactFramework/ConsoleFrontendConfig.in b/cmake/TestImpactFramework/ConsoleFrontendConfig.in index 9338672fbd..ffd24ffb72 100644 --- a/cmake/TestImpactFramework/ConsoleFrontendConfig.in +++ b/cmake/TestImpactFramework/ConsoleFrontendConfig.in @@ -70,6 +70,9 @@ }, "test_target_meta": { "file": "${test_target_type_file}" + }, + "gem_target": { + "file": "${gem_target_file}" } } }, @@ -87,122 +90,6 @@ ], "shard": [ - { - "policy": "fixture_contiguous", - "target": "AzCore.Tests" - }, - { - "policy": "fixture_contiguous", - "target": "AzToolsFramework.Tests" - }, - { - "policy": "test_interleaved", - "target": "Framework.Tests" - }, - { - "policy": "test_interleaved", - "target": "LmbrCentral.Editor.Tests" - }, - { - "policy": "test_interleaved", - "target": "EditorLib.Tests" - }, - { - "policy": "test_interleaved", - "target": "PhysX.Tests" - }, - { - "policy": "test_interleaved", - "target": "ImageProcessing.Tests" - }, - { - "policy": "test_interleaved", - "target": "Atom_RPI.Tests" - }, - { - "policy": "test_interleaved", - "target": "Atom_RHI.Tests" - }, - { - "policy": "test_interleaved", - "target": "AzManipulatorFramework.Tests" - }, - { - "policy": "test_interleaved", - "target": "WhiteBox.Editor.Tests" - }, - { - "policy": "test_interleaved", - "target": "ImageProcessing.Tests" - }, - { - "policy": "test_interleaved", - "target": "AzManipulatorTestFramework.Tests" - }, - { - "policy": "test_interleaved", - "target": "AtomCore.Tests" - }, - { - "policy": "test_interleaved", - "target": "ImageProcessingAtom.Editor.Tests" - }, - { - "policy": "test_interleaved", - "target": "EditorPythonBindings.Tests" - }, - { - "policy": "test_interleaved", - "target": "Atom_Utils.Tests" - }, - { - "policy": "test_interleaved", - "target": "AudioEngineWwise.Editor.Tests" - }, - { - "policy": "test_interleaved", - "target": "Multiplayer.Tests" - }, - { - "policy": "test_interleaved", - "target": "LmbrCentral.Tests" - }, - { - "policy": "fixture_contiguous", - "target": "LyMetricsShared.Tests" - }, - { - "policy": "test_interleaved", - "target": "PhysX.Editor.Tests" - }, - { - "policy": "test_interleaved", - "target": "ComponentEntityEditorPlugin.Tests" - }, - { - "policy": "test_interleaved", - "target": "DeltaCataloger.Tests" - }, - { - "policy": "test_interleaved", - "target": "GradientSignal.Tests" - }, - { - "policy": "test_interleaved", - "target": "LyShine.Tests" - }, - { - "policy": "test_interleaved", - "target": "EMotionFX.Editor.Tests" - }, - { - "policy": "test_interleaved", - "target": "EMotionFX.Tests" - }, - { - "policy": "test_interleaved", - "target": "CrySystem.Tests" - } ] } } diff --git a/cmake/TestImpactFramework/EnumeratedGemTargets.in b/cmake/TestImpactFramework/EnumeratedGemTargets.in new file mode 100644 index 0000000000..183e8ff3bd --- /dev/null +++ b/cmake/TestImpactFramework/EnumeratedGemTargets.in @@ -0,0 +1,7 @@ +{ + "gems": { + [ +${enumerated_gem_targets} + ] + } +} \ No newline at end of file diff --git a/cmake/TestImpactFramework/LYTestImpactFramework.cmake b/cmake/TestImpactFramework/LYTestImpactFramework.cmake index 9789ce187f..a1114ea77d 100644 --- a/cmake/TestImpactFramework/LYTestImpactFramework.cmake +++ b/cmake/TestImpactFramework/LYTestImpactFramework.cmake @@ -18,15 +18,21 @@ option(LY_TEST_IMPACT_INSTRUMENTATION_BIN "Path to test impact framework instrum # Name of test impact framework console static library target set(LY_TEST_IMPACT_CONSOLE_STATIC_TARGET "TestImpact.Frontend.Console.Static") +# Name of test impact framework python coverage gem target +set(LY_TEST_IMPACT_PYTHON_COVERAGE_STATIC_TARGET "PythonCoverage.Editor.Static") + # Name of test impact framework console target set(LY_TEST_IMPACT_CONSOLE_TARGET "TestImpact.Frontend.Console") -# Directory for non-persistent test impact data trashed with each generation of build system +# Directory for test impact artifacts and data set(LY_TEST_IMPACT_WORKING_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/TestImpactFramework") -# Directory for temporary files generated at runtime +# Directory for artifacts generated at runtime set(LY_TEST_IMPACT_TEMP_DIR "${LY_TEST_IMPACT_WORKING_DIR}/Temp") +# Directory for files that persist between runtime runs +set(LY_TEST_IMPACT_PERSISTENT_DIR "${LY_TEST_IMPACT_WORKING_DIR}/Persistent") + # Directory for static artifacts produced as part of the build system generation process set(LY_TEST_IMPACT_ARTIFACT_DIR "${LY_TEST_IMPACT_WORKING_DIR}/Artifact") @@ -36,9 +42,18 @@ set(LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/Map # Directory for build target dependency/depender graphs set(LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/Dependency") -# Master test enumeration file for all test types +# Main test enumeration file for all test types set(LY_TEST_IMPACT_TEST_TYPE_FILE "${LY_TEST_IMPACT_ARTIFACT_DIR}/TestType/All.tests") +# Main gem target file for all shared library gems +set(LY_TEST_IMPACT_GEM_TARGET_FILE "${LY_TEST_IMPACT_ARTIFACT_DIR}/BuildType/All.gems") + +# Path to the config file for each build configuration +set(LY_TEST_IMPACT_CONFIG_FILE_PATH "${LY_TEST_IMPACT_PERSISTENT_DIR}/tiaf.$.json") + +# Preprocessor directive for the config file path +set(LY_TEST_IMPACT_CONFIG_FILE_PATH_DEFINITION "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${LY_TEST_IMPACT_CONFIG_FILE_PATH}\"") + #! ly_test_impact_rebase_file_to_repo_root: rebases the relative and/or absolute path to be relative to repo root directory and places the resulting path in quotes. # # \arg:INPUT_FILE the file to rebase @@ -212,7 +227,7 @@ function(ly_test_impact_extract_python_test_params COMPOSITE_TEST COMPOSITE_SUIT set(${TEST_SUITES} ${test_suites} PARENT_SCOPE) endfunction() -#! ly_test_impact_write_test_enumeration_file: exports the master test lists to file. +#! ly_test_impact_write_test_enumeration_file: exports the main test list to file. # # \arg:TEST_ENUMERATION_TEMPLATE_FILE path to test enumeration template file function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FILE) @@ -262,6 +277,33 @@ function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FI configure_file(${TEST_ENUMERATION_TEMPLATE_FILE} ${LY_TEST_IMPACT_TEST_TYPE_FILE}) endfunction() +#! ly_test_impact_write_gem_target_enumeration_file: exports the main gem target list to file. +# +# \arg:GEM_TARGET_TEMPLATE_FILE path to source to gem target template file +function(ly_test_impact_write_gem_target_enumeration_file GEM_TARGET_TEMPLATE_FILE) + get_property(LY_ALL_TARGETS GLOBAL PROPERTY LY_ALL_TARGETS) + + set(enumerated_gem_targets "") + # Walk the build targets + foreach(aliased_target ${LY_ALL_TARGETS}) + + unset(target) + ly_de_alias_target(${aliased_target} target) + + get_target_property(gem_module ${target} GEM_MODULE) + get_target_property(target_type ${target} TYPE) + if("${gem_module}" STREQUAL "TRUE") + if("${target_type}" STREQUAL "SHARED_LIBRARY" OR "${target_type}" STREQUAL "MODULE_LIBRARY") + list(APPEND enumerated_gem_targets " \"${target}\"") + endif() + endif() + endforeach() + string (REPLACE ";" ",\n" enumerated_gem_targets "${enumerated_gem_targets}") + # Write out source to target mapping file + set(mapping_path "${LY_TEST_IMPACT_GEM_TARGET_FILE}") + configure_file(${GEM_TARGET_TEMPLATE_FILE} ${mapping_path}) +endfunction() + #! ly_test_impact_export_source_target_mappings: exports the static source to target mappings to file. # # \arg:MAPPING_TEMPLATE_FILE path to source to target template file @@ -273,6 +315,7 @@ function(ly_test_impact_export_source_target_mappings MAPPING_TEMPLATE_FILE) unset(target) ly_de_alias_target(${aliased_target} target) + message(TRACE "Exporting static source file mappings for ${target}") # Target name and path relative to root @@ -333,9 +376,8 @@ endfunction() #! ly_test_impact_write_config_file: writes out the test impact framework config file using the data derived from the build generation process. # # \arg:CONFIG_TEMPLATE_FILE path to the runtime configuration template file -# \arg:PERSISTENT_DATA_DIR path to the test impact framework persistent data directory # \arg:BIN_DIR path to repo binary output directory -function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_DIR BIN_DIR) +function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE BIN_DIR) # Platform this config file is being generated for set(platform ${PAL_PLATFORM_NAME}) @@ -366,16 +408,19 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D set(temp_dir "${LY_TEST_IMPACT_TEMP_DIR}") # Active persistent data dir - set(active_dir "${PERSISTENT_DATA_DIR}/active") + set(active_dir "${LY_TEST_IMPACT_PERSISTENT_DIR}/active") # Historic persistent data dir - set(historic_dir "${PERSISTENT_DATA_DIR}/historic") + set(historic_dir "${LY_TEST_IMPACT_PERSISTENT_DIR}/historic") # Source to target mappings dir set(source_target_mapping_dir "${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}") # Test type artifact file set(test_target_type_file "${LY_TEST_IMPACT_TEST_TYPE_FILE}") + + # Gem target file + set(gem_target_file "${LY_TEST_IMPACT_GEM_TARGET_FILE}") # Build dependency artifact dir set(target_dependency_dir "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}") @@ -389,12 +434,10 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D # Write out entire config contents to a file in the build directory of the test impact framework console target file(GENERATE - OUTPUT "${PERSISTENT_DATA_DIR}/$.$.json" + OUTPUT "${LY_TEST_IMPACT_CONFIG_FILE_PATH}" CONTENT ${config_file} ) - # Set the above config file as the default config file to use for the test impact framework console target - target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${PERSISTENT_DATA_DIR}/$.$.json\"") message(DEBUG "Test impact framework post steps complete") endfunction() @@ -404,13 +447,11 @@ function(ly_test_impact_post_step) return() endif() - # Directory per build config for persistent test impact data (to be checked in) - set(persistent_data_dir "${LY_TEST_IMPACT_WORKING_DIR}/persistent") # Directory for binaries built for this profile set(bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$") # Erase any existing non-persistent data to avoid getting test impact framework out of sync with current repo state - file(REMOVE_RECURSE "${LY_TEST_IMPACT_WORKING_DIR}") + file(REMOVE_RECURSE "${LY_TEST_IMPACT_TEMP_DIR}") # Export the soruce to target mapping files ly_test_impact_export_source_target_mappings( @@ -422,10 +463,14 @@ function(ly_test_impact_post_step) "cmake/TestImpactFramework/EnumeratedTests.in" ) + # Export the enumerated gems + ly_test_impact_write_gem_target_enumeration_file( + "cmake/TestImpactFramework/EnumeratedGemTargets.in" + ) + # Write out the configuration file ly_test_impact_write_config_file( "cmake/TestImpactFramework/ConsoleFrontendConfig.in" - ${persistent_data_dir} ${bin_dir} ) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index e37e467fa0..5499dbef84 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -33,7 +33,7 @@ ], "steps": [ "profile_vs2019", - "test_impact_analysis", + "test_impact_analysis_profile_vs2019", "asset_profile_vs2019", "test_cpu_profile_vs2019" ] @@ -82,13 +82,15 @@ "SCRIPT_PARAMETERS": "--platform 3rdParty --type 3rdParty_all" } }, - "test_impact_analysis": { + "test_impact_analysis_profile_vs2019": { "TAGS": [ ], "COMMAND": "python_windows.cmd", "PARAMETERS": { + "OUTPUT_DIRECTORY": "build/windows_vs2019", + "CONFIGURATION": "profile", "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", - "SCRIPT_PARAMETERS": "--testFailurePolicy=continue --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" + "SCRIPT_PARAMETERS": "--testFailurePolicy=continue --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"!OUTPUT_DIRECTORY!/bin/TestImpactFramework/persistent/tiaf.profile.json\"" } }, "debug_vs2019": {