From d7c8c0192ffe008b4a3960de943997cf05ec6130 Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Thu, 8 Jul 2021 17:20:20 -0700 Subject: [PATCH] SPEC-7503 Mac nightly run asset profile failed to process "Translation/scriptcanvas_en_us.ts" (#1742) * copy frameworks to output for loose file case Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * patching lrelease Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Fix for python in loose files and bundles Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * found what the problem is, setting a version just in case Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * fixup_bundle working with Python.framework and codesigning on AssetProcessor.app Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Trying to ignore other files to solve Jenkins Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Fix some qt stuff for mac Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * removal of cstemp files to prevent signature failures Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Updating Qt with fixes to solve some qt framework issues with include paths Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- .../CMakeLists.txt | 1 + .../Outliner/OutlinerDisplayOptionsMenu.cpp | 2 + .../Plugins/FFMPEGPlugin/CMakeLists.txt | 1 + .../FFMPEGPlugin/FFMPEGPlugin_precompiled.h | 11 - .../Plugins/PerforcePlugin/CMakeLists.txt | 2 + .../Code/Platform/Mac/lrelease_mac.cmake | 9 + .../Platform/Mac/BuiltInPackages_mac.cmake | 2 +- .../Common/RuntimeDependencies_common.cmake | 3 +- .../runtime_dependencies_common.cmake.in | 1 + .../Mac/runtime_dependencies_mac.cmake.in | 195 ++++++++++++------ 10 files changed, 154 insertions(+), 73 deletions(-) diff --git a/Code/Editor/Plugins/ComponentEntityEditorPlugin/CMakeLists.txt b/Code/Editor/Plugins/ComponentEntityEditorPlugin/CMakeLists.txt index 4744b4e174..5aab2a2056 100644 --- a/Code/Editor/Plugins/ComponentEntityEditorPlugin/CMakeLists.txt +++ b/Code/Editor/Plugins/ComponentEntityEditorPlugin/CMakeLists.txt @@ -27,6 +27,7 @@ ly_add_target( BUILD_DEPENDENCIES PRIVATE 3rdParty::Qt::Core + 3rdParty::Qt::Widgets AZ::AzCore AZ::AzToolsFramework Legacy::CryCommon diff --git a/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerDisplayOptionsMenu.cpp b/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerDisplayOptionsMenu.cpp index c1af8d76d7..a6c1ed6598 100644 --- a/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerDisplayOptionsMenu.cpp +++ b/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerDisplayOptionsMenu.cpp @@ -66,3 +66,5 @@ namespace EntityOutliner emit OnOptionToggled(DisplayOption::AutoExpand, checked); } } + +#include diff --git a/Code/Editor/Plugins/FFMPEGPlugin/CMakeLists.txt b/Code/Editor/Plugins/FFMPEGPlugin/CMakeLists.txt index 342151f24a..909ec193f3 100644 --- a/Code/Editor/Plugins/FFMPEGPlugin/CMakeLists.txt +++ b/Code/Editor/Plugins/FFMPEGPlugin/CMakeLists.txt @@ -23,6 +23,7 @@ ly_add_target( . BUILD_DEPENDENCIES PRIVATE + 3rdParty::Qt::Core AZ::AzCore Legacy::CryCommon Legacy::EditorLib diff --git a/Code/Editor/Plugins/FFMPEGPlugin/FFMPEGPlugin_precompiled.h b/Code/Editor/Plugins/FFMPEGPlugin/FFMPEGPlugin_precompiled.h index 3b015dd9a5..33ca3390f1 100644 --- a/Code/Editor/Plugins/FFMPEGPlugin/FFMPEGPlugin_precompiled.h +++ b/Code/Editor/Plugins/FFMPEGPlugin/FFMPEGPlugin_precompiled.h @@ -17,19 +17,8 @@ ///////////////////////////////////////////////////////////////////////////// #include - -///////////////////////////////////////////////////////////////////////////// -// STL -///////////////////////////////////////////////////////////////////////////// -#include -#include -#include -#include -#include - ///////////////////////////////////////////////////////////////////////////// // CRY Stuff //////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////// #include -#include "Util/EditorUtils.h" #include "EditorCoreAPI.h" diff --git a/Code/Editor/Plugins/PerforcePlugin/CMakeLists.txt b/Code/Editor/Plugins/PerforcePlugin/CMakeLists.txt index 15a1147166..bd4661fa0f 100644 --- a/Code/Editor/Plugins/PerforcePlugin/CMakeLists.txt +++ b/Code/Editor/Plugins/PerforcePlugin/CMakeLists.txt @@ -31,6 +31,8 @@ ly_add_target( . BUILD_DEPENDENCIES PRIVATE + 3rdParty::Qt::Core + 3rdParty::Qt::Widgets AZ::AzCore Legacy::CryCommon Legacy::EditorLib diff --git a/Gems/LmbrCentral/Code/Platform/Mac/lrelease_mac.cmake b/Gems/LmbrCentral/Code/Platform/Mac/lrelease_mac.cmake index af355b0a51..34f67ccedd 100644 --- a/Gems/LmbrCentral/Code/Platform/Mac/lrelease_mac.cmake +++ b/Gems/LmbrCentral/Code/Platform/Mac/lrelease_mac.cmake @@ -5,6 +5,15 @@ # # +add_custom_command(TARGET LmbrCentral.Editor POST_BUILD + COMMAND "${CMAKE_COMMAND}" -P "${LY_ROOT_FOLDER}/cmake/Platform/Mac/RPathChange.cmake" + "$/lrelease" + @executable_path/../Frameworks + @executable_path + COMMENT "Patching lrelease..." + VERBATIM +) + set(lrelease_files ${QT_LRELEASE_EXECUTABLE} ) diff --git a/cmake/3rdParty/Platform/Mac/BuiltInPackages_mac.cmake b/cmake/3rdParty/Platform/Mac/BuiltInPackages_mac.cmake index 59da804e82..b261f18035 100644 --- a/cmake/3rdParty/Platform/Mac/BuiltInPackages_mac.cmake +++ b/cmake/3rdParty/Platform/Mac/BuiltInPackages_mac.cmake @@ -42,5 +42,5 @@ ly_associate_package(PACKAGE_NAME mikkelsen-1.0.0.4-mac ly_associate_package(PACKAGE_NAME googletest-1.8.1-rev4-mac TARGETS googletest PACKAGE_HASH cbf020d5ef976c5db8b6e894c6c63151ade85ed98e7c502729dd20172acae5a8) ly_associate_package(PACKAGE_NAME googlebenchmark-1.5.0-rev2-mac TARGETS GoogleBenchmark PACKAGE_HASH ad25de0146769c91e179953d845de2bec8ed4a691f973f47e3eb37639381f665) ly_associate_package(PACKAGE_NAME OpenSSL-1.1.1b-rev1-mac TARGETS OpenSSL PACKAGE_HASH 28adc1c0616ac0482b2a9d7b4a3a3635a1020e87b163f8aba687c501cf35f96c) -ly_associate_package(PACKAGE_NAME qt-5.15.2-rev4-mac TARGETS Qt PACKAGE_HASH 08790d03a0e6ad808ad64cf25c3d75abd69a343f3d224fc39927e5c6e8738b98) +ly_associate_package(PACKAGE_NAME qt-5.15.2-rev5-mac TARGETS Qt PACKAGE_HASH 9d25918351898b308ded3e9e571fff6f26311b2071aeafd00dd5b249fdf53f7e) ly_associate_package(PACKAGE_NAME libsamplerate-0.2.1-rev2-mac TARGETS libsamplerate PACKAGE_HASH b912af40c0ac197af9c43d85004395ba92a6a859a24b7eacd920fed5854a97fe) diff --git a/cmake/Platform/Common/RuntimeDependencies_common.cmake b/cmake/Platform/Common/RuntimeDependencies_common.cmake index 734dd94203..31c8191a57 100644 --- a/cmake/Platform/Common/RuntimeDependencies_common.cmake +++ b/cmake/Platform/Common/RuntimeDependencies_common.cmake @@ -254,7 +254,8 @@ function(ly_delayed_generate_runtime_dependencies) # Generate the output file set(target_file_dir "$") - file(READ ${LY_RUNTIME_DEPENDENCIES_TEMPLATE} template_file) + set(target_file "$") + ly_file_read(${LY_RUNTIME_DEPENDENCIES_TEMPLATE} template_file) string(CONFIGURE "${LY_COPY_COMMANDS}" LY_COPY_COMMANDS @ONLY) string(CONFIGURE "${template_file}" configured_template_file @ONLY) file(GENERATE diff --git a/cmake/Platform/Common/runtime_dependencies_common.cmake.in b/cmake/Platform/Common/runtime_dependencies_common.cmake.in index e2a85374b8..9d23a73a26 100644 --- a/cmake/Platform/Common/runtime_dependencies_common.cmake.in +++ b/cmake/Platform/Common/runtime_dependencies_common.cmake.in @@ -13,6 +13,7 @@ function(ly_copy source_file target_directory) if(NOT ${same_location}) file(LOCK ${target_directory}/${target_filename}.lock GUARD FUNCTION TIMEOUT 300) if("${source_file}" IS_NEWER_THAN "${target_directory}/${target_filename}") + message(STATUS "Copying \"${source_file}\" to \"${target_directory}\"...") file(COPY "${source_file}" DESTINATION "${target_directory}" FILE_PERMISSIONS @LY_COPY_PERMISSIONS@ FOLLOW_SYMLINK_CHAIN) endif() endif() diff --git a/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in b/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in index 92ad3514d4..9f9006c4c0 100644 --- a/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in +++ b/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in @@ -19,56 +19,96 @@ endfunction() include(BundleUtilities) cmake_policy(SET CMP0012 NEW) # new policy for the if that evaluates a boolean out of the LY_BUILD_FIXUP_BUNDLE expansion +cmake_policy(SET CMP0009 NEW) # do not traverse symlinks on GLOB_RECURSE set(anything_new FALSE) set(plugin_libs) set(plugin_dirs) +set(depends_on_python FALSE) + +find_program(LY_INSTALL_NAME_TOOL install_name_tool) +if (NOT LY_INSTALL_NAME_TOOL) + message(FATAL_ERROR "Unable to locate 'install_name_tool'") +endif() function(ly_copy source_file target_directory) get_filename_component(target_filename "${source_file}" NAME) - # If source_file is a Framework and target_directory is a bundle - if("${source_file}" MATCHES "\\.[Ff]ramework[^\\.]" AND "${target_directory}" MATCHES "\\.app/Contents/MacOS") + # If target_directory is a bundle + if("${target_directory}" MATCHES "\\.app/Contents/MacOS") - if(NOT @LY_BUILD_FIXUP_BUNDLE@) - return() - endif() + set(target_is_bundle TRUE) + if("${source_file}" MATCHES "\\.[Ff]ramework[^\\.]") - # fixup origin to copy the whole Framework folder and change destination to Contents/Frameworks - string(REGEX REPLACE "(.*\\.[Ff]ramework).*" "\\1" source_file "${source_file}") - get_filename_component(source_file_folder "${source_file}" DIRECTORY) - - set(local_plugin_dirs ${plugin_dirs}) - list(APPEND local_plugin_dirs "${source_file_folder}") - set(plugin_dirs ${local_plugin_dirs} PARENT_SCOPE) - return() + if(NOT @LY_BUILD_FIXUP_BUNDLE@) + return() + endif() - elseif("${source_file}" MATCHES "qt/plugins" AND "${target_directory}" MATCHES "\\.app/Contents/MacOS") + # fixup origin to copy the whole Framework folder and change destination to Contents/Frameworks + string(REGEX REPLACE "(.*\\.[Ff]ramework).*" "\\1" source_file "${source_file}") + get_filename_component(source_file_folder "${source_file}" DIRECTORY) + + # Python.framework produces a bug in BundleUtilities so it needs manual handling + # https://gitlab.kitware.com/cmake/cmake/-/issues/20165 + if("${source_file}" MATCHES "Python.framework") + # fixup the destination so it ends up in Contents/Frameworks + string(REGEX REPLACE "(.*\\.app/Contents)/MacOS" "\\1/Frameworks" target_directory "${target_directory}") + set(local_plugin_dirs ${plugin_dirs}) + list(APPEND local_plugin_dirs "${target_directory}/Python.framework") + set(target_filename Python.framework) + set(plugin_dirs ${local_plugin_dirs} PARENT_SCOPE) + set(depends_on_python TRUE PARENT_SCOPE) + else() + set(local_plugin_dirs ${plugin_dirs}) + list(APPEND local_plugin_dirs "${source_file_folder}") + set(plugin_dirs ${local_plugin_dirs} PARENT_SCOPE) + return() + endif() - if(NOT @LY_BUILD_FIXUP_BUNDLE@) - return() - endif() + elseif("${source_file}" MATCHES "qt/plugins") - # fixup the destination so it ends up in Contents/Plugins - string(REGEX REPLACE "(.*\\.app/Contents)/MacOS" "\\1/plugins" target_directory "${target_directory}") + if(NOT @LY_BUILD_FIXUP_BUNDLE@) + return() + endif() - set(local_plugin_dirs ${plugin_dirs}) - list(APPEND local_plugin_dirs "${target_directory}") - set(plugin_dirs ${local_plugin_dirs} PARENT_SCOPE) - set(local_plugin_libs ${plugin_libs}) - list(APPEND local_plugin_libs "${target_directory}/${target_filename}") - set(plugin_libs ${local_plugin_libs} PARENT_SCOPE) + # fixup the destination so it ends up in Contents/Plugins + string(REGEX REPLACE "(.*\\.app/Contents)/MacOS" "\\1/plugins" target_directory "${target_directory}") - elseif("${source_file}" MATCHES "qt/translations" AND "${target_directory}" MATCHES "\\.app/Contents/MacOS") + set(local_plugin_dirs ${plugin_dirs}) + list(APPEND local_plugin_dirs "${target_directory}") + set(plugin_dirs ${local_plugin_dirs} PARENT_SCOPE) + set(local_plugin_libs ${plugin_libs}) + list(APPEND local_plugin_libs "${target_directory}/${target_filename}") + set(plugin_libs ${local_plugin_libs} PARENT_SCOPE) - return() # skip, is this used? + elseif("${source_file}" MATCHES "qt/translations") + + return() # skip, is this used? + + elseif("${source_file}" MATCHES ".dylib") + + set(local_plugin_dirs ${plugin_dirs}) + list(APPEND local_plugin_dirs "${target_directory}") + set(plugin_dirs ${local_plugin_dirs} PARENT_SCOPE) + + endif() - elseif("${source_file}" MATCHES ".dylib") + else() + + # target is not a bundle + set(target_is_bundle FALSE) + if("${source_file}" MATCHES "\\.[Ff]ramework[^\\.]") + + # fixup origin to copy the whole Framework folder + string(REGEX REPLACE "(.*\\.[Ff]ramework).*" "\\1" source_file "${source_file}") + get_filename_component(target_filename "${source_file}" NAME) + + if("${source_file}" MATCHES "Python.framework") + set(depends_on_python TRUE PARENT_SCOPE) + endif() - set(local_plugin_dirs ${plugin_dirs}) - list(APPEND local_plugin_dirs "${target_directory}") - set(plugin_dirs ${local_plugin_dirs} PARENT_SCOPE) + endif() endif() @@ -79,6 +119,11 @@ function(ly_copy source_file target_directory) endif() if(NOT EXISTS "${target_directory}/${target_filename}" OR "${source_file}" IS_NEWER_THAN "${target_directory}/${target_filename}") message(STATUS "Copying \"${source_file}\" to \"${target_directory}\"...") + if(NOT target_is_bundle) + # if it is a bundle, there is no contention about the files in the destination, each bundle target will copy everything + # we dont want these files to invalidate the bundle and cause a new signature + file(LOCK ${target_directory}/${target_filename}.lock GUARD FUNCTION TIMEOUT 300) + endif() file(COPY "${source_file}" DESTINATION "${target_directory}" FILE_PERMISSIONS @LY_COPY_PERMISSIONS@ FOLLOW_SYMLINK_CHAIN) file(TOUCH ${target_directory}/${target_filename}) set(anything_new TRUE PARENT_SCOPE) @@ -93,6 +138,7 @@ if(NOT @LY_BUILD_FIXUP_BUNDLE@) endif() if(@target_file_dir@ MATCHES ".app/Contents/MacOS") + string(REGEX REPLACE "(.*\\.app)/Contents/MacOS.*" "\\1" bundle_path "@target_file_dir@") set(fixup_timestamp_file "${bundle_path}.fixup.stamp") if(NOT anything_new) @@ -106,31 +152,44 @@ if(@target_file_dir@ MATCHES ".app/Contents/MacOS") if(EXISTS ${bundle_path}/Contents/MacOS/Builders/DirectXShaderCompiler/bin/dxc-3.7) list(APPEND fixup_bundle_ignore dxc-3.7) endif() - # Python.framework being copied by fixup_bundle - #if(EXISTS ${bundle_path}/Contents/Frameworks/Python.framework) - # # LYN-4502: Patch python bundle, it contains some windows executables, some files that fixup_bundle doesnt like and has - # # duplicated binaries between Versions/3.7 and Versions/Current. - # file(GLOB exe_files - # ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/lib/python3.7/distutils/command/*.exe - # ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pip/_vendor/distlib/*.exe - # ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/setuptools/*.exe - # ) - # foreach(exe_file ${exe_files}) - # file(REMOVE ${exe_file}) - # endforeach() - # file(REMOVE_RECURSE - # ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/lib/python3.7/test - # ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/scipy/io/tests - # ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/Resources - # ${bundle_path}/Contents/Frameworks/Python.framework/Python - # ${bundle_path}/Contents/Frameworks/Python.framework/Resources/Python.app - # ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/Python - # ) - # file(REMOVE_RECURSE ${bundle_path}/Contents/Frameworks/Python.framework/Versions/Current) - # execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink 3.7 Current - # WORKING_DIRECTORY ${bundle_path}/Contents/Frameworks/Python.framework/Versions/ - # ) - #endif() + # LYN-4502: Patch python bundle, it contains some windows executables, some files that fixup_bundle doesnt like and has + # other issues that produce signature problems + if(depends_on_python) + message(STATUS "Fixing ${bundle_path}/Contents/Frameworks/Python.framework...") + list(APPEND fixup_bundle_ignore Python python3.7m python3.7) + file(REMOVE_RECURSE + ${bundle_path}/Contents/Frameworks/Python.framework/Versions/Current + ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/Headers + ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/lib/Python + ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/lib/python3.7/test + ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/scipy/io/tests + ${bundle_path}/Contents/Frameworks/Python.framework/Python + ${bundle_path}/Contents/Frameworks/Python.framework/Resources + ${bundle_path}/Contents/Frameworks/Python.framework/Headers + ) + file(GLOB_RECURSE exe_file_list "${bundle_path}/Contents/Frameworks/Python.framework/**/*.exe") + if(exe_file_list) + file(REMOVE_RECURSE ${exe_file_list}) + endif() + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink include/python3.7m Headers + WORKING_DIRECTORY ${bundle_path}/Contents/Frameworks/Python.framework/Versions/3.7 + ) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink 3.7 Current + WORKING_DIRECTORY ${bundle_path}/Contents/Frameworks/Python.framework/Versions/ + ) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink Versions/Current/Python Python + WORKING_DIRECTORY ${bundle_path}/Contents/Frameworks/Python.framework + ) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink Versions/Current/Headers Headers + WORKING_DIRECTORY ${bundle_path}/Contents/Frameworks/Python.framework + ) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink Versions/Current/Resources Resources + WORKING_DIRECTORY ${bundle_path}/Contents/Frameworks/Python.framework + ) + file(CHMOD ${bundle_path}/Contents/Frameworks/Python.framework/Versions/Current/Python + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_WRITE GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + ) + endif() list(REMOVE_DUPLICATES plugin_libs) list(REMOVE_DUPLICATES plugin_dirs) fixup_bundle("${bundle_path}" "${plugin_libs}" "${plugin_dirs}" IGNORE_ITEM ${fixup_bundle_ignore}) @@ -139,11 +198,27 @@ if(@target_file_dir@ MATCHES ".app/Contents/MacOS") # fixup bundle ends up removing the rpath of dxc (despite we exclude it) if(EXISTS ${bundle_path}/Contents/MacOS/Builders/DirectXShaderCompiler/bin/dxc-3.7) - find_program(LY_INSTALL_NAME_TOOL install_name_tool) - if (NOT LY_INSTALL_NAME_TOOL) - message(FATAL_ERROR "Unable to locate 'install_name_tool'") - endif() execute_process(COMMAND ${LY_INSTALL_NAME_TOOL} -add_rpath @executable_path/../lib ${bundle_path}/Contents/MacOS/Builders/DirectXShaderCompiler/bin/dxc-3.7) endif() + + # misplaced .DS_Store files can cause signing to fail + # Interrupted signatures can leave cstemp files behind that fail next signature + file(GLOB_RECURSE remove_file_list + "${bundle_path}/**/.DS_Store" + "${bundle_path/}**/*.cstemp" + ) + if(remove_file_list) + file(REMOVE_RECURSE ${remove_file_list}) + endif() + + endif() + +else() # Non-bundle case + + if(depends_on_python) + # RPATH fix python + execute_process(COMMAND ${LY_INSTALL_NAME_TOOL} -change @rpath/Python @rpath/Python.framework/Versions/Current/Python @target_file@) + endif() + endif()