From 81b849d71241427b5b6008000293b13fa9283286 Mon Sep 17 00:00:00 2001 From: AlQuemist <alquemist@Lyriks> Date: Mon, 22 Apr 2024 15:49:41 +0200 Subject: [PATCH 1/8] GitLab-CI: correct typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e28e1f5420..c1bf16a798e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,7 @@ native_Debian: - pwd - export CC=gcc; export CXX=g++ - export MPLBACKEND=Agg - # avoid using the default Debain Qt framework + # avoid using the default Debian Qt framework - QTCMAKE="/usr/local/Qt6/6.2.3/gcc_64/lib/cmake" - cmake --version stage: build -- GitLab From ab7a687513962830fcbd7dfd75e889c996a3d75e Mon Sep 17 00:00:00 2001 From: AlQuemist <alquemist@Lyriks> Date: Mon, 22 Apr 2024 19:15:29 +0200 Subject: [PATCH 2/8] rm unneeded zshell scripts which were used for making the Python package under MacOS --- devtools/deploy/mac/mk_pypack_macos.zsh.in | 341 -------------------- devtools/deploy/mac/shutils.zsh | 345 --------------------- 2 files changed, 686 deletions(-) delete mode 100755 devtools/deploy/mac/mk_pypack_macos.zsh.in delete mode 100644 devtools/deploy/mac/shutils.zsh diff --git a/devtools/deploy/mac/mk_pypack_macos.zsh.in b/devtools/deploy/mac/mk_pypack_macos.zsh.in deleted file mode 100755 index a4c12fc7100..00000000000 --- a/devtools/deploy/mac/mk_pypack_macos.zsh.in +++ /dev/null @@ -1,341 +0,0 @@ -#!/bin/zsh - -# ************************************************************************** # -# BornAgain: simulate and fit reflection and scattering -# -# @file mk_pypack_macos.sh.in -# @brief Adjusts the library references to make them relocatable (MacOS only) -# -# @homepage http://apps.jcns.fz-juelich.de/BornAgain -# @license GNU General Public License v3 or higher (see COPYING) -# @copyright Forschungszentrum Juelich GmbH 2016 -# @authors Scientific Computing Group at MLZ (see CITATION, AUTHORS) -# ************************************************************************** # - -# usage: -# % zsh mk_pypack_macos.sh -# -# This script extracts the library ids of the dynamic libraries (*.dylib) -# found in the library files and accordingly adjusts the library paths -# referring to the external libraries (like `libfftw3.dylib`) in the given libraries. -# The script searches also for dependencies up to the 2nd order and adds -# them to the external libraries. -# -# For instance, the script finds the dynamic libraries (*.dylib) -# in `.../lib/extra` folder, and adjusts the library reference paths -# in each given library file. -# -# Requires MacOS-specific native tools `otool` and `install_name_tool`. -# NOTE: Starting with macOS Catalina (macOS 10), Macs will use `zsh` as -# the default login shell and interactive shell across the operating system. -# All newly created user accounts in macOS Catalina will use zsh by default; -# see <https://support.apple.com/en-us/HT208050> - -# See further: -# - Runtime linking on Mac <https://matthew-brett.github.io/docosx/mac_runtime_link.html> -# - Apple developer documentation: 'Run-Path Dependent Libraries' <https://apple.co/3HVbMWm> -# - Loading Dynamic Libraries on Mac <http://clarkkromenaker.com/post/library-dynamic-loading-mac> -# - <https://stackoverflow.com/q/66268814> -# - dlopen manpage - -# NOTE: -# - MacPorts: -# By default, MacPorts installs libraries under `/opt/local` (see <https://guide.macports.org>) -# Using -# % port contents qt5-qtbase -# it is found that the Qt root folder is `/opt/local/libexec/qt5`. -# - Homebrew: -# By default, Homebrew installs libraries under `/usr/local/opt/` and `/usr/local/Cellar` (see <https://docs.brew.sh/Manpage>) -# Qt default installation folder will be: -# $HOMEBREW_PREFIX/opt/qt@5 -# $HOMEBREW_PREFIX/opt/qt5 -# $HOMEBREW_PREFIX/Cellar/qt@5/ -# The variable `HOMEBREW_PREFIX` can be found via -# % brew config | grep HOMEBREW_PREFIX -# by default, HOMEBREW_PREFIX='/usr/local'. -# To obtain the value of the config variable use: -# % brew config | sed -nE 's;.*HOMEBREW_PREFIX:[[:blank:]]*(.+)[[:blank:]]*;\1;p' - -# include shell helper functions -# (expected to be in the same folder as the current script) -source "@CMAKE_SOURCE_DIR@/devtools/deploy/mac/shutils.zsh" -#======================================== - -# externally given variables -pyoutdir="@PY_OUTPUT_DIR@" -libdir="@BA_PY_LIBRARY_OUTPUT_DIR@" -xlibdir="@BA_PY_EXTRA_LIBRARY_OUTPUT_DIR@" -wheeldir="@WHEEL_DIR@" -PYTHON="@Python3_EXECUTABLE@" - -# constants -TITLE="MacOS PyPack" -HELP="$TITLE: Usage: zsh mk_pypack_macos.sh" - -if [ -z $libdir ] -then - echo "$TITLE: Error: Provide the directory for the main libraries." - echo "$HELP" - exit 1 -fi - -if [ -z $xlibdir ] -then - echo "$TITLE: Error: Provide the directory for the external libraries." - echo "$HELP" - exit 1 -fi - -# NOTE: For zsh, splitting on IFS could be done using `echo ${=var}`. - -# list of the main library files (BornAgain shared libraries) -libfiles=($libdir/*.so*) - -# turn logging off (0), log to stdout (1) -LOG_FLAG=1 -LOG_TOKEN=".:" -# separator line for output -SEPLINE="==========" - -echo "$TITLE: library directories = '$libdir', '$xlibdir'" -#-- functions - -# Qt default paths -brew1="/usr/local/opt" -brew2="/opt/homebrew/opt" -port1="/opt/local/libexec" - -# NOTE: `echo ... >&2` is used for logging inside functions -function log -{ - if [[ $LOG_FLAG == 1 ]]; then - echo "$LOG_TOKEN $1" >&2 - fi -} - -# use otool to get the base id of a library (raw output) -libname_re='[[:alnum:]_.-]+' -# otool output example: -# ``` -# cmd LC_ID_DYLIB -# cmdsize 56 -# name @rpath/_libBornAgainSample.1.so (offset 24) -# ``` -# eg. 'name @rpath/libA.1.so (offset 24)' => 'libA.1.so' -lib_id_re='s;.+name[[:blank:]]+.+/('$libname_re')[[:blank:]]+\(.+;\1;p' - -function dylib_base_id -{ - # return the base library id for the given library ($1) - otool -XD "$1" | sed -nE $lib_id_re -} - -# get dependencies of a given library on BornAgain libraries -function get_BornAgain_depends -{ - # obtain the BornAgain dependencies for a given library ($1); - # remove the first line which is the name of the file itself; - # find lines containing 'libBornAgain' - dylib_deps "$1" | grep -iF 'libBornAgain' -} - -# get 1st-order dependencies for a given file -function get_depends1 -{ - # obtain the dependencies for a given library ($1) - local optional_deps_re='/(opt|Cellar)/' - local _deps=$(dylib_deps "$1" | grep -E $optional_deps_re) - # make a clean list of dependencies without duplicates - # eg. ' /usr/local/opt/foo.1.dylib (compatibility ...)' => '/usr/local/opt/foo.dylib' - # NOTE: References starting with '@' must be neglected; - # eg., '@rpath/foo.dylib' - local path_re='s;[[:blank:]]*([^@]'$libname_re')[[:blank:]]+.+;\1;p' - echo "$_deps" | sort -u | sed -nE $path_re -} - -# get Python dependence for a given file -function get_python_dependence -{ - # extract the Python dependency from the given library ($1) - # regexp to extract the Python dependence; eg., - # '/usr/local/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/Python' - libdir_re='[[:alnum:]_./-]' - py_fmwk_re='s;[[:blank:]]*([^@]'$libdir_re')/(Python|libpython.+\.dylib)[[:blank:]]+.+;\1/\2;p' - # regexp to correct the Python dependence; eg.: - # '/usr/local/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/Python' => 'libpython3.9.dylib' - # '/usr/local/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/libpython3.9.dylib' => 'libpython3.9.dylib' - pylib_re='s;.+/(Python|libpython)([0-9.]+)\.dylib;libpython\2.dylib;' - # obtain the dependencies - pydeps0=$(dylib_deps "$1") - pydepends_fullpath=$(echo "$pydeps0" | sed -nE $py_fmwk_re) - pydepends_filename=$(echo "$pydepends_fullpath" | sed -E $pylib_re) - # return the Python dependence fullpath and filename - echo "$pydepends_fullpath $pydepends_filename" -} - - -# rename the external libraries to their library ids -for lib in $xlibdir/*.dylib*; do - lib_id=$(dylib_base_id $lib) - mv -f "$lib" "$xlibdir/$lib_id" -done - -# gather all dependencies -declare -i LEVELMAX=20 level=0 -declare -a libs_lv=($libfiles) -declare -A ex_deps deps_lv -echo "$TITLE: Find optional dependencies (up to level $LEVELMAX)..." - -while [[ $libs_lv ]]; do - level+=1 # level nr. - deps_lv=() # dependencies at the current level (without duplicates) - - # avoid going infinitely deep (due to some error) - if (( $level > $LEVELMAX )); then - echo "Error: Dependency level $level is too deep." - break - fi - # eg. at level 3, print '===[3]' - for i in {1..$level}; do print -n '='; done; print "[$level]" - - # obtain all dependencies at the current level - for lib in $libs_lv; do - echo "[L$level] $lib" - deps_=$(get_depends1 $lib) - # NOTE: Associative array is used to make the entries unique - for dp in `echo ${=deps_}`; do - deps_lv[$dp]=1 - done - done - # libraries for the next level are the dependencies at the current level - libs_lv=("${(k)deps_lv[@]}") - # add the dependencies of the current level to set of all dependencies - for lib in $libs_lv; do - # ignore dependence on Python libraries - [[ $lib =~ ".*Python|libpython.*" ]] && continue - # ignore dependence on BornAgain libraries - [[ $lib = "*BornAgain*" ]] && continue - ex_deps[$lib]="${lib##*/}" # the basename of the library - done -done -echo "...Done." -echo "$TITLE: All optional dependencies (last level $level) {full-path => basename}:" -for pth fnm in "${(@kv)ex_deps}"; do - echo " + '$pth' => '$fnm'" -done - -# gather dependencies on BornAgain libraries -declare -A ba_deps -# eg. ' @rpath/_libBornAgainBase.1.so.1.2 (compatibility ...)' -# => '_libBornAgainBase.1.so.1.2' -ba_lib_ref_re='s;[[:blank:]]*(.*/_libBornAgain.+\.[0-9.]*so[0-9.]*).+;\1;p' -# eg. '@rpath/_libBornAgainBase.1.2.so' => '_libBornAgainBase.so' -ba_lib_re='s;.*(_libBornAgain[[:alpha:]]+)\..+;\1.so;p' -for lib in $libfiles; do - deps=$(get_BornAgain_depends "$lib" | sed -nE $ba_lib_ref_re) - for libref in `echo ${=deps}`; do - # if already seen, then do nothing - [[ -v 'ba_deps[$libref]' ]] && continue - fnm=$(echo "$libref" | sed -nE $ba_lib_re) - # if extracted basename is empty, then do nothing - [[ -z $fnm ]] && continue - ba_deps[$libref]="@rpath/$fnm" - done -done - -echo "$TITLE: References to BornAgain libraries {old ref => new ref}..." -for ref0 ref1 in "${(@kv)ba_deps}"; do - echo " + '$ref0' => '$ref1'" -done - -# gather the Python dependence -for lib in $libfiles; do - _dps=$(get_python_dependence "$lib") - _pydep="$_pydep $_dps" -done -# get the first element the list Python dependences (all others must be the same) -# NOTE: zsh array indexing starts at 1 (unless option KSH_ARRAYS is set) -_pydep=(`echo ${=_pydep}`) -pydepends_fullpath=$_pydep[1] -pydepends_filename=$_pydep[2] -declare -a py_fmwk_rpaths=($(get_python_framework_path "$pydepends_fullpath")) - -echo "$TITLE: Python dependence" -echo " + path: '$pydepends_fullpath'" -echo " + library: '$pydepends_filename'" -echo " + framework paths:" -for pth in $py_fmwk_rpaths; do - echo " - '$pth'" -done - -#-- copy external dependencies to the associated folder -echo "$TITLE: Copying external dependencies to '$xlibdir'..." - -for pth fnm in ${(@kv)ex_deps}; do - cp -fL "$pth" "$xlibdir/$fnm" -done - -# report the external libraries -declare -a xlibfiles=($xlibdir/*.(dylib|so)) -echo "$TITLE: External libraries in '$xlibdir':" -for lib in $xlibfiles; do - # display only the filename (drop the dirname) - echo " + "${lib##$xlibdir/} -done - -#-- modify the library references in all libraries (main & external) -# ref to external libraries -for lib in $libfiles $xlibfiles; do - for pth fnm in "${(@kv)ex_deps}"; do - # change the dependency path in the library - # eg. '/usr/local/opt/foo.dylib' => '@rpath/foo.dylib' - rpth="@rpath/$fnm" - install_name_tool "$lib" -change "$pth" "$rpth" 2> >(grep -v "invalidate the code signature") - done - log "Changed external-library references in '$lib'" -done -# ref to BornAgain libraries -for lib in $libfiles; do - for ref0 ref1 in "${(@kv)ba_deps}"; do - # change the dependency path in the library - # eg. '@rpath/_libBornAgainBase.1.so.2' => '@rpath/_libBornAgainBase.so' - install_name_tool "$lib" -change "$ref0" "$ref1" 2> >(grep -v "invalidate the code signature") - done - log "Changed BornAgain-library references in '$lib'" -done -# ref to Python library (only in main libraries) -if [ -z $pydepends_fullpath ] || [ -z $pydepends_filename ]; then - echo "$TITLE: No Python dependence found." -else - for lib in $libfiles; do - install_name_tool "$lib" -change "$pydepends_fullpath" "@rpath/$pydepends_filename" 2> >(grep -v "invalidate the code signature") - done - # add proper framework RPATHs - for lib in $libfiles; do - for rpth in $py_fmwk_rpaths; do - install_name_tool "$lib" -add_rpath "$rpth" 2> >(grep -v "invalidate the code signature") - done - done - echo "$TITLE: Changed references to the Python shared library and added framework RPATHs." -fi - -# add proper RPATHs to external libraries -for lib in $xlibfiles; do - install_name_tool "$lib" -add_rpath "@loader_path" 2> >(grep -v "invalidate the code signature") - echo "$TITLE: Added proper RPATHs to '$lib'" -done - -# add proper RPATHs to BornAgain libraries (if needed) -# for lib in $libfiles; do -# install_name_tool "$lib" \ -# -add_rpath "@loader_path" \ -# -add_rpath "@loader_path/extra" -# log "$TITLE: Added proper RPATHs to '$lib'" -# done - -# make the Python wheel -echo "$TITLE: Making the Python wheel..." -eval $PYTHON -m pip wheel "$pyoutdir" --no-deps --wheel "$wheeldir" - -#-- final message -echo "$TITLE: Done." diff --git a/devtools/deploy/mac/shutils.zsh b/devtools/deploy/mac/shutils.zsh deleted file mode 100644 index fcb78ac868d..00000000000 --- a/devtools/deploy/mac/shutils.zsh +++ /dev/null @@ -1,345 +0,0 @@ -#!/bin/zsh - -# ************************************************************************** # -# BornAgain: simulate and fit reflection and scattering -# -# @file shutils.zsh -# @brief A collection of useful zshell functions. -# -# @homepage http://apps.jcns.fz-juelich.de/BornAgain -# @license GNU General Public License v3 or higher (see COPYING) -# @copyright Forschungszentrum Juelich GmbH 2016 -# @authors Scientific Computing Group at MLZ (see CITATION, AUTHORS) -# ************************************************************************** # - -# Shell helper functions - -# For details of ZSH parameter expansions, consult `man zshexpn` or -# <https://zsh.sourceforge.io/Doc/Release/Expansion.html#Parameter-Expansion-Flags> - -# use extended glob (see <https://zsh.sourceforge.io/Doc/Release/Expansion.html>); -# similar to `shopt -s extglob` in Bash. -setopt KSH_GLOB - -# turn logging off (0), log to stdout (1) -LOG_FLAG=1 -LOG_TOKEN=".:" - -function log -# NOTE: `echo ... >&2` is used for logging inside functions -{ - if [[ $LOG_FLAG == 1 ]]; then - echo "$LOG_TOKEN $@" >&2 - fi -} - -function dylib_id -# return the base library id for the given library ($1) -{ - otool -XD "$1" -} - - -# use otool to get the 1st-order dependencies of a library (raw output) -function dylib_deps -{ - # obtain the dependencies for a given library ($1); - # remove the name of the library itself from the references. - # Under MacOS, a Mach-O binary sometimes depends on itself. - # otool output example: ' /usr/local/opt/foo.dylib (compatibility ...)' - local basename="${1##*/}" - otool -XL "$1" | grep -vF "$basename" -} - -function rm_list_duplicates -# remove duplicates from a given list, $@; (separator is a space char) -{ - echo $@ | tr -s ' ' '\n' | sort -u | tr '\n' ' ' -} - -function find_common_root -# find the longest common root of two given absolute paths ($1 and $2) -{ - if [[ "$1" = "$2" ]]; then - # if paths are equal, the root is equal to either of them - common_root="$1" - else - common_root="" - # convert paths to arrays of directories: - # replace '/' with blank and make an array out of the result; - # eg. '/root/lib/opt' => [root, lib, opt] - declare -ar dirs1=(${=${1//\// }}) dirs2=(${=${2//\// }}) - # NOTE: zsh array indexing starts at 1 (unless option KSH_ARRAYS is set) - for idx in {1..$#dirs1}; do - # extract the head (topmost) directory name from paths - # eg. 'root/lib/opt' => 'root' - head1=$dirs1[$idx] - head2=$dirs2[$idx] - # if both paths have the same head, then add head to the common root; - # otherwise, the longest common root is already obtained - [[ "$head1" != "$head2" ]] && break - # if any of the heads are empty, then root-finding is finished - [[ -z $head1 || -z $head2 ]] && break - # add the common head to the root - common_root+="/$head1" - done - fi - # return the longest common root - print $common_root -} - -function test_find_common_root -{ - path1="/PKG/Foo/Content/Binaries" - path2="/PKG/Foo/Content/Lib/Libraries" - echo "path1='$path1'; path2='$path2'; common_root='" \ - $(find_common_root "$path1" "$path2") \ - "'; expected '/PKG/Foo/Content'" -} - -function find_rpath -# find the proper rpath for given binary pointing to a reference library (needs extended glob) -# usage: find_rpath(bin_abspath, lib_abspath, lib_relpath) -# example: -# ``` -# bin_abspath='/root/usr/opt/bin' -# lib_abspath='/root/usr/Frameworks/Qux/lib' -# lib_relpath='Qux/lib' -# echo $(find_rpath $bin_abspath $lib_abspath $lib_relpath) -# ``` -# returns `../../Frameworks` -{ - # drop the final '/' chars from all paths - bin_abspath="${1%%+(/)}" # target binary for which a rpath is obtained - lib_abspath="${2%%+(/)}" # referenced library - lib_relpath="${3%%+(/)}" # relative path to the referenced library - # lib_relpath="${lib_relpath##+(/)}" - # find the longest common root path - root_path=$(find_common_root "$bin_abspath" "$lib_abspath") - root_path="${root_path%%+(/)}" - # obtain the path from the binary to the root - # eg. '/root/local/opt' => 'local/opt' => '../..' - binpth_to_root=$(echo $bin_abspath | \ - sed -E 's,^'$root_path'(/|$),,;s/[^/]+/../g') - # obtain the path from root to the referenced library; - # eg. '/root/local/opt' => 'local/opt' - # then, drop the relative path of the library from the end - libpth_from_root=$(echo $lib_abspath | \ - sed -E 's,^'$root_path'(/|$),,;s,(^|/)'$lib_relpath',,') - # return the proper relative RPATH to the referenced library - # eg. '../../Frameworks/Qt' - if [[ -z "$binpth_to_root" ]]; then - libpth_from_bin="$libpth_from_root" - else - libpth_from_bin="$binpth_to_root/$libpth_from_root" - fi - print $libpth_from_bin -} - - -function test_find_rpath -{ - msg="$funcstack[1]" - root='/usr/root/local' - bin="$root/opt/bin/" - lib="$root/Frameworks/Qux/Foo.framework/Versions/5/" - librel='Foo.framework/Versions/5/' - rpath_ex='../../Frameworks/Qux' - rpath=$(find_rpath "$bin" "$lib" "$librel") - echo "$msg: bin_abspath='$bin', lib_abspath='$lib', lib_relpath='$librel' => " \ - "rpath='$rpath'; expected '$rpath_ex'" - - echo - lib="$root/lib/" - librel='' - rpath_ex='../../lib' - rpath=$(find_rpath "$bin" "$lib" "$librel") - echo "$msg: bin_abspath='$bin', lib_abspath='$lib', lib_relpath='$librel' => " \ - "rpath='$rpath'; expected '$rpath_ex'" - - echo - lib="$bin" - librel='' - rpath_ex='' - rpath=$(find_rpath "$bin" "$lib" "$librel") - echo "$msg: bin_abspath='$bin', lib_abspath='$lib', lib_relpath='$librel' => " \ - "rpath='$rpath'; expected '$rpath_ex'" - -} - -function is_in -# check if an element exists in an array -# usage: is_in(element, array) -# example: -# ``` -# declare -a xs=("a" "bc" "d") -# echo $(is_in "b" $xs) -# ``` -# returns 0 -{ - elm=$1 - shift # drop the first argument; the rest is the array - arr=$@ - if [[ " $arr " = *" $elm "* ]]; then - exists=1 - else - exists=0 - fi - print $exists -} - - -function test_is_in -{ - msg="$funcstack[1]" - xs_="a b cd d de efg" - declare -ar xs=(${=xs_}) - elm='cd' - echo "$msg: '$elm' is in { $xs }?" $(is_in $elm $xs) "; expected: 1" - elm='c' - echo "$msg: '$elm' is in { $xs }?" $(is_in $elm $xs) "; expected: 0" -} - -function get_depends1 -# get 1st-order dependencies for a given file -{ - # obtain the 'non-system' dependencies for a given library ($1) - # eg. ' /usr/local/opt/foo.1.dylib (compatibility ...)' => '/usr/local/opt/foo.dylib' - # system dependencies pattern: /System/, /Library/, /usr/lib/, /usr/local/lib - path_re='s;[[:blank:]]*([^[:blank:]]+).*;\1;p' - system_dep_re='\/(usr\/lib|usr\/local\/lib|System|Library)\/' - dylib_deps "$1" | sed -nE '/'$system_dep_re'/!'$path_re -} - -function find_dependencies -# gather all dependencies for the given initial libraries ($@) -# up to a maximum (but arbitrary) level. -{ - declare -ir LEVELMAX=20 # max. allowed level - log "$TITLE: Find dependencies (up to level $LEVELMAX)..." - # NOTE: Associative arrays are used to keep a set of entries without repetition - declare -A all_deps # absolute dependencies - declare -a libs_lv=(${=@}) # libraries at the current level - declare -i level=0 # current level nr. - - while [[ $libs_lv ]]; do - level+=1 - declare -A abs_deps_lv=() # _absolute_ dependencies at the current level - # avoid going infinitely deep (due to some error) - if (( $level > $LEVELMAX )); then - log "Error: Dependency level $level exceeds the maximum allowed depth ($LEVELMAX)." - break - fi - # eg. at level 3, print '==>[L3]' - log "==>[L$level]" - # obtain all dependencies at the current level - for lib in $libs_lv; do - # neglect previously-observed libraries - [ ! -z $all_deps[$lib] ] && continue - log "[L$level] $lib" - for dep in $(get_depends1 $lib); do - # store relative dependencies which begin with '@' - # eg. '@rpath/foo.dylib' - if [[ $dep = @* ]]; then - all_deps[$dep]=1 - # abs. path to relative dependencies is not known, therefore - # they should not be added to the current abs. dependencies - continue - fi - # add dependency to the current abs. dependencies - abs_deps_lv[$dep]=1 - done - done - # add libraries of the current level to set of all dependencies; - # libraries at the initial level are neglected. - if (( $level > 1 )); then - for libpath in $libs_lv; do - all_deps[$libpath]=1 - done - fi - # libraries for next level are the absolute dependencies at current level - libs_lv=( ${(@k)abs_deps_lv} ) - done - # return all discovered dependencies - print "${(@k)all_deps}" -} - -function get_python_dependence -# extract the Python dependency from the given dependencies ($@) -# NOTE: Under OSX, Python's library name is either 'Python', or -# for Python3.9, 'libpython3.9.dylib'. -{ - # regexp to extract the Python dependence; - # eg., '/Frameworks/Python.framework/Versions/3.9/Python' - libdir_re='[[:alnum:]_./@-]+' - pylibname_re='Python|libpython.+\.dylib' - py_fmwk_re='s;[[:blank:]]*('$libdir_re')/('$pylibname_re')[[:blank:]]+.*;\1/\2;p' - # regexp to correct the Python dependence; eg.: - # '/Frameworks/Python.framework/Versions/3.9/Python' => 'libpython3.9.dylib' - # '/Frameworks/Python.framework/Versions/3.9/libpython3.9.dylib' => 'libpython3.9.dylib' - pydylib_re='s;.*[pP]ython.+[Vv]ersions/([0-9.]+).+;libpython\1.dylib;' - # obtain the dependencies - pydepends_fullpath=$(echo $@ | sed -nE $py_fmwk_re) - pydepends_filename=$(echo "$pydepends_fullpath" | sed -E $pydylib_re) - # return the Python dependence fullpath and filename - print "$pydepends_fullpath $pydepends_filename" -} - -function get_python_framework_path -# produce proper Python framework paths for a given Python dependency ($1) -{ - pydepends_fullpath=$1 - # regexp to extract the Python framework path - # eg. '/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/Python' - # => '/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/' - py_fmwk_path_re='s;(.*)/(Python|libpython).*;\1;' - py_fmwk_dir=$(echo "$1" | sed -E $py_fmwk_path_re) - # when the library is like '.../Versions/3.9/Python', then - # add an extra '/lib' to the framework path; - # this is needed since it refers to the standard location of the - # Python shared library on OSX, '.../Versions/3.9/lib/libpython39.dylib'. - if [[ $1 = */Python ]]; then - py_fmwk_dir="$py_fmwk_dir/lib" - fi - # regexp to extract the Python version; eg. '3.9' - pyversion_re='s;.+[Vv]ersions/([0-9.]+).*;\1;' - declare -r pyversion=$(echo "$1" | sed -E $pyversion_re) - # RPATHs corresponding to the common OSX framework paths - declare -r py_fmwk_basepath="Python.framework/Versions/$pyversion/lib" - declare -ar framework_paths=( /usr/local/Library/Frameworks - /Library/Frameworks /usr/local/Frameworks ) - # collect proper RPATHs for Python framework - py_fmwk_rpaths="$py_fmwk_dir" - for pth in $framework_paths; do - py_fmwk_rpaths+=" $pth/$py_fmwk_basepath" - done - # return a list of possible framework paths - print $py_fmwk_rpaths -} - -function get_python_framework_path_rel -# produce proper Python framework paths for a given _relative_ Python dependency ($1) -{ - pydepends_relpath=$1 - # regexp to extract the Python framework path - # eg. '@rpath/libpython39.dylib' => 'libpython39.dylib' - pyversion_re='s;.*/libpython([0-9].[0-9]+).*;\1;' - declare -r pyversion=$(echo "$1" | sed -E $pyversion_re) - # RPATHs corresponding to the common OSX framework paths - declare -r py_fmwk_basepath="Python.framework/Versions/$pyversion/lib" - declare -ar framework_paths=( /usr/local/Library/Frameworks - /Library/Frameworks /usr/local/Frameworks ) - # collect proper RPATHs for Python framework - py_fmwk_rpaths="" - for pth in $framework_paths; do - py_fmwk_rpaths+=" $pth/$py_fmwk_basepath" - done - # return a list of possible framework paths - print $py_fmwk_rpaths -} -#======================================== - -#-- perform tests -- -# print "$(test_find_rpath)" -# print "$(test_is_in)" -#======================================== -- GitLab From 68cd027fdf4219da24c771ac9a3d60ae0ce61df5 Mon Sep 17 00:00:00 2001 From: AlQuemist <alquemist@Lyriks> Date: Mon, 22 Apr 2024 14:55:05 +0200 Subject: [PATCH 3/8] PyDependences: add 'wheel' to Python dependencies --- cmake/multipython/PyDependences.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/multipython/PyDependences.cmake b/cmake/multipython/PyDependences.cmake index 49240a97af5..6fe031b13d6 100644 --- a/cmake/multipython/PyDependences.cmake +++ b/cmake/multipython/PyDependences.cmake @@ -21,7 +21,7 @@ message(STATUS "Python package destination directory: ${destination_pypackage}") # check presence of some Python modules message(STATUS "Searching required Python packages...") -set(py_packages "pip;numpy") +set(py_packages "pip;wheel;numpy") if (BA_TESTS) # Python packages needed for tests list(APPEND py_packages "matplotlib;lmfit") -- GitLab From bf0361c04e951a5fd5c7ee4cef14cb55a3c70857 Mon Sep 17 00:00:00 2001 From: AlQuemist <alquemist@Lyriks> Date: Mon, 15 Apr 2024 14:26:11 +0200 Subject: [PATCH 4/8] Add a Python script to determine the wheel name via setuptools --- cmake/multipython/wheelname.py | 81 ++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 cmake/multipython/wheelname.py diff --git a/cmake/multipython/wheelname.py b/cmake/multipython/wheelname.py new file mode 100644 index 00000000000..20ff3d21df9 --- /dev/null +++ b/cmake/multipython/wheelname.py @@ -0,0 +1,81 @@ +""" +Determine the wheel name via setuptools + +Usage: +$ python3 wheelname.py <package-name> <version-string> +""" + +from setuptools import Extension, Distribution +import platform + +HEADER = "wheelname.py" + +def wheel_tags(**kwargs): + """ set up a dummy distribution from arguments + and return a resulting wheel name + """ + + dst = Distribution(attrs=kwargs) + # finalize bdist_wheel command + bdist_wheel_cmd = dst.get_command_obj('bdist_wheel') + bdist_wheel_cmd.ensure_finalized() + + distname = bdist_wheel_cmd.wheel_dist_name # eg. 'testdist-1.2.3' + tags = bdist_wheel_cmd.get_tag() # eg. (cp311, cp311, linux_x86_64) + + return (distname, *tags) + + +def wheel_tag_mac_x64(distname, py_tag, abi_tag, platform_tag): + """ Ad-hoc fix for the platform tag for conda under MacOS-x64. + + Python binaries under conda platform are compiled with an old MacOS (10.9) + in order to be compatible with later MacOS versions. + Therefore, under MacOS, conda's `pip` considers binaries compiled and linked + with higher MacOS versions as _invalid_. + A quick and dirty fix is changing the platform tag to 'macosx_10_9_x86_64'. + + NOTE: Use `python3 -m pip debug -vvv` to obtain valid wheel names + under a Python platform. + """ + + if platform.system().lower() == 'darwin' and platform.machine() == 'x86_64': + # platform tag compatible with MacOS-x64 conda Python platform + platform_tag = "macosx_10_9_x86_64" + + return (distname, py_tag, abi_tag, platform_tag) + + +def wheel_name(*tags): + # eg. 'testdist-1.2.3-cp311-cp311-linux_x86_64' + return '-'.join(*tags) + + +def get_wheel_names(pkg_name:str, version_str:str): + # get the name of the pure-Python wheel + pure_tags = wheel_tags(name=pkg_name, version=version_str) + pure_whl = wheel_name(pure_tags) + + # get tags for the platform-dependent wheel + plt_tags = wheel_tags(name=pkg_name, version=version_str, + ext_modules=[Extension("dummylib", ["dummy.c"])]) + # fix platform tag to be compatible with conda under MacOS-x64 + platform_tags = wheel_tag_mac_x64(*plt_tags) + platform_whl = wheel_name(platform_tags) + + # NOTE: CMake list separator is semicolon + return ';'.join((pure_whl, platform_whl)) + +#---------------------------------------- + +if __name__ == "__main__": + import sys + + args = sys.argv + if len(args) <= 2: + print("Usage: python3 wheelname.py <package-name> <version-string>") + raise ValueError(HEADER + ": package name and version string must be non-empty.") + + pkg_name = args[1].strip() + version_str = args[2].strip() + print(get_wheel_names(pkg_name, version_str)) -- GitLab From 9eb74472bebb18f6280492a69ce0a75b9f5673d6 Mon Sep 17 00:00:00 2001 From: AlQuemist <alquemist@Lyriks> Date: Mon, 15 Apr 2024 15:57:38 +0200 Subject: [PATCH 5/8] MakePythonWheel: obtain the proper names for the Python wheels --- cmake/multipython/MakePythonWheel.cmake | 27 +++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/cmake/multipython/MakePythonWheel.cmake b/cmake/multipython/MakePythonWheel.cmake index 031e6d411d8..f66e3cf455e 100644 --- a/cmake/multipython/MakePythonWheel.cmake +++ b/cmake/multipython/MakePythonWheel.cmake @@ -57,7 +57,24 @@ configure_file(${CMAKE_SOURCE_DIR}/CITATION ${PY_OUTPUT_DIR} COPYONLY) # dummy C extension for the Python package # NOTE: Dummy C extension is used merely to set the correct wheel tag according to PEP 425 -configure_file("${WHEEL_ROOT_DIR}/src/bornagain_c_ext_dummy.c" ${BA_PY_SOURCE_OUTPUT_DIR} COPYONLY) +# determine the full wheel-name +execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/multipython/wheelname.py + ${CMAKE_PROJECT_NAME} ${CMAKE_PROJECT_VERSION} + OUTPUT_VARIABLE _BA_WHEEL_NAMES + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +# name for a pure-Python wheel, eg. 'BA-3.2.1-py3-none-any' +list(GET _BA_WHEEL_NAMES 0 BA_PURE_WHEEL_NAME) +set(BA_PY_PURE_WHEEL_NAME "${BA_PURE_WHEEL_NAME}.whl") + +# name for a platform-dependent Python wheel; eg., 'BA-3.2.1-cp311-cp311-linux_x86_64' +list(GET _BA_WHEEL_NAMES 1 BA_WHEEL_NAME) +set(BA_PY_WHEEL_NAME "${BA_WHEEL_NAME}.whl") + +message(STATUS "Python pure wheel-name: '${BA_PURE_WHEEL_NAME}'") +message(STATUS "Python platform-dependent wheel-name: '${BA_WHEEL_NAME}'") # store init files for the Python package @@ -100,7 +117,7 @@ if(APPLE) add_custom_target(BAPyWheel ALL COMMENT "${header} Script to build the Python wheel: " - "'${${BUILD_VAR_DIR}/mk_pypack_macos.zsh}'" + "'${BUILD_VAR_DIR}/mac_py_package.py'" ) elseif(LINUX) # On Linux, building the Python packages needs further effort @@ -110,12 +127,14 @@ elseif(LINUX) ${BUILD_VAR_DIR}/mk_wheel_multilinux.sh @ONLY) add_custom_target(BAPyWheel ALL COMMENT "${header} Script to build the Python wheel: " - "'${${BUILD_VAR_DIR}/mk_wheel_multilinux.sh}'" + "'${BUILD_VAR_DIR}/mk_wheel_multilinux.sh'" ) else() # MS-Windows add_custom_target(BAPyWheel ALL - COMMAND ${Python3_EXECUTABLE} -m pip wheel ${PY_OUTPUT_DIR} --no-deps --wheel ${WHEEL_DIR} + COMMAND ${Python3_EXECUTABLE} -m pip -v wheel ${PY_OUTPUT_DIR} --no-deps --wheel ${WHEEL_DIR} + COMMAND ${CMAKE_COMMAND} -E rename "${WHEEL_DIR}/${BA_PY_PURE_WHEEL_NAME}" + "${WHEEL_DIR}/${BA_PY_WHEEL_NAME}" COMMENT "${header} Making the Python wheel..." ) endif() -- GitLab From 90f5cbe900fe61947ffa6f474f51e52b1b93e2b1 Mon Sep 17 00:00:00 2001 From: AlQuemist <alquemist@Lyriks> Date: Mon, 22 Apr 2024 18:47:33 +0200 Subject: [PATCH 6/8] Rename the Python wheels to proper names --- devtools/deploy/linux/mk_wheel_multilinux.sh.in | 4 ++++ devtools/deploy/mac/mac_package.py.in | 13 ++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/devtools/deploy/linux/mk_wheel_multilinux.sh.in b/devtools/deploy/linux/mk_wheel_multilinux.sh.in index ac6e2599752..5c986b11686 100755 --- a/devtools/deploy/linux/mk_wheel_multilinux.sh.in +++ b/devtools/deploy/linux/mk_wheel_multilinux.sh.in @@ -24,6 +24,8 @@ pyoutdir="@PY_OUTPUT_DIR@" libdir="@BA_PY_LIBRARY_OUTPUT_DIR@" xlibdir="@BA_PY_EXTRA_LIBRARY_OUTPUT_DIR@" wheeldir="@WHEEL_DIR@" +purewheelname="@BA_PY_PURE_WHEEL_NAME@" +wheelname="@BA_PY_WHEEL_NAME@" PYTHON="@Python3_EXECUTABLE@" # constants @@ -133,6 +135,8 @@ echo # make the Python wheel echo "$TITLE: Making the Python wheel, input directory = ${pyoutdir}" $PYTHON -m pip wheel "$pyoutdir" --no-deps --wheel "$wheeldir/raw" +# rename the wheel to a platform-dependent wheel +mv "$wheeldir/raw/$purewheelname" "$wheeldir/raw/$wheelname" echo #-- Overwrite platform tag with a manylinux tag diff --git a/devtools/deploy/mac/mac_package.py.in b/devtools/deploy/mac/mac_package.py.in index 6606f9cbe47..8e2f370d080 100644 --- a/devtools/deploy/mac/mac_package.py.in +++ b/devtools/deploy/mac/mac_package.py.in @@ -286,10 +286,12 @@ class PythonFramework: return py_fmwk_rpaths, pyversion @staticmethod - def make_wheel(python_exe, py_output_dir, wheel_dir): + def make_wheel(python_exe, py_output_dir, wheel_dir, pure_wheelname, wheelname): # make the Python wheel - subp.check_output([python_exe, '-m', 'pip', 'wheel', py_output_dir, + subp.check_output([python_exe, '-m', 'pip', '-v', 'wheel', py_output_dir, '--no-deps', '--wheel', wheel_dir]) + # rename the pure-Python wheel to a platform-dependent wheel + os.rename(pure_wheelname, wheelname) class dylibRPATHs: """ Obtains the RPATHs of a library """ @@ -735,6 +737,8 @@ pkg_py_root_dir = os.path.dirname("@BA_PY_INIT_OUTPUT_DIR@/") pkg_py_lib_dir = os.path.dirname("@BA_PY_LIBRARY_OUTPUT_DIR@/") pkg_py_extra_lib_dir = os.path.dirname("@BA_PY_EXTRA_LIBRARY_OUTPUT_DIR@/") pkg_py_wheel_dir = os.path.dirname("@WHEEL_DIR@/") +pkg_py_pure_wheel_name = os.path.realpath("@WHEEL_DIR@/@BA_PY_PURE_WHEEL_NAME@") +pkg_py_wheel_name = os.path.realpath("@WHEEL_DIR@/@BA_PY_WHEEL_NAME@") # Qt-related variables # eg. input Qt dir = '/usr/local/opt/qt@5/lib/cmake/Qt' @@ -897,7 +901,10 @@ for lib in pkg.dst_tbl.values(): if pkg.python_mode: print("%s: creating Python wheel in '%s'..." % (TITLE, pkg_py_wheel_dir)) print(" Python executable = '%s'" % python_exe) - PythonFramework.make_wheel(python_exe, pkg_py_output_dir, pkg_py_wheel_dir) + print(" rename wheel: '%s' => '%s'" % (pkg_py_pure_wheel_name, pkg_py_wheel_name)) + + PythonFramework.make_wheel(python_exe, pkg_py_output_dir, + pkg_py_wheel_dir, pkg_py_pure_wheel_name, pkg_py_wheel_name) #-- END print("\n%s: Done.\n" % TITLE) -- GitLab From 77ad3c37505ccfada08ff9fa7b5b0a140f34b4a7 Mon Sep 17 00:00:00 2001 From: AlQuemist <alquemist@Lyriks> Date: Mon, 22 Apr 2024 14:38:59 +0200 Subject: [PATCH 7/8] rm dummy C code used for making the Python wheel --- Wrap/Python/setup.py.in | 12 ++---------- Wrap/Python/src/bornagain_c_ext_dummy.c | 10 ---------- cmake/multipython/MakePythonWheel.cmake | 3 --- 3 files changed, 2 insertions(+), 23 deletions(-) delete mode 100644 Wrap/Python/src/bornagain_c_ext_dummy.c diff --git a/Wrap/Python/setup.py.in b/Wrap/Python/setup.py.in index 56df5619ba6..9c7336b4c96 100644 --- a/Wrap/Python/setup.py.in +++ b/Wrap/Python/setup.py.in @@ -17,13 +17,5 @@ if __name__ == "__main__": '@PY_OUTPUT_DIR@/src/bornagain/lib/extra/*.dll', '@PY_OUTPUT_DIR@/src/bornagain/lib/extra/*.pyd', '@PY_OUTPUT_DIR@/src/bornagain/lib/extra/*.dylib', - ]}, - # dummy extension to set the correct wheel tag - # according to PEP 425 - ext_modules=[ - setuptools.Extension( - name="bornagain.lib.__bornagain_dummy_c_ext", - sources=["@WHEEL_C_EXT@"], - include_dirs=["@WHEEL_C_EXT_INC_DIRS@"], - library_dirs=["@WHEEL_C_EXT_LIB_DIRS@"]) - ]) + ]} +) diff --git a/Wrap/Python/src/bornagain_c_ext_dummy.c b/Wrap/Python/src/bornagain_c_ext_dummy.c deleted file mode 100644 index 506d883d4e1..00000000000 --- a/Wrap/Python/src/bornagain_c_ext_dummy.c +++ /dev/null @@ -1,10 +0,0 @@ -// dummy C extension to set the correct wheel tag according to PEP 425 -//*** Please DO NOT change the contents ***// - -#include <Python.h> - -PyMODINIT_FUNC PyInit___bornagain_dummy_c_ext(void) -{ - return NULL; -} -// END diff --git a/cmake/multipython/MakePythonWheel.cmake b/cmake/multipython/MakePythonWheel.cmake index f66e3cf455e..209d2a50929 100644 --- a/cmake/multipython/MakePythonWheel.cmake +++ b/cmake/multipython/MakePythonWheel.cmake @@ -16,9 +16,6 @@ set(header "Python wheel (${Python3_VERSION}):") set(WHEEL_ROOT_DIR "${CMAKE_SOURCE_DIR}/Wrap/Python") -set(WHEEL_C_EXT_INC_DIRS "${Python3_INCLUDE_DIRS}") -set(WHEEL_C_EXT_LIB_DIRS "${Python3_LIBRARY_DIRS}") - # output directories: set(PY_OUTPUT_DIR "${CMAKE_BINARY_DIR}/py") -- GitLab From 599b62c280f4aa1555d01fa4996dc80836ecfa07 Mon Sep 17 00:00:00 2001 From: AlQuemist <alquemist@Lyriks> Date: Mon, 22 Apr 2024 17:15:43 +0200 Subject: [PATCH 8/8] restore MANIFEST.in, seemingly needed by pip under Windows to include binaries See <https://stackoverflow.com/a/14159430> --- Wrap/Python/MANIFEST.in | 6 ++++++ Wrap/Python/setup.py.in | 17 +---------------- cmake/multipython/MakePythonWheel.cmake | 5 +++-- 3 files changed, 10 insertions(+), 18 deletions(-) create mode 100644 Wrap/Python/MANIFEST.in diff --git a/Wrap/Python/MANIFEST.in b/Wrap/Python/MANIFEST.in new file mode 100644 index 00000000000..af1d8c3d9a2 --- /dev/null +++ b/Wrap/Python/MANIFEST.in @@ -0,0 +1,6 @@ +# include extra descriptors +include CITATION +# include shared libraries +recursive-include src/bornagain/lib *.so *.so.* *.lib *.dll *.pyd *.dylib +# exclude compiled and debug files +global-exclude *.py[co] __pycache__ diff --git a/Wrap/Python/setup.py.in b/Wrap/Python/setup.py.in index 9c7336b4c96..d7601fd2543 100644 --- a/Wrap/Python/setup.py.in +++ b/Wrap/Python/setup.py.in @@ -3,19 +3,4 @@ import setuptools if __name__ == "__main__": - setuptools.setup( - package_data={'bornagain': ['@PY_OUTPUT_DIR@/CITATION', - '@PY_OUTPUT_DIR@/src/bornagain/lib/*.so', - '@PY_OUTPUT_DIR@/src/bornagain/lib/*.so.*', - '@PY_OUTPUT_DIR@/src/bornagain/lib/*.lib', - '@PY_OUTPUT_DIR@/src/bornagain/lib/*.dll', - '@PY_OUTPUT_DIR@/src/bornagain/lib/*.pyd', - '@PY_OUTPUT_DIR@/src/bornagain/lib/*.dylib', - '@PY_OUTPUT_DIR@/src/bornagain/lib/extra/*.so', - '@PY_OUTPUT_DIR@/src/bornagain/lib/extra/*.so.*', - '@PY_OUTPUT_DIR@/src/bornagain/lib/extra/*.lib', - '@PY_OUTPUT_DIR@/src/bornagain/lib/extra/*.dll', - '@PY_OUTPUT_DIR@/src/bornagain/lib/extra/*.pyd', - '@PY_OUTPUT_DIR@/src/bornagain/lib/extra/*.dylib', - ]} -) + setuptools.setup() diff --git a/cmake/multipython/MakePythonWheel.cmake b/cmake/multipython/MakePythonWheel.cmake index 209d2a50929..746ab80acc9 100644 --- a/cmake/multipython/MakePythonWheel.cmake +++ b/cmake/multipython/MakePythonWheel.cmake @@ -37,20 +37,21 @@ file(MAKE_DIRECTORY ) # configure setup files - set(_setup_cfg ${PY_OUTPUT_DIR}/setup.cfg) set(_setup_py ${PY_OUTPUT_DIR}/setup.py) configure_file(${WHEEL_ROOT_DIR}/setup.cfg.in ${_setup_cfg} @ONLY) configure_file(${WHEEL_ROOT_DIR}/setup.py.in ${_setup_py} @ONLY) configure_file(${WHEEL_ROOT_DIR}/pyproject.toml ${PY_OUTPUT_DIR} COPYONLY) +# MANIFEST.in is needed for inclusion of binary files under Windows +configure_file(${WHEEL_ROOT_DIR}/MANIFEST.in ${PY_OUTPUT_DIR} COPYONLY) # copy text files - configure_file(${CMAKE_SOURCE_DIR}/README.md ${PY_OUTPUT_DIR} COPYONLY) configure_file(${CMAKE_SOURCE_DIR}/COPYING ${PY_OUTPUT_DIR} COPYONLY) configure_file(${CMAKE_SOURCE_DIR}/CITATION ${PY_OUTPUT_DIR} COPYONLY) + # dummy C extension for the Python package # NOTE: Dummy C extension is used merely to set the correct wheel tag according to PEP 425 -- GitLab