Skip to content
Snippets Groups Projects

Migrate to pyenv Python Platform; Fix MacOS package (Major change)

Merged Migrate to pyenv Python Platform; Fix MacOS package (Major change)
fixMacPackage into r21.1
Merged Ammar Nejati requested to merge fixMacPackage into r21.1
Viewing commit 860ee51c
Show latest version
1 file
+ 0
− 543
Compare changes
  • Side-by-side
  • Inline
devtools/deploy/mac/mk_mac_package.py deleted 100644 → 0
+ 0
− 543
"""
Build BornAgain with multiple versions of Python
"""
import sys, os, shutil, glob, platform, re, subprocess as subp
from collections import namedtuple
def mkdir(path:str):
if not os.path.exists(path):
os.makedirs(path)
def removeprefix(str_:str, pfx:str):
return str_.split(pfx, 1)[-1]
def splat(str_list:str, separator = ' '):
return [x for x in str_list.split(separator) if x]
def copyfile(src:str, dst:str, overwrite=False):
if not overwrite and os.path.exists(dst): return
shutil.copy(src, dst)
def dylib_id(libpath:str):
""" return the base library id for the given library """
return subp.check_output(["otool", "-XD", libpath], encoding='utf-8')
def rm_list_duplicates(list0:list):
""" remove duplicates from a given list """
return list(set(list0))
def dylib_deps(libpath:str):
""" use otool to get the 1st-order dependencies of a library (raw output) """
# obtain the dependencies for a given library;
# remove the name of the library itself from the references.
# Under MacOS, a Mach-O binary sometimes depends on itself.
# otool output example: ' /usr/local/opt/foo.dylib (compatibility ...)'
basename = os.path.basename(libpath)
basename_rx = re.compile(basename)
deps = subp.check_output(["otool", "-XL", libpath], encoding='utf-8').split('\n')
return [d.strip() for d in deps if d and not basename_rx.search(d)]
def find_dylibs(abspath:str):
""" return the filenames corresponding to the pattern *.so or *.dylib """
return glob.glob(abspath + '/*.so') + glob.glob(abspath + '/*.dylib')
def find_common_root(abspath1:str, abspath2:str):
""" find the longest common root of two given _absolute_ paths """
if abspath1 == abspath2:
# if paths are equal, the root is equal to either of them
return abspath1
# convert paths to arrays of directories:
# replace '/' with blank and make an array out of the result;
# eg. '/root/lib/opt' => [root, lib, opt]
dirs1 = abspath1.split('/')
idxMax1 = len(dirs1)
dirs2 = abspath2.split('/')
idxMax2 = len(dirs2)
common_root = ""
for idx in range(min(idxMax1, idxMax2)):
# extract the head (topmost) directory name from paths
# eg. 'root/lib/opt' => 'root'
head1 = dirs1[idx]
head2 = dirs2[idx]
# if both paths have the same head, then add head to the common root;
# otherwise, the longest common root is already obtained
if head1 != head2: break
# add the common head to the root
common_root += '/' + head1
# return the longest common root
return common_root
def find_rpath(bin_abspath:str, lib_abspath:str, lib_relpath:str):
""" find the proper rpath for given binary pointing to a reference library
# usage: find_rpath(bin_abspath, lib_abspath, lib_relpath)
# example:
# bin_abspath='/root/usr/opt/bin'
# lib_abspath='/root/usr/Frameworks/Qux/lib'
# lib_relpath='Qux/lib'
# returns: `../../Frameworks`
"""
# drop the final '/' chars from all paths
bin_abspath = os.path.dirname(bin_abspath) # target binary for which a rpath is obtained
lib_abspath = os.path.dirname(lib_abspath) # referenced library
lib_relpath = os.path.dirname(lib_relpath) # relative path to the referenced library
# find the longest common root path
root_path = find_common_root(bin_abspath, lib_abspath)
root_path = os.path.dirname(root_path)
# obtain the path from the binary to the root
# eg. '/root/local/opt' => 'local/opt' => '../..'
binpth_to_root = re.sub(root_path + '(/|$)', '', bin_abspath)
binpth_to_root = re.sub('[^/]+', '..', binpth_to_root)
# obtain the path from root to the referenced library;
# eg. '/root/local/opt' => 'local/opt'
# then, drop the relative path of the library from the end
libpth_from_root = re.sub(root_path + '(/|$)', lib_abspath, '')
libpth_from_root = re.sub('(^|/)' + lib_relpath, libpth_from_root, '')
# return the proper relative RPATH to the referenced library
# eg. '../../Frameworks/Qt'
if not binpth_to_root:
libpth_from_bin = libpth_from_root
else:
libpth_from_bin = binpth_to_root + '/' + libpth_from_root
return libpth_from_bin
def get_depends1(libpath:str):
""" get 1st-order dependencies for a given file """
# obtain the 'non-system' dependencies for a given library
# eg. ' /usr/local/opt/foo.1.dylib (compatibility ...)' => '/usr/local/opt/foo.dylib'
# system dependencies pattern: /System/, /Library/, /usr/lib/, /usr/local/lib
deps_ = [d for d in dylib_deps(libpath)
if not re.match(r'/(usr/lib|usr/local/lib|System|Library)/', d.strip())]
deps = list()
for d in deps_:
# extract library path
m_ = re.match('\s*([\w@/\.]+)\s+.*', d)
if m_: deps.append(m_.group(1))
return deps
def get_python_dependence(dependencies:list) -> (str, str):
""" extract the Python dependency from the given dependencies
# NOTE: Under OSX, Python's library name is either 'Python', or
# for Python3.9, 'libpython3.9.dylib'.
"""
# regexp to extract the Python dependence;
# eg., '/Frameworks/Python.framework/Versions/3.9/Python'
libdir_re = r'[\w\./@-]+'
pylibname_re = r'Python|libpython.+\.dylib'
py_fmwk_re = re.compile('(' + libdir_re + ')/(' + pylibname_re + ')')
# regexp to correct the Python dependence; eg.:
# '/Users/usr1/.pyenv/versions/3.9.18/lib/libpython3.9.dylib' => 'libpython3.9.dylib'
# '/Frameworks/Python.framework/Versions/3.9/Python' => 'libpython3.9.dylib'
# '/Frameworks/Python.framework/Versions/3.9/libpython3.9.dylib' => 'libpython3.9.dylib'
# obtain the dependencies
pydepends_path_list = [d for d in dependencies if py_fmwk_re.match(d)]
pydepends_path = pydepends_filename = None
if pydepends_path_list:
pydepends_filename = pydepends_path_list[0]
mtch = py_fmwk_re.match(pydepends_filename)
pydepends_path = mtch[1]
pydepends_filename = mtch[2]
# return the Python dependence fullpath and filename
return (pydepends_path, pydepends_filename)
def get_python_framework_path(pydepends_path:str):
""" produce proper Python framework paths for a given Python dependency """
pydepends_path = pydepends_path.lower()
# regexp to extract the Python framework path
# eg. '/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/Python'
# => '/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/'
py_fmwk_path_re = r'(.*)/(Python|libpython).*'
py_fmwk_dir = re.match(py_fmwk_path_re, pydepends_path)[1]
# when the library is like '.../Versions/3.9/Python', then
# add an extra '/lib' to the framework path;
# this is needed since it refers to the standard location of the
# Python shared library on OSX, '.../Versions/3.9/lib/libpython39.dylib'.
if pydepends_path.endswith('Python'):
py_fmwk_dir += '/lib'
# regexp to extract the Python version; eg. '3.9'
pyversion = None
pyversion_m = re.match(r'.+versions/([0-9.]+).*', pydepends_path)
if pyversion_m:
pyversion = pyversion_m[1]
else:
pyversion_m = re.match(r'.+/libpython(.+)\.dylib', pydepends_path)
if pyversion_m:
pyversion = pyversion_m[1]
# '3.9.18' => '3.9'
pyversion = re.match(r'(\d+.\d+).*', pyversion)[1]
if not pyversion:
raise ValueError("Cannot extract Python version from path '%s'"
% pydepends_path)
# RPATHs corresponding to the common OSX framework paths
py_fmwk_basepath = "Python.framework/Versions/" + pyversion + "/lib"
framework_paths = ("/usr/local/Library/Frameworks",
"/Library/Frameworks", "/usr/local/Frameworks",
"/opt/homebrew/Frameworks")
# collect proper RPATHs for Python framework
py_fmwk_rpaths = [py_fmwk_dir]
for pth in framework_paths:
py_fmwk_rpaths.append(pth + '/' + py_fmwk_basepath)
# return a list of possible framework paths
return py_fmwk_rpaths
def find_dependencies(libraries, LEVELMAX = 10):
""" gather all dependencies for the given initial libraries
up to a maximum (but arbitrary) level.
"""
print("%s: Find dependencies (up to level %s)..." % (TITLE, LEVELMAX))
# NOTE: Sets are used to keep a list of entries without repetition
all_deps = set() # absolute dependencies
libs_lv = set(libraries) # libraries at the current level
libs_chk = set() # libraries for which the dependencies have been found
level = 0 # current level nr.
while libs_lv:
level += 1
# avoid going infinitely deep (eg. due to some mistake)
if level > LEVELMAX:
print("Error: Dependency level %i exceeds the maximum allowed depth (%i)."
% (level, LEVELMAX))
break
# eg. at level 3, print '==>[L3]'
print("==>[L%i]" % level)
# obtain all dependencies at the current level
abs_deps_lv = set() # _absolute_ dependencies at the current level
for lib in libs_lv:
# neglect previously-observed libraries
if lib in all_deps: continue
print("[L%i] %s" % (level, lib))
libs_chk.add(lib)
for dep in get_depends1(lib):
# add dependency reference (relative/absolute) to list of dependencies
all_deps.add(dep)
# relative dependencies which begin with '@'
# eg. '@rpath/foo.dylib'
if dep.startswith('@'):
# abs. path to relative dependencies is not known, therefore
# they should not be added to the current abs. dependencies
continue
# add dependency to the current abs. dependencies
abs_deps_lv.add(dep)
# libraries for next level are the absolute dependencies at current level
libs_lv = abs_deps_lv
# return all discovered dependencies
return all_deps
def install_qt_plugins(qt_plugins:list, pkg_root:str,
qt_framework_root:str, qt_plugins_rel_dir:str,
pkgbindir:dict):
""" install the required Qt plugins """
print("%s: Copy required Qt plugins from '%s':" % (TITLE, qt_framework_root))
pkg_plugins_dir = pkg_root + '/' + pkgbindir['FW_qt_plugin']
mkdir(pkg_plugins_dir)
plugins = []
for plg in qt_plugins:
# full path of the plugin;
# eg. '/opt/qt@5/plugins/platforms/libqcocoa.dylib'
plgpth0 = qt_framework_root + '/' + qt_plugins_rel_dir + '/' + plg
# copy the plugin to the same directory under the _package_ plugins dir;
# eg. '<Package>/PlugIns/platforms/libqcocoa.dylib'
pth = pkg_plugins_dir + '/' + plg
pth_dir = os.path.dirname(pth)
mkdir(pth_dir)
copyfile(plgpth0, pth)
# add Qt plugin to the list of initial binaries
plugins.append(pth)
return plugins
#========================================
_extra_libs="@MACPKG_EXTRA_LIBS@"
pkg_root = sys.argv[1] # root dir, eg. '/tmp/bornagain.app/Contents'
main_exe = pkg_root + "/MacOS/bornagain"
extra_libs = splat(_extra_libs) # TODO: use ';' as splitting
# eg. input Qt dir = '/usr/local/opt/qt@5/lib/cmake/Qt'
# => '/usr/local/opt/qt@5'
qtdir = "@Qt_DIR@"
# Qt plugins paths relative to Qt root dir
qt_plugins_rel_dir = "@MACPKG_QT_PLUGINS_RELDIR@"
qt_framework_root = qtdir
# list of required Qt plugins
_qt_plugins = "@MACPKG_QT_PLUGINS@"
qt_plugins_dir = "@MACPKG_QT_PLUGINS_DIR@"
qt_plugins = splat(_qt_plugins) # TODO: use ';' as splitting
TITLE="* MacOS Package"
if not pkg_root:
raise ValueError("%s: Error: Provide the root directory of the package." % TITLE)
if not qt_framework_root:
raise ValueError("%s: Error: Provide the root directory of the Qt framework." % TITLE)
if not main_exe:
raise ValueError("%s: Error: Main executable does not exist ($main_exe)." % TITLE)
print("%s: package root = '%s'" % (TITLE, pkg_root))
print("%s: main executable = '%s'" % (TITLE, main_exe))
print("%s: Qt framework root = '%s'" % (TITLE, qt_framework_root))
#-- directories (relative to root) which define the package structure
fwdir = "Frameworks"
pkgbindir = {
"lib": "lib", # main library dir
"exlib": "Library", # external libraries dir
"FW": fwdir, # frameworks dir
"FW_qt": fwdir, # Qt framework dir
"FW_qt_plugin": qt_plugins_dir # Qt plugins dir
}
#-- copy extra libraries to the Framework-libraries dir;
# the name of the libraries must be the same as the library ids.
if extra_libs:
dst = pkg_root + '/' + pkgbindir['exlib']
print("%s: Copy extra libraries to '%s':" % (TITLE, dst))
mkdir(dst)
for lib in extra_libs:
# eg. 'usr/local/opt/libA.9.dylib' -> '<Package>/Library/libA.9.dylib'
fnm = dylib_id(lib)
copyfile(lib, dst + '/' + os.path.basename(fnm))
#-- collect a list of binaries which are already placed in the package
libs_init = [main_exe] \
+ find_dylibs(pkg_root + '/' + pkgbindir['lib']) \
+ find_dylibs(pkg_root + '/' + pkgbindir['exlib'])
qtplugins = install_qt_plugins(qt_plugins, pkg_root,
qt_framework_root, qt_plugins_rel_dir, pkgbindir)
libs_init.extend(qtplugins)
print("%s: Initially installed binaries under '%s':" % (TITLE, pkg_root))
for fnm in libs_init:
# eg., '+ lib/libA.dylib'
print(" + '%s'" % removeprefix(fnm, pkg_root + '/'))
#-- find the dependencies of the binaries
refs_all = find_dependencies(libs_init)
print("%s: All dependencies:" % TITLE)
# a sorted list of dependencies
for lib in sorted(refs_all):
print(" + '%s'" % lib)
#-- distinguish absolute and relative references within dependencies
abs_refs = []
rel_refs = []
py_refs = []
fw_refs = []
for ref in refs_all:
_ref = ref.lower()
if 'python' in _ref and 'boost' not in _ref:
# Python dependencies must be considered separately (exclude 'libboost_python')
# eg. '/opt/py@3.9/Frameworks/Python.framework/Versions/3.9/Python'
py_refs.append(ref)
elif _ref.startswith('@'):
# relative reference; eg. '@rpath/foo.dylib'
rel_refs.append(ref)
elif '.framework/' in _ref:
# Frameworks must be considered separately
# eg. '/opt/qt@5/lib/QtGui.framework/Versions/5/QtGui'
fw_refs.append(ref)
else:
# absolute reference; eg. '/usr/opt/libA.so'
abs_refs.append(ref)
#-- copy all absolute dependencies to the package
dst = pkg_root + '/' + pkgbindir['exlib']
mkdir(dst)
print("%s: Copy external libraries to '%s':" % (TITLE, dst))
# pkglib: map{ library reference => destination full-path }
# e.g., /usr/local/opt/libA.dylib => /pkg-root/Library/libA.dylib
pkglib = dict()
for ref in abs_refs:
pth = dst + '/' + os.path.basename(ref) # destination full-path
pkglib[ref] = pth
copyfile(ref, pth)
#-- copy all framework dependencies to the package
# Qt framework
qt_fwdir = pkg_root + '/' + pkgbindir['FW_qt']
mkdir(qt_fwdir)
print("%s: Copy Qt-framework libraries to '%s':" % (TITLE, qt_fwdir))
# extract framework path:
# eg. '/usr/local/opt/qt@5/lib/QtWidgets.framework/Versions/5/QtWidgets (...)'
# => 'QtWidgets.framework/Versions/5/QtWidgets'
framework_re = re.compile(r'.+/([^/]+\.[Ff]ramework/[^\s]+).*')
for ref in fw_refs:
# only Qt framework is considered
if os.path.basename(ref).startswith('Qt'):
# eg., copy '/opt/qt@5/lib/QtGui.framework/Versions/5/QtGui'
# to '<Frameworks>/Qt/QtGui.framework/Versions/5/QtGui'
qtfwdir0 = framework_re.match(ref).group(1)
pth = qt_fwdir + '/' + qtfwdir0
mkdir(os.path.dirname(pth))
pkglib[ref] = pth
copyfile(ref, pth)
else:
print("Framework '%s' neglected." % ref)
# Add relatively-referenced Qt framework;
# e.g., QtDBus framework referenced as `@rpath/QtDBus.framework/Versions/A/QtDBus`
print("%s: Add relatively-referenced Qt framework libraries:" % TITLE)
qt_exlib_dir = pkg_root + '/' + pkgbindir['exlib']
for ref in rel_refs:
# select only Qt relative references
if os.path.basename(ref).startswith('Qt'):
# '@rpath/QtDBus.framework/Versions/A/QtDBus' => 'QtDBus.framework/Versions/A/QtDBus'
ref_qt = removeprefix(ref, '@rpath/')
qt_src = qt_framework_root + '/lib/' + ref_qt
qt_dst = qt_fwdir + '/' + ref_qt
qt_dst_dir = os.dirname(qt_dst)
pkglib["REL-" + qt_dst] = qt_dst
mkdir(qt_dst_dir)
copyfile(qt_src, qt_dst)
# add the *1st-order optional* dependencies of the Qt library (if any)
# to the same folder as that of the Qt library
# deps1 = $(dylib_deps "$qt_dst" | sed -nE "s;[[:blank:]]*(/.+/opt/.+)[[:blank:]]\(.+;\1;p")
has_deps = 0
for libpth_ in deps1:
has_deps = 1
# '/usr/local/opt/libA.1.2.dylib' => 'libA.1.2.dylib'
libname_ = os.path.basename(libpth_)
# copy the library to the external library dir of the package
libdst = qt_exlib_dir + '/' + libname_
pkglib[libpth_] = libdst
copyfile(libpth_, libdst)
#-- collect all package binaries for later process
pkgbins = sorted(rm_list_duplicates([main_exe] + list(pkglib.values()) + libs_init))
print("%s: All binaries:" % TITLE)
# a sorted list of binaries
for lib in pkgbins:
print(" + '" + removeprefix(lib, pkg_root + '/'))
#-- adjust references to libraries
print("%s: Adjust references to libraries:" % TITLE)
libdir = pkg_root + '/' + pkgbindir['lib']
rpaths = dict()
bin_deps = dict()
for bin in pkgbins:
rpaths_tmp = set()
bin_deps[bin] = set(dylib_deps(bin))
bindir = os.path.dirname(bin) # abs. dir of target binary
# abspth0 = original abs. full-path of the library
# abspth_pkg = abs. full-path of the library in the package
for abspth0, abspth_pkg in pkglib.items():
# if the binary does not depend on the current library, do nothing
if not abspth0 in bin_deps[bin]: continue
# change the library reference in the binary
# eg. '/usr/local/opt/lib/libA.5.dylib' => '@rpath/libA.5.dylib'
libname = os.path.basename(abspth_pkg) # library filename
libdir = os.path.dirname(abspth_pkg) # abs. dir of the referenced library
librelpth = "" # rel. path of the library
# make a framework-relative path for the Qt framework libraries
# eg. '/opt/qt@5/lib/QtGui.framework/Versions/5/QtGui'
# => 'QtGui.framework/Versions/5'
if libname.startswith('Qt'):
# rm framework root dir from the beginning;
# eg. '/opt/qt@5/lib/QtGui.framework/Versions/5/QtGui'
# => 'QtGui.framework/Versions/5/QtGui'
librelpth = removeprefix(abspth_pkg, qt_fwdir + '/')
# rm filename from the end
# eg. 'QtGui.framework/Versions/5/QtGui'
# => 'QtGui.framework/Versions/5'
librelpth = os.path.dirname(librelpth)
ref_new = libname
# prepend with library rel. path, if any
if not librelpth:
ref_new = librelpth + '/' + ref_new
subp.check_output(["install_name_tool", bin, "-change",
abspath0, "@rpath/" + ref_new], encoding='utf-8')
# make a proper RPATH to refer to the library within the package
# eg. '@loader_path/../Frameworks/Qt/'
rpath = "@loader_path/" + find_rpath(bindir, libdir, librelpth)
rpaths_tmp.add(rpath.strip())
# store a duplicateless list of rpaths needed for the binary,
# only if some rpaths are set.
# NOTE: libraries under the package lib dir. often need
# an extra RPATHs '../Library'.
if bin.startswith(libdir + '/'):
rpaths_tmp.add('@loader_path/../' + pkgbindir['exlib'])
if rpaths_tmp:
rpaths[bin] = rpaths_tmp
# find the Python dependence for the *main* libraries
libs_main = find_dylibs(pkg_root + '/' + pkgbindir['lib'])
libs_pydep = list() # list of Python-dependent libraries
for lib in libs_main:
# get the first element the list Python dependences (all others must be the same)
_pydep = get_python_dependence(bin_deps[lib])
# record the list of Python-dependent libraries
if not _pydep[1]:
libs_pydep.append(lib)
# NOTE: zsh array indexing starts at 1 (unless option KSH_ARRAYS is set)
if not _pydep[1]:
raise RuntimeError("No Python dependence found in the binaries.")
pydepends_path=_pydep[0]
pydepends_filename=_pydep[1]
pydepends_fullpath = pydepends_path + '/' + pydepends_filename
# relative Python reference; e.g. '@rpath/libpython3.11.dylib'
if pydepends_path.startswith('@'):
py_fmwk_rpaths = get_python_framework_path_rel(pydepends_fullpath)
else:
py_fmwk_rpaths = get_python_framework_path(pydepends_fullpath)
for lib in libs_pydep:
rpaths[lib].extend(py_fmwk_rpaths)
print("%s: Python dependence for the main libraries in '%s/%s':"
% (TITLE, pkg_root, pkgbindir['lib']))
print(" + path: '%s'" % pydepends_path)
print(" + library: '%s'" % pydepends_filename)
print(" + framework paths:")
for pth in py_fmwk_rpaths:
print(" - '%s'" % pth)
print("%s: Add proper RPATHs to the binaries:" % TITLE)
for bin, rpaths_bin in rpaths.items():
# eg. RPATHS for 'lib/libA.dylib': ../Library , ../Frameworks/Qt
if not rpaths_bin: continue
# eg. install_name_tool libA.so -add_rpath RPATH1 -add_rpath RPATH2 ...
cmd_list = ["install_name_tool", bin]
for cmd in [["-add_rpath", r] for r in rpaths_bin]:
cmd_list.extend(cmd)
out = subp.check_output(cmd_list, encoding='utf-8')
print("%s: Done." % TITLE)