diff --git a/devtools/deploy/mac/mk_mac_package.py b/devtools/deploy/mac/mk_mac_package.py new file mode 100644 index 0000000000000000000000000000000000000000..c4654ca3d1d3cdf4154acc031063f462cb286348 --- /dev/null +++ b/devtools/deploy/mac/mk_mac_package.py @@ -0,0 +1,543 @@ +""" +Build BornAgain with multiple versions of Python +""" + +import sys, os, shutil, glob, platform, re, subprocess as subp +from collections import namedtuple + +def mkdir(path:str): + if not os.path.exists(path): + os.makedirs(path) + +def removeprefix(str_:str, pfx:str): + return str_.split(pfx, 1)[-1] + +def splat(str_list:str, separator = ' '): + return [x for x in str_list.split(separator) if x] + +def copyfile(src:str, dst:str, overwrite=False): + if not overwrite and os.path.exists(dst): return + shutil.copy(src, dst) + +def dylib_id(libpath:str): + """ return the base library id for the given library """ + return subp.check_output(["otool", "-XD", libpath], encoding='utf-8') + +def rm_list_duplicates(list0:list): + """ remove duplicates from a given list """ + return list(set(list0)) + +def dylib_deps(libpath:str): + """ use otool to get the 1st-order dependencies of a library (raw output) """ + # obtain the dependencies for a given library; + # remove the name of the library itself from the references. + # Under MacOS, a Mach-O binary sometimes depends on itself. + # otool output example: ' /usr/local/opt/foo.dylib (compatibility ...)' + basename = os.path.basename(libpath) + basename_rx = re.compile(basename) + deps = subp.check_output(["otool", "-XL", libpath], encoding='utf-8').split('\n') + return [d.strip() for d in deps if d and not basename_rx.search(d)] + +def find_dylibs(abspath:str): + """ return the filenames corresponding to the pattern *.so or *.dylib """ + return glob.glob(abspath + '/*.so') + glob.glob(abspath + '/*.dylib') + +def find_common_root(abspath1:str, abspath2:str): + """ find the longest common root of two given _absolute_ paths """ + if abspath1 == abspath2: + # if paths are equal, the root is equal to either of them + return abspath1 + + # convert paths to arrays of directories: + # replace '/' with blank and make an array out of the result; + # eg. '/root/lib/opt' => [root, lib, opt] + dirs1 = abspath1.split('/') + idxMax1 = len(dirs1) + dirs2 = abspath2.split('/') + idxMax2 = len(dirs2) + common_root = "" + for idx in range(min(idxMax1, idxMax2)): + # extract the head (topmost) directory name from paths + # eg. 'root/lib/opt' => 'root' + head1 = dirs1[idx] + head2 = dirs2[idx] + # if both paths have the same head, then add head to the common root; + # otherwise, the longest common root is already obtained + if head1 != head2: break + # add the common head to the root + common_root += '/' + head1 + + # return the longest common root + return common_root + +def find_rpath(bin_abspath:str, lib_abspath:str, lib_relpath:str): + """ find the proper rpath for given binary pointing to a reference library + # usage: find_rpath(bin_abspath, lib_abspath, lib_relpath) + # example: + # bin_abspath='/root/usr/opt/bin' + # lib_abspath='/root/usr/Frameworks/Qux/lib' + # lib_relpath='Qux/lib' + # returns: `../../Frameworks` + """ + # drop the final '/' chars from all paths + bin_abspath = os.path.dirname(bin_abspath) # target binary for which a rpath is obtained + lib_abspath = os.path.dirname(lib_abspath) # referenced library + lib_relpath = os.path.dirname(lib_relpath) # relative path to the referenced library + # find the longest common root path + root_path = find_common_root(bin_abspath, lib_abspath) + root_path = os.path.dirname(root_path) + # obtain the path from the binary to the root + # eg. '/root/local/opt' => 'local/opt' => '../..' + binpth_to_root = re.sub(root_path + '(/|$)', '', bin_abspath) + binpth_to_root = re.sub('[^/]+', '..', binpth_to_root) + # obtain the path from root to the referenced library; + # eg. '/root/local/opt' => 'local/opt' + # then, drop the relative path of the library from the end + libpth_from_root = re.sub(root_path + '(/|$)', lib_abspath, '') + libpth_from_root = re.sub('(^|/)' + lib_relpath, libpth_from_root, '') + # return the proper relative RPATH to the referenced library + # eg. '../../Frameworks/Qt' + if not binpth_to_root: + libpth_from_bin = libpth_from_root + else: + libpth_from_bin = binpth_to_root + '/' + libpth_from_root + + return libpth_from_bin + +def get_depends1(libpath:str): + """ get 1st-order dependencies for a given file """ + # obtain the 'non-system' dependencies for a given library + # eg. ' /usr/local/opt/foo.1.dylib (compatibility ...)' => '/usr/local/opt/foo.dylib' + # system dependencies pattern: /System/, /Library/, /usr/lib/, /usr/local/lib + deps_ = [d for d in dylib_deps(libpath) + if not re.match(r'/(usr/lib|usr/local/lib|System|Library)/', d.strip())] + + deps = list() + for d in deps_: + # extract library path + m_ = re.match('\s*([\w@/\.]+)\s+.*', d) + if m_: deps.append(m_.group(1)) + + return deps + +def get_python_dependence(dependencies:list) -> (str, str): + """ extract the Python dependency from the given dependencies + # NOTE: Under OSX, Python's library name is either 'Python', or + # for Python3.9, 'libpython3.9.dylib'. + """ + + # regexp to extract the Python dependence; + # eg., '/Frameworks/Python.framework/Versions/3.9/Python' + libdir_re = r'[\w\./@-]+' + pylibname_re = r'Python|libpython.+\.dylib' + py_fmwk_re = re.compile('(' + libdir_re + ')/(' + pylibname_re + ')') + # regexp to correct the Python dependence; eg.: + # '/Users/usr1/.pyenv/versions/3.9.18/lib/libpython3.9.dylib' => 'libpython3.9.dylib' + # '/Frameworks/Python.framework/Versions/3.9/Python' => 'libpython3.9.dylib' + # '/Frameworks/Python.framework/Versions/3.9/libpython3.9.dylib' => 'libpython3.9.dylib' + # obtain the dependencies + pydepends_path_list = [d for d in dependencies if py_fmwk_re.match(d)] + pydepends_path = pydepends_filename = None + if pydepends_path_list: + pydepends_filename = pydepends_path_list[0] + mtch = py_fmwk_re.match(pydepends_filename) + pydepends_path = mtch[1] + pydepends_filename = mtch[2] + + # return the Python dependence fullpath and filename + return (pydepends_path, pydepends_filename) + +def get_python_framework_path(pydepends_path:str): + """ produce proper Python framework paths for a given Python dependency """ + + pydepends_path = pydepends_path.lower() + # regexp to extract the Python framework path + # eg. '/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/Python' + # => '/opt/python@3.9/Frameworks/Python.framework/Versions/3.9/' + py_fmwk_path_re = r'(.*)/(Python|libpython).*' + py_fmwk_dir = re.match(py_fmwk_path_re, pydepends_path)[1] + # when the library is like '.../Versions/3.9/Python', then + # add an extra '/lib' to the framework path; + # this is needed since it refers to the standard location of the + # Python shared library on OSX, '.../Versions/3.9/lib/libpython39.dylib'. + if pydepends_path.endswith('Python'): + py_fmwk_dir += '/lib' + + # regexp to extract the Python version; eg. '3.9' + pyversion = None + pyversion_m = re.match(r'.+versions/([0-9.]+).*', pydepends_path) + if pyversion_m: + pyversion = pyversion_m[1] + else: + pyversion_m = re.match(r'.+/libpython(.+)\.dylib', pydepends_path) + if pyversion_m: + pyversion = pyversion_m[1] + + # '3.9.18' => '3.9' + pyversion = re.match(r'(\d+.\d+).*', pyversion)[1] + if not pyversion: + raise ValueError("Cannot extract Python version from path '%s'" + % pydepends_path) + + # RPATHs corresponding to the common OSX framework paths + py_fmwk_basepath = "Python.framework/Versions/" + pyversion + "/lib" + framework_paths = ("/usr/local/Library/Frameworks", + "/Library/Frameworks", "/usr/local/Frameworks", + "/opt/homebrew/Frameworks") + + # collect proper RPATHs for Python framework + py_fmwk_rpaths = [py_fmwk_dir] + for pth in framework_paths: + py_fmwk_rpaths.append(pth + '/' + py_fmwk_basepath) + + # return a list of possible framework paths + return py_fmwk_rpaths + +def find_dependencies(libraries, LEVELMAX = 10): + """ gather all dependencies for the given initial libraries + up to a maximum (but arbitrary) level. + """ + print("%s: Find dependencies (up to level %s)..." % (TITLE, LEVELMAX)) + # NOTE: Sets are used to keep a list of entries without repetition + all_deps = set() # absolute dependencies + libs_lv = set(libraries) # libraries at the current level + libs_chk = set() # libraries for which the dependencies have been found + level = 0 # current level nr. + + while libs_lv: + level += 1 + # avoid going infinitely deep (eg. due to some mistake) + if level > LEVELMAX: + print("Error: Dependency level %i exceeds the maximum allowed depth (%i)." + % (level, LEVELMAX)) + break + + # eg. at level 3, print '==>[L3]' + print("==>[L%i]" % level) + # obtain all dependencies at the current level + abs_deps_lv = set() # _absolute_ dependencies at the current level + for lib in libs_lv: + # neglect previously-observed libraries + if lib in all_deps: continue + + print("[L%i] %s" % (level, lib)) + libs_chk.add(lib) + for dep in get_depends1(lib): + # add dependency reference (relative/absolute) to list of dependencies + all_deps.add(dep) + # relative dependencies which begin with '@' + # eg. '@rpath/foo.dylib' + if dep.startswith('@'): + # abs. path to relative dependencies is not known, therefore + # they should not be added to the current abs. dependencies + continue + + # add dependency to the current abs. dependencies + abs_deps_lv.add(dep) + + # libraries for next level are the absolute dependencies at current level + libs_lv = abs_deps_lv + + # return all discovered dependencies + return all_deps + +def install_qt_plugins(qt_plugins:list, pkg_root:str, + qt_framework_root:str, qt_plugins_rel_dir:str, + pkgbindir:dict): + """ install the required Qt plugins """ + print("%s: Copy required Qt plugins from '%s':" % (TITLE, qt_framework_root)) + pkg_plugins_dir = pkg_root + '/' + pkgbindir['FW_qt_plugin'] + mkdir(pkg_plugins_dir) + plugins = [] + for plg in qt_plugins: + # full path of the plugin; + # eg. '/opt/qt@5/plugins/platforms/libqcocoa.dylib' + plgpth0 = qt_framework_root + '/' + qt_plugins_rel_dir + '/' + plg + # copy the plugin to the same directory under the _package_ plugins dir; + # eg. '<Package>/PlugIns/platforms/libqcocoa.dylib' + pth = pkg_plugins_dir + '/' + plg + pth_dir = os.path.dirname(pth) + mkdir(pth_dir) + copyfile(plgpth0, pth) + # add Qt plugin to the list of initial binaries + plugins.append(pth) + + return plugins +#======================================== + +_extra_libs="@MACPKG_EXTRA_LIBS@" +pkg_root = sys.argv[1] # root dir, eg. '/tmp/bornagain.app/Contents' +main_exe = pkg_root + "/MacOS/bornagain" +extra_libs = splat(_extra_libs) # TODO: use ';' as splitting + +# eg. input Qt dir = '/usr/local/opt/qt@5/lib/cmake/Qt' +# => '/usr/local/opt/qt@5' +qtdir = "@Qt_DIR@" +# Qt plugins paths relative to Qt root dir +qt_plugins_rel_dir = "@MACPKG_QT_PLUGINS_RELDIR@" +qt_framework_root = qtdir +# list of required Qt plugins +_qt_plugins = "@MACPKG_QT_PLUGINS@" +qt_plugins_dir = "@MACPKG_QT_PLUGINS_DIR@" +qt_plugins = splat(_qt_plugins) # TODO: use ';' as splitting + + +TITLE="* MacOS Package" + +if not pkg_root: + raise ValueError("%s: Error: Provide the root directory of the package." % TITLE) + +if not qt_framework_root: + raise ValueError("%s: Error: Provide the root directory of the Qt framework." % TITLE) + +if not main_exe: + raise ValueError("%s: Error: Main executable does not exist ($main_exe)." % TITLE) + +print("%s: package root = '%s'" % (TITLE, pkg_root)) +print("%s: main executable = '%s'" % (TITLE, main_exe)) +print("%s: Qt framework root = '%s'" % (TITLE, qt_framework_root)) + +#-- directories (relative to root) which define the package structure +fwdir = "Frameworks" +pkgbindir = { + "lib": "lib", # main library dir + "exlib": "Library", # external libraries dir + "FW": fwdir, # frameworks dir + "FW_qt": fwdir, # Qt framework dir + "FW_qt_plugin": qt_plugins_dir # Qt plugins dir +} + +#-- copy extra libraries to the Framework-libraries dir; +# the name of the libraries must be the same as the library ids. + +if extra_libs: + dst = pkg_root + '/' + pkgbindir['exlib'] + print("%s: Copy extra libraries to '%s':" % (TITLE, dst)) + mkdir(dst) + + for lib in extra_libs: + # eg. 'usr/local/opt/libA.9.dylib' -> '<Package>/Library/libA.9.dylib' + fnm = dylib_id(lib) + copyfile(lib, dst + '/' + os.path.basename(fnm)) + +#-- collect a list of binaries which are already placed in the package +libs_init = [main_exe] \ + + find_dylibs(pkg_root + '/' + pkgbindir['lib']) \ + + find_dylibs(pkg_root + '/' + pkgbindir['exlib']) + +qtplugins = install_qt_plugins(qt_plugins, pkg_root, + qt_framework_root, qt_plugins_rel_dir, pkgbindir) +libs_init.extend(qtplugins) + +print("%s: Initially installed binaries under '%s':" % (TITLE, pkg_root)) +for fnm in libs_init: + # eg., '+ lib/libA.dylib' + print(" + '%s'" % removeprefix(fnm, pkg_root + '/')) + +#-- find the dependencies of the binaries +refs_all = find_dependencies(libs_init) + +print("%s: All dependencies:" % TITLE) +# a sorted list of dependencies +for lib in sorted(refs_all): + print(" + '%s'" % lib) + +#-- distinguish absolute and relative references within dependencies +abs_refs = [] +rel_refs = [] +py_refs = [] +fw_refs = [] + +for ref in refs_all: + _ref = ref.lower() + if 'python' in _ref and 'boost' not in _ref: + # Python dependencies must be considered separately (exclude 'libboost_python') + # eg. '/opt/py@3.9/Frameworks/Python.framework/Versions/3.9/Python' + py_refs.append(ref) + elif _ref.startswith('@'): + # relative reference; eg. '@rpath/foo.dylib' + rel_refs.append(ref) + elif '.framework/' in _ref: + # Frameworks must be considered separately + # eg. '/opt/qt@5/lib/QtGui.framework/Versions/5/QtGui' + fw_refs.append(ref) + else: + # absolute reference; eg. '/usr/opt/libA.so' + abs_refs.append(ref) + +#-- copy all absolute dependencies to the package +dst = pkg_root + '/' + pkgbindir['exlib'] +mkdir(dst) +print("%s: Copy external libraries to '%s':" % (TITLE, dst)) +# pkglib: map{ library reference => destination full-path } +# e.g., /usr/local/opt/libA.dylib => /pkg-root/Library/libA.dylib +pkglib = dict() +for ref in abs_refs: + pth = dst + '/' + os.path.basename(ref) # destination full-path + pkglib[ref] = pth + copyfile(ref, pth) + +#-- copy all framework dependencies to the package +# Qt framework +qt_fwdir = pkg_root + '/' + pkgbindir['FW_qt'] +mkdir(qt_fwdir) + +print("%s: Copy Qt-framework libraries to '%s':" % (TITLE, qt_fwdir)) +# extract framework path: +# eg. '/usr/local/opt/qt@5/lib/QtWidgets.framework/Versions/5/QtWidgets (...)' +# => 'QtWidgets.framework/Versions/5/QtWidgets' +framework_re = re.compile(r'.+/([^/]+\.[Ff]ramework/[^\s]+).*') +for ref in fw_refs: + # only Qt framework is considered + if os.path.basename(ref).startswith('Qt'): + # eg., copy '/opt/qt@5/lib/QtGui.framework/Versions/5/QtGui' + # to '<Frameworks>/Qt/QtGui.framework/Versions/5/QtGui' + qtfwdir0 = framework_re.match(ref).group(1) + pth = qt_fwdir + '/' + qtfwdir0 + mkdir(os.path.dirname(pth)) + pkglib[ref] = pth + copyfile(ref, pth) + else: + print("Framework '%s' neglected." % ref) + +# Add relatively-referenced Qt framework; +# e.g., QtDBus framework referenced as `@rpath/QtDBus.framework/Versions/A/QtDBus` +print("%s: Add relatively-referenced Qt framework libraries:" % TITLE) +qt_exlib_dir = pkg_root + '/' + pkgbindir['exlib'] +for ref in rel_refs: + # select only Qt relative references + if os.path.basename(ref).startswith('Qt'): + # '@rpath/QtDBus.framework/Versions/A/QtDBus' => 'QtDBus.framework/Versions/A/QtDBus' + ref_qt = removeprefix(ref, '@rpath/') + qt_src = qt_framework_root + '/lib/' + ref_qt + qt_dst = qt_fwdir + '/' + ref_qt + qt_dst_dir = os.dirname(qt_dst) + pkglib["REL-" + qt_dst] = qt_dst + mkdir(qt_dst_dir) + copyfile(qt_src, qt_dst) + + # add the *1st-order optional* dependencies of the Qt library (if any) + # to the same folder as that of the Qt library + # deps1 = $(dylib_deps "$qt_dst" | sed -nE "s;[[:blank:]]*(/.+/opt/.+)[[:blank:]]\(.+;\1;p") + has_deps = 0 + for libpth_ in deps1: + has_deps = 1 + # '/usr/local/opt/libA.1.2.dylib' => 'libA.1.2.dylib' + libname_ = os.path.basename(libpth_) + # copy the library to the external library dir of the package + libdst = qt_exlib_dir + '/' + libname_ + pkglib[libpth_] = libdst + copyfile(libpth_, libdst) + +#-- collect all package binaries for later process +pkgbins = sorted(rm_list_duplicates([main_exe] + list(pkglib.values()) + libs_init)) +print("%s: All binaries:" % TITLE) +# a sorted list of binaries +for lib in pkgbins: + print(" + '" + removeprefix(lib, pkg_root + '/')) + +#-- adjust references to libraries +print("%s: Adjust references to libraries:" % TITLE) +libdir = pkg_root + '/' + pkgbindir['lib'] +rpaths = dict() +bin_deps = dict() + +for bin in pkgbins: + rpaths_tmp = set() + bin_deps[bin] = set(dylib_deps(bin)) + bindir = os.path.dirname(bin) # abs. dir of target binary + # abspth0 = original abs. full-path of the library + # abspth_pkg = abs. full-path of the library in the package + for abspth0, abspth_pkg in pkglib.items(): + # if the binary does not depend on the current library, do nothing + if not abspth0 in bin_deps[bin]: continue + + # change the library reference in the binary + # eg. '/usr/local/opt/lib/libA.5.dylib' => '@rpath/libA.5.dylib' + libname = os.path.basename(abspth_pkg) # library filename + libdir = os.path.dirname(abspth_pkg) # abs. dir of the referenced library + librelpth = "" # rel. path of the library + # make a framework-relative path for the Qt framework libraries + # eg. '/opt/qt@5/lib/QtGui.framework/Versions/5/QtGui' + # => 'QtGui.framework/Versions/5' + if libname.startswith('Qt'): + # rm framework root dir from the beginning; + # eg. '/opt/qt@5/lib/QtGui.framework/Versions/5/QtGui' + # => 'QtGui.framework/Versions/5/QtGui' + librelpth = removeprefix(abspth_pkg, qt_fwdir + '/') + # rm filename from the end + # eg. 'QtGui.framework/Versions/5/QtGui' + # => 'QtGui.framework/Versions/5' + librelpth = os.path.dirname(librelpth) + + ref_new = libname + # prepend with library rel. path, if any + if not librelpth: + ref_new = librelpth + '/' + ref_new + + subp.check_output(["install_name_tool", bin, "-change", + abspath0, "@rpath/" + ref_new], encoding='utf-8') + # make a proper RPATH to refer to the library within the package + # eg. '@loader_path/../Frameworks/Qt/' + rpath = "@loader_path/" + find_rpath(bindir, libdir, librelpth) + rpaths_tmp.add(rpath.strip()) + + # store a duplicateless list of rpaths needed for the binary, + # only if some rpaths are set. + # NOTE: libraries under the package lib dir. often need + # an extra RPATHs '../Library'. + if bin.startswith(libdir + '/'): + rpaths_tmp.add('@loader_path/../' + pkgbindir['exlib']) + + if rpaths_tmp: + rpaths[bin] = rpaths_tmp + + +# find the Python dependence for the *main* libraries +libs_main = find_dylibs(pkg_root + '/' + pkgbindir['lib']) +libs_pydep = list() # list of Python-dependent libraries +for lib in libs_main: + # get the first element the list Python dependences (all others must be the same) + _pydep = get_python_dependence(bin_deps[lib]) + # record the list of Python-dependent libraries + if not _pydep[1]: + libs_pydep.append(lib) + +# NOTE: zsh array indexing starts at 1 (unless option KSH_ARRAYS is set) + +if not _pydep[1]: + raise RuntimeError("No Python dependence found in the binaries.") + +pydepends_path=_pydep[0] +pydepends_filename=_pydep[1] +pydepends_fullpath = pydepends_path + '/' + pydepends_filename + +# relative Python reference; e.g. '@rpath/libpython3.11.dylib' +if pydepends_path.startswith('@'): + py_fmwk_rpaths = get_python_framework_path_rel(pydepends_fullpath) +else: + py_fmwk_rpaths = get_python_framework_path(pydepends_fullpath) + +for lib in libs_pydep: + rpaths[lib].extend(py_fmwk_rpaths) + +print("%s: Python dependence for the main libraries in '%s/%s':" + % (TITLE, pkg_root, pkgbindir['lib'])) +print(" + path: '%s'" % pydepends_path) +print(" + library: '%s'" % pydepends_filename) +print(" + framework paths:") +for pth in py_fmwk_rpaths: + print(" - '%s'" % pth) + +print("%s: Add proper RPATHs to the binaries:" % TITLE) +for bin, rpaths_bin in rpaths.items(): + # eg. RPATHS for 'lib/libA.dylib': ../Library , ../Frameworks/Qt + if not rpaths_bin: continue + # eg. install_name_tool libA.so -add_rpath RPATH1 -add_rpath RPATH2 ... + cmd_list = ["install_name_tool", bin] + for cmd in [["-add_rpath", r] for r in rpaths_bin]: + cmd_list.extend(cmd) + + out = subp.check_output(cmd_list, encoding='utf-8') + +print("%s: Done." % TITLE)