Add Dockerfile to build manylinux wheels

To build,

   $ docker build . --tag pyeclib-build-wheel
   $ for v in cp27-cp27m cp27-cp27mu cp35-cp35m ; do
   > docker run --rm --env PYTHON_VERSION=$v --env UID=$UID \
   > --env GID=$(id -g) --volume $PWD:/output:Z pyeclib-build-wheel
   > done

It should create x86_64 wheels suitable for CPython 2.7 and 3.5+ that
include both liberasurecode and ISA-L libraries.

Note that the pack_wheel.py script is useful even without the manylinux
Docker container. It can even build self-contained wheels on OS X,
though I've only tested on an old x86_64 mac, not the new arm64 hotness.

Change-Id: Id0eb192da37dcc83646bffa9137c96b7749b179f

.dockerignore

@@ -0,0 +1,9 @@
+.git
+.tox
+test
+**/__pycache__
+*.egg-info
+*.so
+*.whl
+build
+dist

.gitignore

@@ -20,6 +20,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+*.whl
 
 # PyInstaller
 #  Usually these files are written by a python script from a template 

Dockerfile

@@ -0,0 +1,49 @@
+# manylinux2010 has oldest build chain that can still build modern ISA-L
+# 2021-02-06-3d322a5 is newest tag that still had 2.7 support
+
+FROM quay.io/pypa/manylinux2010_x86_64:2021-02-06-3d322a5
+MAINTAINER OpenStack Swift
+
+# can also take branch names, e.g. "master"
+ARG LIBERASURECODE_TAG=1.6.4
+ARG ISAL_TAG=v2.31.0
+
+ARG SO_SUFFIX=-pyeclib
+ENV SO_SUFFIX=${SO_SUFFIX}
+ENV UID=1000
+# Alternatively, try cp27-cp27m, cp27-cp27mu
+ENV PYTHON_VERSION=cp35-cp35m
+
+RUN mkdir /opt/src /output
+RUN yum install -y zlib-devel
+# Update auditwheel so it can improve our tag to manylinux1 automatically
+# Not *too far*, though, since we've got the old base image
+RUN /opt/_internal/tools/bin/pip install -U 'auditwheel<5.2'
+
+# Server includes `Content-Encoding: x-gzip`, so ADD unwraps it
+ADD https://www.nasm.us/pub/nasm/releasebuilds/2.16.01/nasm-2.16.01.tar.gz /opt/src/nasm.tar
+RUN tar -C /opt/src -x -f /opt/src/nasm.tar
+RUN cd /opt/src/nasm-* && \
+    ./autogen.sh && \
+    ./configure --prefix=/usr && \
+    make nasm && \
+    install -c nasm /usr/bin/nasm
+
+ADD https://github.com/intel/isa-l/archive/${ISAL_TAG}.tar.gz /opt/src/isa-l.tar.gz
+RUN tar -C /opt/src -x -f /opt/src/isa-l.tar.gz -z
+RUN cd /opt/src/isa-l-* && \
+    ./autogen.sh && \
+    ./configure --prefix=/usr && \
+    make && \
+    make install
+
+ADD https://github.com/openstack/liberasurecode/archive/${LIBERASURECODE_TAG}.tar.gz /opt/src/liberasurecode.tar.gz
+RUN tar -C /opt/src -x -f /opt/src/liberasurecode.tar.gz -z
+RUN cd /opt/src/liberasurecode*/ && \
+    ./autogen.sh && \
+    CFLAGS="-DLIBERASURECODE_SO_SUFFIX='"'"'"${SO_SUFFIX}"'"'"'" ./configure --prefix=/usr && \
+    make && \
+    make install
+
+COPY . /opt/src/pyeclib/
+ENTRYPOINT ["/bin/sh", "-c", "/opt/python/${PYTHON_VERSION}/bin/python /opt/src/pyeclib/pack_wheel.py /opt/src/pyeclib/ --repair --so-suffix=${SO_SUFFIX} --wheel-dir=/output"]

pack_wheel.py

@@ -0,0 +1,336 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Build a pyeclib wheel that contains precompiled liberasurecode libraries.
+
+The goal is to build manylinux, abi3 wheels that are actually useful.
+
+   - ``manylinux`` ensures installability on a variety of distributions,
+     almost regardless of libc version.
+   - ``abi3`` ensures compatibility across a variety of python minor versions.
+   - "Actually useful" means you can not only import pyeclib, but use it to
+     perform encoding/decoding.
+   - Where possible, we want to bundle in ISA-L support, too.
+
+You might expect ``auditwheel repair`` to be able to do this for us. However,
+that's primarily designed around dynamic *linking* -- and while that's
+necessary, it's not sufficient; liberasurecode makes extensive use of dynamic
+*loading* as well, and so the dynamically loaded modules need to be included.
+"""
+
+import argparse
+import base64
+import errno
+import functools
+import hashlib
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import zipfile
+
+
+ENV_KEY = ('DYLD_LIBRARY_PATH' if sys.platform == 'darwin'
+           else 'LD_LIBRARY_PATH')
+if ENV_KEY in os.environ:
+    os.environ[ENV_KEY] += ':/usr/lib:/usr/local/lib'
+else:
+    os.environ[ENV_KEY] = '/usr/lib:/usr/local/lib'
+
+
+def locate_library(name, missing_ok=False):
+    """
+    Find a library.
+
+    :param name: The name of the library to find, not including the
+                 leading "lib".
+    :param missing_ok: If true, return ``None`` when the library cannot be
+                       located.
+    :raises RuntimeError: If the library cannot be found and ``missing_ok``
+                          is ``False``.
+    :returns: The full path to the library.
+    """
+    expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)
+    cmd = ['ld', '-t']
+    if sys.platform == 'darwin':
+        cmd.extend(['-arch', platform.machine()])
+    libpath = os.environ.get(ENV_KEY)
+    if libpath:
+        for d in libpath.split(':'):
+            cmd.extend(['-L', d.rstrip('/')])
+    cmd.extend(['-o', os.devnull, '-l%s' % name])
+    try:
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE,
+                             universal_newlines=True)
+        out, _ = p.communicate()
+    except subprocess.CalledProcessError:
+        pass
+    else:
+        if hasattr(os, 'fsdecode'):
+            out = os.fsdecode(out)
+        res = re.search(expr, out)
+        if res:
+            return os.path.realpath(res.group(0))
+
+    if missing_ok:
+        return None
+
+    raise RuntimeError('Failed to locate %s (checked %s)' % (name, libpath))
+
+
+def build_wheel(src_dir):
+    """
+    Build the base wheel, returning the path to the wheel.
+
+    Caller is responsible for cleaning up the tempdir.
+    """
+    tmp = tempfile.mkdtemp()
+    try:
+        subprocess.check_call([
+            sys.executable, 'setup.py',
+            'bdist_wheel', '-d', tmp, '--py-limited-api=cp35',
+        ], cwd=src_dir)
+        files = os.listdir(tmp)
+        assert len(files) == 1, files
+        return os.path.join(tmp, files[0])
+    except Exception:
+        shutil.rmtree(tmp)
+        raise
+
+
+def repack_wheel(whl, so_suffix, out_whl=None):
+    """
+    Repack a wheel to bundle in liberasurecode libraries.
+
+    This unpacks the wheel, copies all the supporting libraries to the
+    unpacked wheel, adjusts rpath etc for the libraries, rebuilds the
+    dist-info to include the libraries, and rebuilds the wheel.
+    """
+    if out_whl is None:
+        out_whl = whl
+
+    tmp = tempfile.mkdtemp()
+    try:
+        # unpack wheel
+        zf = zipfile.ZipFile(whl, 'r')
+        zf.extractall(tmp)
+
+        relocate_libs(tmp, so_suffix)
+        rebuild_dist_info_record(tmp)
+        build_zip(tmp, out_whl)
+    finally:
+        shutil.rmtree(tmp)
+
+
+def relocate_libs(tmp, so_suffix):
+    """
+    Bundle libraries into a unpacked-wheel tree.
+
+    :param tmp: the temp dir containing the tree for the unzipped wheel
+    :param so_suffix: the LIBERASURECODE_SO_SUFFIX used to build
+                      liberasurecode; this should be used to avoid
+                      interfering with system libraries
+    """
+    lib_dir = 'pyeclib.libs'
+    all_libs = [os.path.join(tmp, lib)
+                for lib in os.listdir(tmp) if '.so' in lib]
+    for lib in all_libs:  # NB: pypy builds may create multiple .so's
+        update_rpath(lib, '/' + lib_dir)
+
+    inject = functools.partial(
+        inject_lib,
+        tmp,
+        so_suffix=so_suffix,
+        lib_dir=lib_dir,
+        all_libs=all_libs,
+    )
+
+    # Be sure to move liberasurecode first, so we can fix its links to
+    # the others
+    relocated_libec = inject(locate_library('erasurecode'))
+    # Since liberasurecode links against other included libraries,
+    # need to update rpath
+    update_rpath(relocated_libec)
+
+    # These guys all stand on their own, so don't need the rpath update
+    inject(locate_library('nullcode'))
+    inject(locate_library('Xorcode'))
+    inject(locate_library('erasurecode_rs_vand'))
+
+    # Nobody actually links against this, but we want it anyway if available
+    isal = locate_library('isal', missing_ok=True)
+    if isal:
+        inject(isal)
+
+
+def update_rpath(lib, rpath_suffix=''):
+    if sys.platform == 'darwin':
+        subprocess.check_call([
+            'install_name_tool',
+            '-add_rpath', '@loader_path' + rpath_suffix,
+            lib,
+        ])
+    else:
+        subprocess.check_call([
+            'patchelf', '--set-rpath', '$ORIGIN' + rpath_suffix, lib])
+
+
+def inject_lib(
+    whl_dir,
+    src_lib,
+    so_suffix='-pyeclib',
+    lib_dir='pyeclib.libs',
+    all_libs=None,
+):
+    try:
+        os.mkdir(os.path.join(whl_dir, lib_dir))
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
+    if sys.platform == 'darwin':
+        old_lib = src_lib
+        name = os.path.basename(old_lib).split('.', 1)[0]
+        new_lib = name + so_suffix + '.dylib'
+    else:
+        name, _, version = os.path.basename(src_lib).partition('.so')
+        major = '.'.join(version.split('.', 2)[:2])
+        old_lib = name + '.so' + major
+        new_lib = name + so_suffix + '.so' + major
+
+    print('Injecting ' + new_lib)
+    relocated = os.path.join(whl_dir, lib_dir, new_lib)
+    shutil.copy2(src_lib, relocated)
+    if sys.platform == 'darwin':
+        subprocess.check_call([
+            'install_name_tool', '-id', new_lib, relocated])
+    else:
+        subprocess.check_call(['patchelf', '--set-soname', new_lib, relocated])
+
+    if all_libs:
+        # Fix linkage in the libs already moved -- this is mainly an issue for
+        # liberasurecode.so. Jerasure *would* need it for GF-Complete, but it
+        # seems unlikely we'd be able to include those any time soon
+        if sys.platform == 'darwin':
+            for lib in all_libs:
+                subprocess.check_call([
+                    'install_name_tool',
+                    '-change', old_lib,
+                    '@rpath/' + new_lib,
+                    lib,
+                ])
+        else:
+            subprocess.check_call([
+                'patchelf', '--replace-needed', old_lib, new_lib] + all_libs)
+        all_libs.append(relocated)
+
+    return relocated
+
+
+def rebuild_dist_info_record(tmp):
+    """
+    Update the dist-info RECORD information.
+
+    There are likely new files, and pre-existing files may have changed;
+    rebuild the whole thing.
+
+    See https://packaging.python.org/en/latest/specifications/
+    recording-installed-packages/#the-record-file for more info.
+    """
+    tmp = tmp.rstrip('/') + '/'
+    dist_info_dir = [d for d in os.listdir(tmp) if d.endswith('.dist-info')]
+    assert len(dist_info_dir) == 1, dist_info_dir
+    record_file = os.path.join(tmp, dist_info_dir[0], 'RECORD')
+    with open(record_file, 'w') as fp:
+        for dir_path, _, files in os.walk(tmp):
+            for file in files:
+                file = os.path.join(dir_path, file)
+                if file == record_file:
+                    fp.write(file[len(tmp):] + ',,\n')
+                    continue
+                hsh, sz = sha256(file)
+                fp.write('%s,sha256=%s,%d\n' % (file[len(tmp):], hsh, sz))
+
+
+def sha256(file):
+    hasher = hashlib.sha256()
+    sz = 0
+    with open(file, 'rb') as fp:
+        for chunk in iter(lambda: fp.read(128 * 1024), b''):
+            hasher.update(chunk)
+            sz += len(chunk)
+    hsh = base64.urlsafe_b64encode(hasher.digest())
+    return hsh.decode('ascii').strip('='), sz
+
+
+def build_zip(tmp, out_file):
+    """
+    Zip up all files in a tree, with archive names relative to the root.
+    """
+    tmp = tmp.rstrip('/') + '/'
+    with zipfile.ZipFile(out_file, 'w') as zf:
+        for dir_path, _, files in os.walk(tmp):
+            for file in files:
+                file = os.path.join(dir_path, file)
+                zf.write(file, file[len(tmp):])
+
+
+def repair_wheel(whl):
+    """
+    Run ``auditwheel repair`` to ensure appropriate platform tags.
+    """
+    if sys.platform == 'darwin':
+        return whl  # auditwheel only works on linux
+    whl_dir = os.path.dirname(whl)
+    subprocess.check_call(['auditwheel', 'repair', whl, '-w', whl_dir])
+    wheels = [f for f in os.listdir(whl_dir) if f != os.path.basename(whl)]
+    assert len(wheels) == 1, wheels
+    return os.path.join(whl_dir, wheels[0])
+
+
+def fix_ownership(whl):
+    uid = int(os.environ.get('UID', '1000'))
+    gid = int(os.environ.get('GID', uid))
+    os.chown(whl, uid, gid)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('src_dir')
+    parser.add_argument('-w', '--wheel-dir', default='.')
+    parser.add_argument('-s', '--so-suffix', default='-pyeclib')
+    parser.add_argument('-r', '--repair', action='store_true')
+    args = parser.parse_args()
+    whl = build_wheel(args.src_dir)
+    whl_dir = os.path.dirname(whl)
+    try:
+        repack_wheel(whl, args.so_suffix)
+        if args.repair:
+            whl = repair_wheel(whl)
+        output_whl = os.path.join(
+            args.wheel_dir, os.path.basename(whl))
+        shutil.move(whl, output_whl)
+        if os.geteuid() == 0:
+            # high likelihood of running in a docker container or something
+            fix_ownership(output_whl)
+    finally:
+        shutil.rmtree(whl_dir)
+
+
+if __name__ == '__main__':
+    main()

tox.ini

@@ -23,7 +23,7 @@ skip_install = True
 deps=
     hacking
 commands=
-    flake8 pyeclib/ setup.py test/
+    flake8 pyeclib/ setup.py test/ pack_wheel.py
 
 [testenv:venv]
 commands = {posargs}