gistfile1.txt

# This Dockerfile creates a very small AppImage using static
# linking and LTO. It is 100% portable for all x86_64 *OR*
# aarch64 systems.
#
# The general idea is to create a single file you can `scp`
# into all of your system and have a Lua based self contained
# config "just working".
FROM debian:buster-slim as bootstrap
MAINTAINER Emmanuel Lepage Vallee <[email protected]>

# Bare minimum of the Debian toolchain we need to bootstrap
# out own. `makeinfo` doesn't need to *work*, it needs to
# exist.
RUN apt update && apt install -y gcc g++ make\
    pkg-config wget gperf m4 && touch /usr/bin/makeinfo &&\
    chmod +x /usr/bin/makeinfo

# Compile binutils, just because we have a custom GCC...
RUN wget https://ftp.gnu.org/gnu/binutils/binutils-2.34.tar.bz2 && \
    tar -xpf binut*; cd binut*; ./configure --prefix=/opt/lto-toolchain \
    --target=x86_64-linux-musl --disable-multilib --disable-nls --enable-gold=yes\
    --disable-werror --disable-doc && make -j16 && make install && \
    cd / && rm -rf binut*

# Install Ancient Linux kernel headers. This will make sure we
# don't use systemcall newer than CentOS 6.0 (2010). This is
# the oldest system we can assume to work. Older RHEL were not
# super good at 64 bit anyway. If anyone still uses 10+ years
# old Ubuntu, they are insane. People with retro-computing
# are not on 64bit. So this kernel should work for everybody
# (using Linux).
RUN wget https://mirrors.edge.kernel.org/pub/linux/kernel/v2.6/linux-2.6.34.1.tar.bz2; \
    tar -xpf linux-*; cd linux-2.6.34.1; \
    make ARCH=x86_64 INSTALL_HDR_PATH=/opt/lto-toolchain/x86_64-linux-musl headers_install; \
    cd /; rm -rf linux-2*

# Let's build GCC, everybody loves building custom GCCs, right?
# The real reason is to disable thread local storage. LuaJIT
# doesn't use pthread for portability, which means it is also
# pretty dumb because it doesn't re-implement all the safety
# logic pthread provides. If it wasn't for this issue, the
# `musl-tools` GCC config included in Debian would have worked.
RUN wget https://ftpmirror.gnu.org/gcc/gcc-9.1.0/gcc-9.1.0.tar.gz; \
    tar -xpf gcc*; cd gcc*;contrib/download_prerequisites;\
    mkdir build-src;cd build-src;../configure --disable-tls --disable-multilib --enable-languages=c \
    --disable-libstdcxx  --disable-nls --disable-libgomp --disable-libitm     \
    --disable-libquadmath --disable-libsanitizer --disable-libssp             \
    --disable-libvtv --disable-libstdc__-v3 --enable-lto --enable-gold=yes    \
    --enable-ld=no --target=x86_64-linux-musl --host=x86_64-linux-gnu         \
    --disable-bootstrap --prefix=/opt/lto-toolchain && \
    make -j16 all-gcc && make install-gcc; rm -rf ./*

# We need a custom built musl for this, otherwise it will
# abort when LuaJIT tries to use too much stack. We also need
# to enable LTO. In my testing, 256kB never had issues, 128kB
# crashed a couple time. Note that it is important to set the
# variables for CC, CFLAGS and LD directly. We can't enable
# LTO yet and can't set the ENV because GCC in the next step
# still uses the host compiler.

RUN wget https://musl.libc.org/releases/musl-1.2.0.tar.gz && tar -xpf musl*; cd musl* && \
    sed -i 's/DEFAULT_STACK_SIZE 131072/DEFAULT_STACK_SIZE 262144/' src/internal/pthread_impl.h && \
    CFLAGS="-Os -fPIC" CC=/opt/lto-toolchain/bin/x86_64-linux-musl-gcc LD=/opt/lto-toolchain/bin/x86_64-linux-musl-ld \
    CROSS_COMPILE=/opt/lto-toolchain/ ./configure  --prefix=/opt/lto-toolchain/x86_64-linux-musl --disable-shared && \
    CFLAGS="-Os -fPIC" CC=/opt/lto-toolchain/bin/x86_64-linux-musl-gcc LD=/opt/lto-toolchain/bin/x86_64-linux-musl-ld \
    CROSS_COMPILE=/opt/lto-toolchain/ make AR=/opt/lto-toolchain/bin/x86_64-linux-musl-ar \
    RANLIB=/opt/lto-toolchain/bin/x86_64-linux-musl-ranlib && make install

# Nothing special here, we are targetting small size and
# high portability, so no magic allowed. Note that the GCC
# version was carefully chosen, not all of them work with LTO.
ENV CFLAGS='-march=x86-64 -Os -fPIC -fuse-linker-plugin -ffunction-sections -fdata-sections'
ENV LDFLAGS="$CFLAGS -Wl,-O1 -Wl,--as-needed"
ENV CXXFLAGS="$CFLAGS"

# Back to GCC, finish the job: add C++, libgcc and libstdc++
# (msgpack *needs* it, but doesn't *use* it...). Note that GCC must
# be compiled in 2 steps to do a bootstrap because it needs the libc
# header and the libc header are slightly dependant on the compiler
# setup.
#
# The patch is to fix CMake when compiled to static binaries
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58638
RUN cd /gcc*;\
    sed -iE 's/glibcxx_lt_pic_flag=$/glibcxx_lt_pic_flag="-prefer-pic"/' libstdc++-v3/configure.ac &&\
    sed -iE 's/glibcxx_compiler_pic_flag=$/glibcxx_compiler_pic_flag="$lt_prog_compiler_pic_CXX"/' libstdc++-v3/configure.ac &&\
    cd build-src;../configure \
    --disable-tls --disable-multilib --enable-languages=c,c++            \
    --disable-bootstrap --disable-nls --disable-libssp --disable-libvtv  \
    --disable-libsanitizer --disable-libgomp --disable-libitm            \
    --enable-lto --enable-gold=yes --enable-ld=yes --with-pic            \
    --target=x86_64-linux-musl --host=x86_64-linux-gnu                   \
    --prefix=/opt/lto-toolchain && make -j16 && make install;rm -rf ./* 

# Switch to the new toolchain. It is set directly using
# ENV rather than --target and/or CMake toolchain files
# for now. This will work because the target arch and
# host arch are the same. It would not work for cross
# compiling. However the focus for now is bootstrapping
# the system on which the cross compiling toolchain will
# be built. This way we have a nice musl-on-musl+PIC ABI
# from the get go rather than having all the weird errors
# when the host and target toolchains don't have the same
# flags and libc.
ENV CC="/opt/lto-toolchain/bin/x86_64-linux-musl-gcc"
ENV AR=/opt/lto-toolchain/bin/x86_64-linux-musl-gcc-ar
ENV CXX=/opt/lto-toolchain/bin/x86_64-linux-musl-g++
ENV RANLIB=/opt/lto-toolchain/bin/x86_64-linux-musl-gcc-ranlib
ENV NM=/opt/lto-toolchain/bin/x86_64-linux-musl-gcc-nm
ENV PREFIX=/opt/lto-toolchain/x86_64-linux-musl/

# Back to Musl, finish the job: Add shared library support. We wont use
# it, but it helps with buggy autotools looking for libc.so in their checks.
# The non-static `luajit` executable will also use it to support `require()`.
# This is used by some scripts in NeoVIM build system.
RUN cd /musl*; CROSS_COMPILE=/opt/lto-toolchain/ ./configure \
    --prefix=$PREFIX --enable-shared && make clean &&\
    CROSS_COMPILE=/opt/lto-toolchain/ make AR=$AR RANLIB=$RANLIB LD=$LD && make install -j3

# Enable Gold and LTO, for some programs ld.bfd now works.
# This is much better than a few years ago, so gold might
# eventually be retired. However, I still see more LTO
# failures with BFD.
ENV LD=$PREFIX/bin/ld.gold
ENV CFLAGS='-march=x86-64 -Os -flto -fPIC -fuse-linker-plugin -ffunction-sections -fdata-sections -fuse-ld=gold'
ENV LDFLAGS="$CFLAGS -flto=8 -fuse-linker-plugin -Wl,-O1 -Wl,--as-needed -Wl,-flto"
ENV CXXFLAGS="$CFLAGS -static-libgcc -static-libstdc++"

# Include LuaJIT in the toolchain since it is used by pretty
# much all of Reclaim, including the builders.
RUN wget https://github.com/LuaJIT/LuaJIT/archive/v2.1.0-beta3.tar.gz; tar -xpvf v2.1.0-beta3*;\
    cd LuaJ*;\
    sed -i "s|export PREFIX= /usr/local|export PREFIX= $PREFIX|g" Makefile;\
    sed -i "s|DEFAULT_CC = gcc|DEFAULT_CC = $CC -static-libgcc -flto|g" src/Makefile;\
    make -j16 install

# I despise libtool, it is an horrible idea hiding simple things
# beind a broken abstraction to make everybody life harder.
RUN wget http://ftpmirror.gnu.org/libtool/libtool-2.4.6.tar.gz && \
    tar -xpf libtool*; cd libtoo* && ./configure --prefix=/opt/lto-toolchain/ \
    --host=x86_64-linux-musl --program-prefix=x86_64-linux-musl- \
    --disable-ltdl-install && make install

# Avoid copy/pasting the same long commands over and over
RUN echo './configure \
        --libdir=$PREFIX/lib --host=x86_64-linux-gnu --target=x86_64-linux-musl \
        --disable-shared --enable-static \
        --includedir=$PREFIX/include --prefix=$PREFIX $@ ' >> $PREFIX/bin/cc_configure &&\
    chmod +x $PREFIX/bin/cc_configure &&\
    echo 'make -j16 CFLAGS+="-D_GNU_SOURCE=1" LDFLAGS+="-all-static" $@' > $PREFIX/bin/libtool_make &&\
    chmod +x $PREFIX/bin/libtool_make

ENV LIBTOOL=/opt/lto-toolchain/bin/x86_64-linux-musl-libtool
ENV PKG_CONFIG=$PREFIX/bin/pkg-config
ENV PATH=$PREFIX/bin/:$PATH
ENV ACLOCAL_PATH=$PREFIX/share/aclocal

# OpenSSL is needed by some programs to download their dependencies.
# `wget` itself needs it to fetch https:// links and CMake needs
# it for it's built in external dependency and file fetching
# capability.
RUN wget https://www.openssl.org/source/openssl-1.1.1g.tar.gz;    \
    tar -xpf openssl*; cd openssl*; ./Configure linux-x86_64      \
    --prefix=$PREFIX/ no-shared && make -j16 && make install &&   \
    rm -rf $PREFIX/share/man $PREFIX/share/doc &&                 \
    cp $PREFIX/lib64/* $PREFIX/lib/ || true;                      \
    cp /etc/ssl/certs/* $PREFIX/ssl/certs/

RUN wget https://www.zlib.net/zlib-1.2.11.tar.gz; tar -xpf zlib*; cd zlib*;\
    ./configure --libdir=$PREFIX/lib \
        --prefix=$PREFIX --static && make install -j16

# Translation is not optional.
RUN wget https://ftp.gnu.org/pub/gnu/gettext/gettext-0.20.2.tar.gz && \
    tar -xpf gettext*; cd gettext*; \
    cc_configure && libtool_make install

# Fix musl support an an old version of glib. Eventually an option
# could be to use the last version shipped with CentOS 6 rather
# than the pure upstream.
RUN apt install libglib2.0-dev -y &&\
    wget https://ftp.gnome.org/pub/gnome/sources/glib/2.25/glib-2.25.17.tar.gz && \
    tar -xpf glib-*; cd /glib-2.25.17; \
    sed -i '1i#include <sys/sysmacros.h>' ./gio/gdbusmessage.c;\
    CFLAGS="$CFLAGS -D_GNU_SOURCE=1"\
    ac_cv_func_posix_getgrgid_r=set ac_cv_func_posix_getpwuid_r=set glib_cv_uscore=set glib_cv_stack_grows=set \
    cc_configure && make -j16 install &&\
    apt remove libglib2.0-dev -y && apt autoremove -y

# pkg-config requires glib, but the ancient one is enough.
# As for all the `static` flags, pkg-config tries hard to
# strip them. Then breaks. It *really* has to be static to
# to work.
RUN wget https://pkg-config.freedesktop.org/releases/pkg-config-0.29.2.tar.gz && tar -xpf pkg-config*; cd pkg-*; \
    export CFLAGS="$CFLAGS -static -I $PREFIX/include/glib-2.0/ -I $PREFIX/lib/glib-2.0/include/"; \
    GLIB_CFLAGS="-Os -fPIC" \
    GLIB_LIBS=$PREFIX/lib/libglib-2.0.a cc_configure && libtool_make install -j16

# We will need this later on.
RUN wget https://ftp.gnu.org/gnu/wget/wget-1.20.3.tar.gz; tar -xpf wget*; cd wget*;\
    CFLAGS="$CFLAGS -static" OPENSSL_CFLAGS="-Os -fPIC" \
    LDFLAGS="$LDFLAGS $PREFIX/lib64/libssl.a $PREFIX/lib64/libcrypto.a" \
    OPENSSL_LIBS="$PREFIX/lib64/libssl.a $PREFIX/lib64/libcrypto.a" \
    cc_configure --with-ssl=openssl && make install -j16

# LPEG is used by the builders (like NeoVIM) and runtime (like NotMuch).
RUN wget http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.0.2.tar.gz;\
    tar -xpf lpeg*; cd lpeg*;\
    sed -i "s|CC = gcc|CC = $CC|g" makefile; \
    sed -i "s|LUADIR = ../lua|LUADIR = $PREFIX/include/luajit-2.1|g" makefile; \
    mkdir -p $PREFIX/lib/lua/5.1/; \
    make && cp ./lpeg.so $PREFIX/lib/lua/5.1/

# CMake is used a lot. Using the Debian one is "fine", but
# it pulls a lot of dependencies that cause problems with
# buggy Makefiles and libtool consumers.
# Don't use LTO, it breaks the build.
#
# Making a true static CMake isn't possible. Either CC
# is allowed to use shared and CXX forced static *or* it
# is possible to use a CC with -static wrapped in a shell
# script that removes -static if -shared is present. Otherwise
# the compiler tests it produces will be "useless" as they
# wont test what then intend to test and this will blow up
# when compiling large and complex software. I made both
# solutions work. Using -static for CXX only is simpler
# given the goal is having a static `cmake` binary. For the
# record, the script is:
#    !/bin/sh
#    echo "$@" | grep shared > /dev/null
#    if [ $? == 0 ]; then for arg do; shift
#    [ "$arg" = "-static" ] && continue
#    set -- "$@" "$arg"; done; fi
#    /opt/lto-toolchain/bin/x86_64-linux-musl-gcc $@
RUN wget https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1.tar.gz && \
    tar -xpf cmake*; cd cmake*; \
    CFLAGS="-Os -fPIC -static-libgcc" \
    LDFLAGS="-Os -fPIC -static-libgcc -static-libstdc++" \
    LD=$PREFIX/bin/ld.bfd \
    LD_LIBRARY_PATH=$PREFIX/lib/:$PREFIX/lib64/ \
    CXXFLAGS="-static-libgcc -static-libstdc++ -fPIC -Os -static --static" \
    ./configure --prefix=$PREFIX/ --verbose &&\
    CXXFLAGS="-static-libgcc -static-libstdc++ -fPIC -Os" make -j17 && make install &&\
    rm -rf /cmake*

# While zlib/gz is still de-facto the most common format,
# some software no longer provide binaries and have moved
# to lxma/xz. Most software now provide xz releases.
RUN wget https://tukaani.org/xz/xz-5.2.5.tar.gz; tar -xpf xz*; cd xz*; \
    cc_configure && libtool_make install

# For now we used the Debian make, but it wont be installed in
# going forward.
RUN wget https://ftp.gnu.org/gnu/make/make-4.3.tar.gz; tar -xpf make*; cd make*;\
    CFLAGS="$CFLAGS -static" cc_configure && make install

# gperf is used for code generation in a lot of makefiles
RUN wget http://ftp.gnu.org/pub/gnu/gperf/gperf-3.1.tar.gz; tar -xpf gperf*; cd gperf*;\
    CXXFLAGS="$CXXFLAGS -static" cc_configure && make install

# Fill the toolchain OS with more content + cleanups.
RUN for FILE in $(find /opt/lto-toolchain/bin -maxdepth 1); do \
    cp -v $FILE $PREFIX/bin/$(basename $FILE | sed 's/x86_64-linux-musl-//') ; \
done; strip -s /$PREFIX/bin/* || true

# Now, lets drop the bloat from the original bootstrap and
# build the AppImageKit. We "need" to do it instead of the
# prebuilt binary because it uses the system GLibC and
# doesn't run in Reclaim because it's not installed
# (on purpose). It also doesn't run on some servers I
# actually work on because either libfuse isn't compatible
# or the are Alpine/buildroot/Yocto/BottleRocket based.
FROM debian:buster-slim as appimagesdk
MAINTAINER Emmanuel Lepage Vallee <[email protected]>

COPY --from=bootstrap /opt/lto-toolchain/ /opt/lto-toolchain/

RUN apt update && apt install git desktop-file-utils xxd automake xsltproc -y

# Setup the toolchain.
ENV CFLAGS='-march=x86-64 -Os -fPIC -fuse-linker-plugin -ffunction-sections -fdata-sections'
ENV LDFLAGS='-march=x86-64 -Os -fuse-linker-plugin -Wl,-O1 -Wl,--as-needed'
ENV CC="/opt/lto-toolchain/bin/x86_64-linux-musl-gcc"
ENV AR=/opt/lto-toolchain/bin/x86_64-linux-musl-gcc-ar
ENV CXX=/opt/lto-toolchain/bin/x86_64-linux-musl-g++
ENV RANLIB=/opt/lto-toolchain/bin/x86_64-linux-musl-gcc-ranlib
ENV NM=/opt/lto-toolchain/bin/x86_64-linux-musl-gcc-nm
ENV PREFIX=/opt/lto-toolchain/x86_64-linux-musl
ENV LD=$PREFIX/bin/ld.gold
ENV CFLAGS='-march=x86-64 -Os -flto -fuse-linker-plugin -ffunction-sections -fdata-sections -fuse-ld=gold'
ENV LDFLAGS='-march=x86-64 -Os -flto=8 -fuse-linker-plugin -Wl,-O1 -Wl,--as-needed -Wl,-flto'
ENV LIBTOOL=/opt/lto-toolchain/bin/x86_64-linux-musl-libtool
ENV PKG_CONFIG=$PREFIX/bin/pkg-config
ENV PATH=$PREFIX/bin/:$PATH
ENV ACLOCAL_PATH=$PREFIX/share/aclocal
ENV ENV=/root/.profile

RUN wget https://github.com/libusb/libusb/archive/v1.0.14.tar.gz; tar -xpvf v1.0.14.tar.gz; cd libusb*; \
    ./autogen.sh;\
    cc_configure && make -j16 install

RUN wget https://mirrors.edge.kernel.org/pub/linux/utils/usb/usbutils/usbutils-003.tar.gz && tar -xpf usbutils*; cd usbutils*; \
    CFLAGS="$CFLAGS -I$PREFIX/include/libusb-1.0/" \
    LIBUSB_CFLAGS=$CFLAGS LIBUSB_LIBS=$PREFIX/lib/libusb-1.0.a \
    cc_configure && make -j16 install

RUN wget https://pci-ids.ucw.cz/v2.2/pci.ids.gz; gunzip pci.ids.gz

# Gentoo eudev is the next best choice, but it is too recent for
# the kernel headers we are using. It's not that I hate SystemD,
# it works for me, it's that making portable binaries means
# only using udev. Portable binaries will never support SystemD.
# Containers, mobile and embedded will likely never use it since
# they are not affected by the problems it tries to solve.
#
# Udev 175 requires kernel 2.6.34. Older ones are worst, newer ones
# pull more dependencies and require newer kernels.
#
# Note that the patching is done because it was never designed
# to work with musl and doesn't even compile with newer GCC.
RUN wget https://mirrors.edge.kernel.org/pub/linux/utils/kernel/hotplug/udev-175.tar.gz; tar -xpf udev*; cd udev*; \
    sed -i '1i#include <sys/sysmacros.h>' udev/udev.h;\
    sed -i '1i#include <sys/sysmacros.h>' ./libudev/libudev-private.h;\
    sed -i 's|const struct key\* lookup_key |//|' extras/keymap/keymap.c;\
    sed -i '1iconst struct key* lookup_key (register const char *str, register size_t len);' extras/keymap/keymap.c;\
    sed -i '1i#include <stdlib.h>' extras/keymap/keymap.c;\
    CFLAGS="$CFLAGS -Du8=uint8_t -I$PREFIX/include/" \
    GLIB_LIBS=$PREFIX/lib/libglib-2.0.a GLIB_CFLAGS="-Os -fPIC" \
    cc_configure --disable-introspection \
        --disable-gudev --disable-gtk-doc --with-usb-ids-path=$PREFIX/share/usb.ids --with-pci-ids-path=/ \
        --disable-mtd_probe && make -j16 install

RUN wget https://www.cairographics.org/releases/pixman-0.40.0.tar.gz; tar -xpf pixm*; cd pixm*;\
   cc_configure && make -j16 install

RUN wget https://www.cairographics.org/releases/cairo-1.16.0.tar.xz; tar -xpf cairo*; cd cairo*;\
    cc_configure --disable-png --disable-svg && make -j16 install

# Fix musl support, this is a very old version of libfuse
RUN wget https://github.com/libfuse/libfuse/archive/fuse_2_8_4.tar.gz; tar -xpf fuse_2_8_4.tar.gz; cd libfuse*;\
    sed -i 's|/usr/share/gettext/config.rpath|$PREFIX/share/gettext/config.rpath|' makeconf.sh; \
    sed -i '1i#include <paths.h>' lib/mount_util.c;\
    sed -i '1i#include <paths.h>' util/fusermount.c;\
    ./makeconf.sh; \
    cc_configure && libtool_make install
    
RUN wget https://www.libarchive.org/downloads/libarchive-3.4.2.tar.gz; tar -xpf libarchive*; cd libarch*; \
    cc_configure && libtool_make install

RUN wget https://github.com/vasi/squashfuse/releases/download/0.1.103/squashfuse-0.1.103.tar.gz; tar -xpf squashfuse-0.1.103.tar.gz; cd squash*; \
    cc_configure && libtool_make install

RUN git clone https://github.com/AppImage/AppImageKit.git && \
    cd AppImageKit; git pull --recurse-submodules;git submodule update --init --recursive; \
    mkdir build; cd build; CFLAGS="$CFLAGS -D__BEGIN_DECLS=/**/ -D__END_DECLS=/**/ -I/squashfuse-0.1.103/" \
     cmake .. \
        -DBUILD_TESTING=OFF -DUSE_SYSTEM_XZ=ON -DUSE_SYSTEM_LIBARCHIVE=ON -DUSE_SYSTEM_SQUASHFUSE=ON \
        -DLibArchive_LIBRARY=$PREFIX/lib/libarchive.a \
        -DLibArchive_INCLUDE_DIR=$PREFIX/include/ &&\
    PATH=$PATH:$PREFIX/bin make -j16