Skip to content

Instantly share code, notes, and snippets.

@theory

theory/docker-compose.yml

Last active June 20, 2024 19:49
Show Gist options
  • Save theory/7dc164e5772cae652d838a1c508972ae to your computer and use it in GitHub Desktop.
Save theory/7dc164e5772cae652d838a1c508972ae to your computer and use it in GitHub Desktop.
Download ZIP
POC: Create and install Postgres Trunk Binaries
Raw
docker-compose.yml
version: "3.9"
name: trunking
# Start a Zot service for pushing and pulling OCI images, and a
# pgxn/pgxn-tools image for building and testing Linux/AMD binaries.
#
# docker compose up -d
# docker compose exec linux bash
# make clean && make && make trunk
#
# On macOS:
# make clean && make && make trunk
# ./push_trunk localhost:5000/theory/semver semver-0.32.1+pg16-darwin-23.5.0-arm64 semver-0.32.1+pg16-linux-amd64
# clean -dfx --exclude=.vscode
# find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*' -exec rm -rf "{}" \;
#
# ./install_trunk localhost:5000/theory/semver:v1
# find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'
#
# Back on Linux
# ./install_trunk zot:5000/theory/semver:v1
# find "$(pg_config --sharedir)" "$(pg_config --pkglibdir)" "$(pg_config --docdir)" -name '*semver*'
# Name the network for all of the services to join.
networks:
default:
name: pgxnnet
services:
zot:
image: ghcr.io/project-zot/zot-linux-arm64:latest
container_name: zot
ports:
- 5000:5000
hostname: zot
linux:
image: pgxn/pgxn-tools
platform: "linux/amd64"
container_name: linux
hostname: linux
working_dir: /work
volumes:
- ".:/work"
# Install oras, then start Postgres 16 and install rsync and jq
entrypoint:
- "/bin/sh"
- -ecx
- |
cd /tmp
curl -LO "https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz"
mkdir -p oras-install/
tar -zxf oras_1.2.0_*.tar.gz -C oras-install/
sudo mv oras-install/oras /usr/local/bin/
rm -rf oras_1.2.0_*.tar.gz oras-install/
pg-start 16 rsync jq
tail -f /dev/null
Raw
install_trunk
#!/bin/bash
# POC for installing Trunk format binaries created with trunk.mk. Requires:
#
# * bash
# * tar
# * shasum
# * jq
# * uname
# * pg_config
# * rsync
trap 'exit' ERR
set -E
install_trunk() {
local trunk=$1
if [ -z "$trunk" ]; then
printf "Usage:\n\n %s PACKAGE_PATH\n" "$0"
exit 1
fi
# Determine the platform.
local my_os my_osv my_arch platform
my_os=$(uname | tr '[:upper:]' '[:lower:]')
my_osv=$(uname -r)
my_arch="$(uname -m)"
if [ "$my_arch" == "x86_64" ]; then my_arch=amd64; fi
if [ "$my_os" == "darwin" ]; then
platform="$my_os/$my_arch:$my_osv"
else
platform="$my_os/$my_arch"
fi
# Download.
local file
file=$(basename "$(oras pull --no-tty --plain-http --format 'go-template={{(first .files).path}}' --platform "$platform" "$trunk")")
# Unpack.
printf "Unpacking %s\n" "$file"
tar zxf "$file"
# Go into the directory
local dir="${file%.*}"
cd "$dir" || exit
# Verify the checksums.
printf "Verifying all checksums..."
# XXX This does not fail if a file isn't present in digests.
shasum --check -b --strict digests
printf "Done!\n"
# Verify the Trunk version
printf "Verifying compatibility with Trunk package 0.1.0\n"
local tv
tv=$(jq -r .trunk trunk.json)
if [ "$tv" != "0.1.0" ]; then
printf "Unsupported Trunk format version %s\n" "$tv"
exit 1
fi
# Verify the Postgres version.
local pkg_pg my_pg
my_pg=$(pg_config --version | sed -E 's/^[^ ]+ ([^ ]+).*$/\1/')
printf "Verifying compatibility with PostgreSQL %s\n" "$my_pg"
pkg_pg=$(jq -r .postgres.version trunk.json)
if [ "$pkg_pg" != "$my_pg" ]; then
printf "Trunk package contains binaries for Postgres %s but this host runs Postgres %s\n" "$pkg_pg" "$my_pg"
exit 1
fi
printf "Verifying compatibility with %s\n" "$platform"
local pkg_os
pkg_os=$(jq -r .platform.os trunk.json)
if [ "$pkg_os" != "any" ]; then
# Verify the OS
if [ "$pkg_os" != "$my_os" ]; then
printf "Trunk package contains %s binaries but this host runs %s\n" "$pkg_os" "$my_os"
exit 1
fi
# Verify the architecture.
local pkg_arch
pkg_arch=$(jq -r .platform.arch trunk.json)
if [ "$pkg_arch" != "$my_arch" ]; then
printf "Trunk package contains %s binaries but this host runs %s\n" "$pkg_arch" "$my_arch"
exit 1
fi
fi
# Make sure we have pgsql directory.
if [ ! -d 'pgsql' ]; then
printf "Package contains no install files; exiting\n"
exit 1
fi
cd 'pgsql' || exit
for subdir in *; do
[[ -d "$subdir" ]] || continue
case $subdir in
share)
install_dir "$subdir" "$(pg_config --sharedir)"
;;
pkglib)
install_dir "$subdir" "$(pg_config --pkglibdir)"
;;
pkginclude)
install_dir "$subdir" "$(pg_config --pkgincludedir)"
;;
lib)
install_dir "$subdir" "$(pg_config --libdir)"
;;
include)
install_dir "$subdir" "$(pg_config --includedir)"
;;
bin)
install_dir "$subdir" "$(pg_config --bindir)"
;;
doc)
install_dir "$subdir" "$(pg_config --docdir)"
;;
man)
install_dir "$subdir" "$(pg_config --mandir)"
;;
html)
install_dir "$subdir" "$(pg_config --htmldir)"
;;
locale)
install_dir "$subdir" "$(pg_config --localedir)"
;;
sysconf)
install_dir "$subdir" "$(pg_config --sysconfdir)"
;;
*)
printf "Unknown install directory %s; skipping\n" "$subdir"
;;
esac
done
}
install_dir() {
local src="$1"
local dst="$2"
printf "Installing %s into %s..." "$src" "$dst"
cd "$src" || exit
rsync -q -a -v . "$dst" || exit
printf "Done\n"
cd ..
}
install_trunk "$@"
Raw
push_trunk
#!/bin/bash
# POC for publishing Trunk packages to an OCI repository with an image index
# to allow pulling a platform-specific binary. Requires:
#
# * oras
# * jq
# * zot: docker run -d -p 5000:5000 --name oras-quickstart ghcr.io/project-zot/zot-linux-arm64:latest
#
# Inspired by the Homebrew implementation of the pattern as referenced in
# https://github.com/oras-project/oras/issues/237, and with thanks to the
# denizens of the #oras and #zot channels on the CNCF Slack.
trap 'exit' ERR
set -E
# OCI types to create.
ARTIFACT_TYPE=application/vnd.pgxn.trunk.layer.v1
MEDIA_TYPE=application/vnd.oci.image.layer.v1.tar+gzip
CONFIG_TYPE=application/vnd.oci.image.config.v1+json
OCI_DIR=oci_dir
INDEX_FILE=image_index.json
push_image() {
# Push the image into the OCI layout directory $OCI_DIR.
local trunk=$1
oras push --no-tty \
--oci-layout "$OCI_DIR" \
--artifact-type "${ARTIFACT_TYPE}" \
--config "${trunk}_config.json":"$CONFIG_TYPE" \
--format go-template='{{.digest}}' \
--annotation-file "${trunk}_annotations.json" \
"$trunk.trunk":"$MEDIA_TYPE"
}
make_manifest() {
local trunk=$1
local digest=$2
local anno platform
# Extract just the pgxn.trunk annotations.
anno=$(jq -c \
'.["$manifest"] | with_entries(select(.key | startswith("org.pgxn.trunk.")))' \
"${trunk}_annotations.json"
)
# Extract the platform config.
platform=$(jq \
'pick(.os, .["os.version"], .architecture)| with_entries(select(.value |. !=null and . != ""))' \
"$trunk"_config.json
)
# Create and return the image manifest.
oras manifest fetch --oci-layout "$OCI_DIR@${digest}" --descriptor \
| jq --argjson anno "$anno" --argjson platform "$platform" \
'{
mediaType: .mediaType,
size: .size,
digest: .digest,
platform: $platform,
annotations: $anno
}'
}
write_index() {
darwin_manifest=$1
linux_manifest=$2
# Build the image index with the two manifests.
jq -n --argjson linux "$linux_manifest" \
--argjson darwin "$darwin_manifest" \
--argjson annotations "$(cat semver_annotations.json)" \
'{
schemaVersion: 2,
mediaType: "application/vnd.oci.image.index.v1+json",
manifests: [$linux, $darwin],
annotations: $annotations
}' > "$INDEX_FILE"
}
push_trunk() {
# Only testing for Darwin and Linux rn.
local repo=$1
local darwin_trunk=$2
local linux_trunk=$3
if [ -z "$repo" ] || [ -z "$darwin_trunk" ] || [ -z "$linux_trunk" ]; then
printf "Usage:\n\n %s REPO DARWIN.trunk LINUX.trunk\n" "$0"
exit 1
fi
# Push the images and grab the resulting digests.
darwin_digest=$(push_image "$darwin_trunk")
linux_digest=$(push_image "$linux_trunk")
# Create the image manifests.
darwin_manifest=$(make_manifest "$darwin_trunk" "$darwin_digest")
linux_manifest=$(make_manifest "$linux_trunk" "$linux_digest")
# Write out and push the image index.
write_index "$darwin_manifest" "$linux_manifest"
oras manifest push --oci-layout ./"$OCI_DIR":image-index "$INDEX_FILE"
# Push everything from the local layout to the remote registry.
oras cp --from-oci-layout ./"$OCI_DIR":image-index --to-plain-http "${repo}:v1"
# View the remote image index manifest.
oras manifest get --plain-http "${repo}:v1" | jq
# Cleanup.
rm -rf "$OCI_DIR" "$INDEX_FILE"
}
push_trunk "$@"
Raw
trunk.mk
# Trunk packaging. To use, create a standard PGXS Makefile for your extension.
# The only extra variables are DISTVERSION, TITLE, DESCRIPTION, LICENSE,
# LANGUAGE, VENDOR, URL, and REPO_URL.
#
# ``` make
# DISTVERSION = 1.2.1
# TITLE = Bike
# DESCRIPTION = A bicycle inside your database.
# LICENSE = mit
# LANGUAGE = c
# VENDOR = PGXN
# URL = https://pgxn.org/dist/bike
# REPO_URL = https://github.com/pgxn/bike
#
# EXTENSION = bike
# MODULEDIR = $(EXTENSION)
# DATA = $(wildcard sql/*.sql)
# DOCS = $(wildcard doc/*.mmd)
# MODULES = $(patsubst %.c,%,$(wildcard src/*.c))
# PG_CONFIG ?= pg_config
#
# PGXS := $(shell $(PG_CONFIG) --pgxs)
# include $(PGXS)
# include ./trunk.mk
# ```
#
# Then build it:
#
# ``` sh
# make trunk
# ```
#
# This will create a file with a name similar to
#
# bike-1.2.1+pg16-darwin-23.5.0-arm64
#
# Requires:
#
# * pg_config
# * PGXS (from `pg_config --pgxs`)
# * tar
# * shasum
# * jq
LAUNGUAGE ?= c
LICENSE ?= PostgreSQL
TITLE ?= $(EXTENSION)
VENDOR ?= PGXN
URL ?= $(REPO_URL)
pkg_arch := $(shell uname -m)
ifeq ($(pkg_arch),x86_64)
pkg_arch := amd64
endif
ifeq ($(PORTNAME),darwin)
pkg_os_ver := $(shell uname -r)
pkg := $(EXTENSION)-$(DISTVERSION)+pg$(MAJORVERSION)-$(PORTNAME)-$(pkg_os_ver)-$(pkg_arch)
else
pkg := $(EXTENSION)-$(DISTVERSION)+pg$(MAJORVERSION)-$(PORTNAME)-$(pkg_arch)
endif
pkg_dir := $(pkg)
pkg_installdir := $(pkg_dir)/pgsql
pkg_sharedir := $(pkg_installdir)/share
pkg_libdir := $(pkg_installdir)/lib
pkg_pkglibdir := $(pkg_installdir)/pkglib
pkg_docdir := $(pkg_installdir)/doc/$(docmoduledir)
pkg_bindir := $(pkg_installdir)/bin
pkg_incdir := $(pkg_installdir)/include/server/$(docmoduledir)
ifdef SO_MAJOR_VERSION
pkg_pkgconfigdir = $(pkg_libdir)/pkgconfig
endif
pkg_info_files ?= $(wildcard README* readme* Readme* LICENSE* license* License* CHANGE* change* Change*)
EXTRA_CLEAN += $(EXTENSION)-$(DISTVERSION)+*/
# Phony target to create the trunk and OCI JSON files.
trunk: $(pkg).trunk $(pkg)_config.json $(pkg)_annotations.json $(EXTENSION)_annotations.json
# Use jq to create the OCI image index annoations.
# https://github.com/opencontainers/image-spec/blob/main/annotations.md
$(EXTENSION)_annotations.json:
jq -n \
--arg org.opencontainers.image.created "$$(date +%Y-%m-%dT%TZ)" \
--arg org.opencontainers.image.licenses "$(LICENSE)" \
--arg org.opencontainers.image.title "$(TITLE)" \
--arg org.opencontainers.image.description "$(DESCRIPTION)" \
--arg org.opencontainers.image.source "$(REPO_URL)" \
--arg org.opencontainers.image.vendor "$(VENDOR)" \
--arg org.opencontainers.image.ref.name "$(DISTVERSION)" \
--arg org.opencontainers.image.version "$(DISTVERSION)" \
--arg org.opencontainers.image.url "$(URL)" \
'$$ARGS.named | with_entries(select(.value |. !=null and . != ""))' > $@
# Use jq to create the OCI image configuration.
$(pkg)_config.json:
@jq -n \
--arg os "$(PORTNAME)" \
--arg os.version "$(pkg_os_ver)" \
--arg architecture "$(pkg_arch)" \
--arg created "$$(date +%Y-%m-%dT%TZ)" \
'$$ARGS.named | with_entries(select(.value |. !=null and . != ""))' > $@
# Use jq to create the OCI image manifest annotations.
$(pkg)_annotations.json: $(pkg).trunk $(pkg)_config.json
@anno=$$(jq -n \
--arg org.opencontainers.image.created "$$(date +%Y-%m-%dT%TZ)" \
--arg org.opencontainers.image.title "$(pkg).trunk" \
--arg org.opencontainers.image.licenses "$(LICENSE)" \
--arg org.opencontainers.image.description "$(DESCRIPTION)" \
--arg org.opencontainers.image.source "$(REPO_URL)" \
--arg org.opencontainers.image.vendor "$(VENDOR)" \
--arg org.opencontainers.image.ref.name "$(DISTVERSION)" \
--arg org.opencontainers.image.version "$(DISTVERSION)" \
--arg org.opencontainers.image.url "$(URL)" \
--arg org.pgxn.trunk.pg.version "$(VERSION)" \
--arg org.pgxn.trunk.pg.major "$(MAJORVERSION)" \
--arg org.pgxn.trunk.pg.version_num "$(VERSION_NUM)" \
--arg org.pgxn.trunk.version "0.1.0" \
'$$ARGS.named | with_entries(select(.value |. !=null and . != ""))' \
) && jq -n \
--argjson '$$manifest' "$$anno" \
'$$ARGS.named' > $@
$(pkg).trunk: package
tar zcvf $@ $(pkg_dir)
# Use jq to build trunk.json. XXX Add dependencies
$(pkg_dir)/trunk.json: pkg_dirs
@pg=$$(jq -n \
--arg version "$(VERSION)" \
--arg major "$(MAJORVERSION)" \
--argjson number "$(VERSION_NUM)" \
--arg libs "$(LIBS)" \
--arg cppflags "$(CPPFLAGS)" \
--arg cflags "$(CFLAGS)" \
--arg ldflags "$(LDFLAGS)" \
'$$ARGS.named' \
) && pkg=$$(jq -n \
--arg name "$(EXTENSION)" \
--arg version $(DISTVERSION) \
--arg language "$(LAUNGUAGE)" \
--arg license "$(LICENSE)" \
'$$ARGS.named' \
) && plat=$$(jq -n \
--arg os "$(PORTNAME)" \
--arg version "$(pkg_os_ver)" \
--arg arch $(pkg_arch) \
'$$ARGS.named | with_entries(select(.value |. !=null and . != ""))' \
) && jq -n \
--arg trunk 0.1.0 \
--argjson package "$$pkg" \
--argjson postgres "$$pg" \
--argjson platform "$$plat" \
'$$ARGS.named' > $@
# Based on install in https://github.com/postgres/postgres/blob/REL_17_BETA1/src/makefiles/pgxs.mk#L237C1-L276
package: all pkg_dirs $(pkg_dir)/trunk.json
ifneq (,$(EXTENSION))
$(INSTALL_DATA) $(addprefix $(srcdir)/, $(addsuffix .control, $(EXTENSION))) '$(pkg_sharedir)/extension/'
endif # EXTENSION
ifneq (,$(DATA)$(DATA_built))
$(INSTALL_DATA) $(addprefix $(srcdir)/, $(DATA)) $(DATA_built) '$(pkg_sharedir)/$(datamoduledir)/'
endif # DATA
ifneq (,$(DATA_TSEARCH))
$(INSTALL_DATA) $(addprefix $(srcdir)/, $(DATA_TSEARCH)) '$(pkg_sharedir)/tsearch_data/'
endif # DATA_TSEARCH
ifdef MODULES
$(INSTALL_SHLIB) $(addsuffix $(DLSUFFIX), $(MODULES)) '$(pkg_pkglibdir)/'
ifeq ($(with_llvm), yes)
$(foreach mod, $(MODULES), $(call package_llvm_module,$(mod),$(mod).bc))
endif # with_llvm
endif # MODULES
ifdef DOCS
ifdef docdir
$(INSTALL_DATA) $(addprefix $(srcdir)/, $(DOCS)) '$(pkg_docdir)/'
endif # docdir
endif # DOCS
ifdef PROGRAM
$(INSTALL_PROGRAM) $(PROGRAM)$(X) '$(pkg_bindir)'
endif # PROGRAM
ifdef SCRIPTS
$(INSTALL_SCRIPT) $(addprefix $(srcdir)/, $(SCRIPTS)) '$(pkg_bindir)/'
endif # SCRIPTS
ifdef SCRIPTS_built
$(INSTALL_SCRIPT) $(SCRIPTS_built) '$(pkg_bindir)/'
endif # SCRIPTS_built
ifneq (,$(strip $(HEADER_dirs)))
$(foreach dir,$(HEADER_dirs),$(call package_headers,$(dir),$(HEADER_files_$(dir))))
endif # HEADERS
ifdef MODULE_big
ifeq ($(with_llvm), yes)
$(call package_llvm_module,$(MODULE_big),$(OBJS))
endif # with_llvm
package: package-lib
endif # MODULE_big
ifneq (,$(pkg_info_files))
$(INSTALL_DATA) $(addprefix $(srcdir)/, $(pkg_info_files)) '$(pkg_dir)/'
endif
rm -f "$(pkg_dir)/digests"
cd "$(pkg_dir)/" && find * -type f | xargs shasum --tag -ba 256 > digests
# Based on installdirs in https://github.com/postgres/postgres/blob/REL_17_BETA1/src/makefiles/pgxs.mk#L279C1-L303
pkg_dirs:
ifneq (,$(EXTENSION))
$(MKDIR_P) '$(pkg_sharedir)/extension'
endif
ifneq (,$(DATA)$(DATA_built))
$(MKDIR_P) '$(pkg_sharedir)/$(datamoduledir)'
endif
ifneq (,$(DATA_TSEARCH))
$(MKDIR_P) '$(pkg_sharedir)/tsearch_data'
endif
ifneq (,$(MODULES))
$(MKDIR_P) '$(pkg_pkglibdir)'
endif
ifdef DOCS
$(MKDIR_P) '$(pkg_docdir)'
endif
ifneq (,$(PROGRAM)$(SCRIPTS)$(SCRIPTS_built))
$(MKDIR_P) '$(pkg_bindir)'
endif
ifdef MODULE_big
pkg_dirs: pkg_dirs-lib
endif # MODULE_big
# Based on https://github.com/postgres/postgres/blob/REL_17_BETA1/src/Makefile.shlib#L364-L399.
.PHONY: package-lib package-lib-static package-lib-shared installdirs-lib
package-lib: package-lib-shared
ifdef soname
package-lib: package-lib-static
package-lib: package-lib-pc
endif
package-lib-pc: lib$(NAME).pc installdirs-lib
$(INSTALL_DATA) $< '$(pkg_pkgconfigdir)/lib$(NAME).pc'
package-lib-static: $(stlib) installdirs-lib
$(INSTALL_STLIB) $< '$(pkg_libdir)/$(stlib)'
package-lib-shared: $(shlib) installdirs-lib
ifdef soname
# we don't install $(shlib) on AIX
# (see http://archives.postgresql.org/message-id/52EF20B2E3209443BC37736D00C3C1380A6E79FE@EXADV1.host.magwien.gv.at)
ifneq ($(PORTNAME), aix)
$(INSTALL_SHLIB) $< '$(pkg_libdir)/$(shlib)'
ifneq ($(PORTNAME), cygwin)
ifneq ($(PORTNAME), win32)
ifneq ($(shlib), $(shlib_major))
cd '$(pkg_libdir)' && \
rm -f $(shlib_major) && \
$(LN_S) $(shlib) $(shlib_major)
endif
ifneq ($(shlib), $(shlib_bare))
cd '$(pkg_libdir)' && \
rm -f $(shlib_bare) && \
$(LN_S) $(shlib) $(shlib_bare)
endif
endif # not win32
endif # not cygwin
endif # not aix
ifneq (,$(findstring $(PORTNAME),win32 cygwin))
$(INSTALL_SHLIB) $< '$(DESTDIR)$(bindir)/$(shlib)'
endif
else # no soname
$(INSTALL_SHLIB) $< '$(DESTDIR)$(pkglibdir)/$(shlib)'
endif
# Based on installdirs-lib in https://github.com/postgres/postgres/blob/REL_17_BETA1/src/Makefile.shlib#L402-L407.
pkg_dirs-lib:
ifdef soname
$(MKDIR_P) '$(pkg_libdir)' '$(pkg_pkgconfigdir)' $(if $(findstring $(PORTNAME),win32 cygwin),'$(pkg_bindir)')
else
$(MKDIR_P) '$(pkg_pkgconfigdir)'
endif
# Based on install_llvm_module in https://github.com/postgres/postgres/blob/REL_17_BETA1/src/Makefile.global.in#L1090-L1107.
define package_llvm_module
$(MKDIR_P) '$(DESTDIR)${bitcodedir}/$(1)'
$(MKDIR_P) $(sort $(dir $(addprefix '$(pkg_pkglibdir)/bitcode'/$(1)/, $(2))))
$(foreach obj, ${2}, $(INSTALL_DATA) $(patsubst %.o,%.bc, $(obj)) '$(pkg_pkglibdir)/bitcode'/$(1)/$(dir $(obj))
)
cd '$(pkg_pkglibdir)/bitcode' && $(LLVM_BINPATH)/llvm-lto -thinlto -thinlto-action=thinlink -o $(1).index.bc $(addprefix $(1)/,$(patsubst %.o,%.bc, $(2)))
endef
# Based on install_headers in https://github.com/postgres/postgres/blob/REL_17_BETA1/src/makefiles/pgxs.mk#L202-L207
define package_headers
$(MKDIR_P) '$(pkg_incdir)/$(1)/'
$(INSTALL_DATA) $(2) '$(pkg_incdir)/$(1)/'
endef
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment