Skip to content

Instantly share code, notes, and snippets.

@rrbutani

rrbutani/..bazel-host-paths-experiment

Last active July 23, 2024 18:09
Show Gist options
  • Save rrbutani/f42e3d9f7dd51951c43b658ce7a6d402 to your computer and use it in GitHub Desktop.
Save rrbutani/f42e3d9f7dd51951c43b658ce7a6d402 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
..bazel-host-paths-experiment
Meant to be used together with this Bazel fork:
- https://github.com/rrbutani/bazel/compare/70fc57d29ce21204f28f8462627badbd5b750787...b8553d4fcf157acddbac855a3945c74d0f962c27
Raw
.bazelrc
common --enable_bzlmod=true
## CC:
common --incompatible_enable_cc_toolchain_resolution
common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
## Sandboxing:
# https://bazel.build/docs/sandboxing
common --reuse_sandbox_directories=true
common --incompatible_strict_action_env=true
common --incompatible_exclusive_test_sandboxed=true
# disallow `local`
#
# while we don't specify `linux-sandbox`/`darwin-sandbox` explicitly here we
# expect that Bazel will use it when possible (i.e. unless the user is in a
# container, etc.)
common --spawn_strategy=remote,worker,sandboxed
common --incompatible_legacy_local_fallback=false
common:aggressive_sandboxing --incompatible_sandbox_hermetic_tmp=true
common:aggressive_sandboxing --experimental_use_hermetic_linux_sandbox=true
common:aggressive_sandboxing --sandbox_default_allow_network=false
common:aggressive_sandboxing --sandbox_fake_hostname=true
common:aggressive_sandboxing --sandbox_fake_username=true
# common:aggressive_sandboxing --sandbox_add_mount_pair=/etc/ld.so.cache:/etc/ld.so.cache # necessary on EC
# common:aggressive_sandboxing --sandbox_add_mount_pair=/nix/store/:/nix/store/
# sandbox_add_mount_pair, sandbox_block_path
common --config=aggressive_sandboxing
common --extra_toolchains=//:local_sh_toolchain
common --extra_toolchains=//:local_perl_5_toolchain
# TODO: output_filter!
# hide all `info` messages, for example
# https://github.com/bazelbuild/bazel/issues/8455
Raw
.gitignore
/bazel-*
.bazel
/*.log
# empty file to appease the bzl LSP
WORKSPACE
MODULE.bazel.lock
Raw
bin_repo.bzl
def _bin_repo_impl(rctx):
base_path = rctx.attr.path
base_path = rctx.path(base_path)
for b in rctx.attr.cmds:
bin_path = base_path.get_child(b)
rctx.symlink(bin_path, "bin/" + b)
rctx.file("BUILD.bazel", """
filegroup(
name = "binaries",
srcs = glob(["bin/*"]),
)
exports_files(glob(["bin/*"]))
""", False)
bin_repo = repository_rule(
implementation = _bin_repo_impl,
environ = [],
attrs = {
"cmds": attr.string_list(),
"path": attr.string(),
},
configure = True,
local = True,
)
Raw
BUILD.bazel
################################################################################
load("@rules_sh//sh:sh.bzl", "sh_binaries")
load("@rules_sh//sh/experimental:posix_hermetic.bzl", "sh_posix_hermetic_toolchain")
posix_binaries = [
"awk", "basename", "bc", "cat", "chgrp", "chmod", "chown", "cp", "cut",
"diff", "dirname", "echo", "env", "expand", "expr", "false", "find",
"fold", "grep", "head", "iconv", "join", "kill", "link", "ln", "locale",
"ls", "m4", "mkdir", "mkfifo", "more", "mv", "nice", "od", "paste", "patch",
"printf", "pwd", "rm", "rmdir", "sed", "sh", "sleep", "sort", "split",
"tail", "tee", "touch", "tr", "true", "uname", "unlink", "wc", "xargs"
]
sh_binaries(
name = "commands",
deps = [ "@host_deps//:{}".format(b) for b in posix_binaries ],
)
sh_posix_hermetic_toolchain(
name = "local_sh",
cmds = ":commands",
)
toolchain(
name = "local_sh_toolchain",
toolchain = ":local_sh",
toolchain_type = "@rules_sh//sh/posix:toolchain_type",
# exec_compatible_with = [ ... ],
# target_compatible_with = [ ... ],
)
################################################################################
load("@rules_perl//perl:toolchain.bzl", "perl_toolchain")
perl_toolchain(
name = "local_perl_5_toolchain_impl",
runtime = [
"@host_perl//:bin/perl",
# dylib deps of perl:
"@host_perl//:lib64/5.26.1/x86_64-linux-thread-multi/CORE/libperl.so",
# these are taken from `host_deps_info.bzl`; relisted here to be
# explicit (and to use `libperl` from `@host_perl` instead)
"@host_deps//:ld-2",
"@host_deps//:libpthread-2",
"@host_deps//:libnsl-2",
"@host_deps//:libdl-2",
"@host_deps//:libm-2",
"@host_deps//:libcrypt-2",
"@host_deps//:libutil-2",
"@host_deps//:libc-2",
# Other bins:
"@host_perl//:bin/xsubpp",
# Headers:
"@host_perl//:core_headers",
# "@host_deps//:",
# "@host_deps//:perl",
],
)
toolchain(
name = "local_perl_5_toolchain",
toolchain = ":local_perl_5_toolchain_impl",
toolchain_type = "@rules_perl//:toolchain_type",
# exec_compatible_with = [ ... ],
# target_compatible_with = [ ... ],
)
################################################################################
alias(
name = "host_bash",
actual = "@host_deps//:bash",
)
################################################################################
exports_files(["update-shared-libs.py"])
################################################################################
genrule(
name = "test",
outs = ["out"],
# cmd = "$(BASH) -c 'echo yo > $@'",
# cmd = "bash -c 'echo yo > $@' ",
# cmd = "$(POSIX_BASENAME) -c 'echo yo > $@'",
# cmd = "LD_VERBOSE=1 LD_DEBUG=libs strace -o $@ -f /lib64/ld-linux-x86-64.so.2 --list /bin/ls; echo /bin/l* /etc/ld.so.* /lib/* /lib64/noelision/*; /bin/ls -l external/bash/lib/bash/basename; ls -l /tmp/bazel-source-roots/0/bin; ldd external/bash/lib/bash/basename",
# cmd = "LD_VERBOSE=1 LD_DEBUG=libs strace -o $@ -f /lib64/ld-linux-x86-64.so.2 --list /bin/ls",
# cmd = "LD_VERBOSE=1 LD_DEBUG=libs /lib64/ld-linux-x86-64.so.2 --list $(POSIX_BASENAME)",
# cmd = "$(POSIX_LS) /; $(PERL_PERL) --help; $(PERL) --help",
cmd = "$(POSIX_LS) /; $(PERL) --help",
# cmd = "$(POSIX_LS) /",
# cmd = "echo /",
# cmd = "LD_VERBOSE=1 LD_DEBUG=libs /lib64/ld-linux-x86-64.so.2 --list $(POSIX_LS)",
# cmd = "/usr/bin/strace",
tools = [
# "@host_bash//:bin/bash",
":host_bash",
],
toolchains = [
"@rules_sh//sh/posix:make_variables",
# "@host_deps//:perl",
"@rules_perl//:current_toolchain",
],
)
Raw
host_deps_info.bzl
# Generated by `update-shared-libs.py`; prefer rerunning the script to modifying
# this file manually.
#
# Run as: `bazel run @host_deps//:regenerate`
#
# You can also pass `-- <extra binaries to add>` to the above invocation.
# Modifications to this list *will* persist when you run `:regenerate`.
#
# However, comments and starlark constructs will *not* be preserved. See
# `EXTRA_BINARIES` below if this is required.
BINARIES = [
"/bin/bash",
"/usr/bin/awk",
"/usr/bin/basename",
"/usr/bin/bc",
"/usr/bin/cat",
"/usr/bin/chgrp",
"/usr/bin/chmod",
"/usr/bin/chown",
"/usr/bin/cp",
"/usr/bin/cut",
"/usr/bin/diff",
"/usr/bin/dirname",
"/usr/bin/echo",
"/usr/bin/env",
"/usr/bin/expand",
"/usr/bin/expr",
"/usr/bin/false",
"/usr/bin/find",
"/usr/bin/fold",
"/usr/bin/grep",
"/usr/bin/head",
"/usr/bin/iconv",
"/usr/bin/join",
"/usr/bin/kill",
"/usr/bin/link",
"/usr/bin/ln",
"/usr/bin/locale",
"/usr/bin/ls",
"/usr/bin/m4",
"/usr/bin/mkdir",
"/usr/bin/mkfifo",
"/usr/bin/more",
"/usr/bin/mv",
"/usr/bin/nice",
"/usr/bin/od",
"/usr/bin/paste",
"/usr/bin/patch",
"/usr/bin/printf",
"/usr/bin/pwd",
"/usr/bin/rm",
"/usr/bin/rmdir",
"/usr/bin/sed",
"/usr/bin/sh",
"/usr/bin/sleep",
"/usr/bin/sort",
"/usr/bin/split",
"/usr/bin/tail",
"/usr/bin/tee",
"/usr/bin/touch",
"/usr/bin/tr",
"/usr/bin/true",
"/usr/bin/uname",
"/usr/bin/unlink",
"/usr/bin/wc",
"/usr/bin/xargs",
]
REPO_NAME = "host_deps"
################################################################################
# begin: preserved
# Preserved Section
#
# The contents of this section are preserved verbatim.
# Extra binaries to pass to `update-shared-libs.py`.
#
# These will not land in `BINARIES` in the regenerated version of this file; use
# this for binary paths that, for example, come from non-constant starlark
# constructs.
EXTRA_BINARIES = [
"/usr/intel/pkgs/perl/5.26.1-threads/bin/perl",
"/usr/intel/pkgs/bash/5.2.15/bin/bash",
]
# Extra symlinks for the repository rule to create.
#
# Map from source path to (repo relative) dest path.
EXTRA_SYMLINKS = {
"/etc/ld.so.cache": "etc/ld.so.cache",
}
# Extra data dependencies to add to the library filegroups.
#
# Map from the absolute realpath label of the library to a list of labels.
EXTRA_LIB_DATA_DEPS = {
# Necessary so `ld.so` can find non-standard paths like our pthread library.
":lib64/ld-2.22.so": [":etc/ld.so.cache"],
}
# end: preserved
################################################################################
def _mk_regen_script(ctx):
script = ctx.executable.script
script_runfiles = ctx.attr.script[DefaultInfo].default_runfiles
args = [
"'./{}'".format(script.path),
"--repo-name '{}'".format(ctx.attr.repo_name),
"--package '{}'".format(ctx.attr.package),
"--file-path '{}'".format(ctx.attr.file_path),
"--package-within-main-repo-for-script '{}'".format(ctx.attr.script_package),
"--file-path-within-package-for-script '{}'".format(ctx.attr.script_file_path),
"--show-diff",
] + [
"'{}'".format(bin) for bin in ctx.attr.binaries
] + [
"--extra-binary '{}'".format(bin) for bin in ctx.attr.extra_binaries
] + [ '"${@}"' ]
out = "#!/usr/bin/env bash\n"
out += "exec {}".format(" ".join(args))
update_script = ctx.actions.declare_file(ctx.label.name)
ctx.actions.write(update_script, out, is_executable = True)
runfiles = ctx.runfiles([ctx.executable.script]).merge(script_runfiles)
return [
DefaultInfo(executable = update_script, runfiles = runfiles)
]
make_regenerate_script = rule(
implementation = _mk_regen_script,
attrs = {
"script": attr.label(
executable = True,
cfg = "exec",
allow_single_file = True,
),
"binaries": attr.string_list(allow_empty = True),
"extra_binaries": attr.string_list(allow_empty = True),
"repo_name": attr.string(),
"package": attr.string(),
"file_path": attr.string(),
"script_package": attr.string(),
"script_file_path": attr.string(),
},
executable = True,
)
def create_symlinks(rctx):
### Libraries
rctx.symlink("/lib64/ld-2.22.so", "lib64/ld-2.22.so")
rctx.symlink("/lib64/ld-linux-x86-64.so.2", "lib64/ld-linux-x86-64.so.2")
rctx.symlink("/lib64/libc-2.22.so", "lib64/libc-2.22.so")
rctx.symlink("/lib64/libc.so.6", "lib64/libc.so.6")
rctx.symlink("/lib64/libtinfo.so.5.9", "lib64/libtinfo.so.5.9")
rctx.symlink("/lib64/libtinfo.so.5", "lib64/libtinfo.so.5")
rctx.symlink("/lib64/libreadline.so.6.3", "lib64/libreadline.so.6.3")
rctx.symlink("/lib64/libreadline.so.6", "lib64/libreadline.so.6")
rctx.symlink("/lib64/libdl-2.22.so", "lib64/libdl-2.22.so")
rctx.symlink("/lib64/libdl.so.2", "lib64/libdl.so.2")
rctx.symlink("/lib64/libm-2.22.so", "lib64/libm-2.22.so")
rctx.symlink("/lib64/libm.so.6", "lib64/libm.so.6")
rctx.symlink("/lib64/noelision/libpthread-2.22.so", "lib64/noelision/libpthread-2.22.so")
rctx.symlink("/lib64/noelision/libpthread.so.0", "lib64/noelision/libpthread.so.0")
rctx.symlink("/usr/lib64/libpcre.so.1.2.13", "usr/lib64/libpcre.so.1.2.13")
rctx.symlink("/usr/lib64/libpcre.so.1", "usr/lib64/libpcre.so.1")
rctx.symlink("/lib64/libselinux.so.1", "lib64/libselinux.so.1")
rctx.symlink("/lib64/libattr.so.1.1.0", "lib64/libattr.so.1.1.0")
rctx.symlink("/lib64/libattr.so.1", "lib64/libattr.so.1")
rctx.symlink("/lib64/libacl.so.1.1.0", "lib64/libacl.so.1.1.0")
rctx.symlink("/lib64/libacl.so.1", "lib64/libacl.so.1")
rctx.symlink("/usr/lib64/libgmp.so.10.1.3", "usr/lib64/libgmp.so.10.1.3")
rctx.symlink("/usr/lib64/libgmp.so.10", "usr/lib64/libgmp.so.10")
rctx.symlink("/lib64/libcap.so.2.26", "lib64/libcap.so.2.26")
rctx.symlink("/lib64/libcap.so.2", "lib64/libcap.so.2")
rctx.symlink("/lib64/libnsl-2.22.so", "lib64/libnsl-2.22.so")
rctx.symlink("/lib64/libnsl.so.1", "lib64/libnsl.so.1")
rctx.symlink("/lib64/libcrypt-2.22.so", "lib64/libcrypt-2.22.so")
rctx.symlink("/lib64/libcrypt.so.1", "lib64/libcrypt.so.1")
rctx.symlink("/lib64/libutil-2.22.so", "lib64/libutil-2.22.so")
rctx.symlink("/lib64/libutil.so.1", "lib64/libutil.so.1")
rctx.symlink("/usr/intel/pkgs/perl/5.26.1-threads/lib64/5.26.1/x86_64-linux-thread-multi/CORE/libperl.so", "usr/intel/pkgs/perl/5.26.1-threads/lib64/5.26.1/x86_64-linux-thread-multi/CORE/libperl.so")
### Binaries
rctx.symlink("/bin/bash", "bin/bash")
rctx.symlink("/usr/bin/awk", "usr/bin/awk")
rctx.symlink("/usr/bin/basename", "usr/bin/basename")
rctx.symlink("/usr/bin/bc", "usr/bin/bc")
rctx.symlink("/usr/bin/cat", "usr/bin/cat")
rctx.symlink("/usr/bin/chgrp", "usr/bin/chgrp")
rctx.symlink("/usr/bin/chmod", "usr/bin/chmod")
rctx.symlink("/usr/bin/chown", "usr/bin/chown")
rctx.symlink("/usr/bin/cp", "usr/bin/cp")
rctx.symlink("/usr/bin/cut", "usr/bin/cut")
rctx.symlink("/usr/bin/diff", "usr/bin/diff")
rctx.symlink("/usr/bin/dirname", "usr/bin/dirname")
rctx.symlink("/usr/bin/echo", "usr/bin/echo")
rctx.symlink("/usr/bin/env", "usr/bin/env")
rctx.symlink("/usr/bin/expand", "usr/bin/expand")
rctx.symlink("/usr/bin/expr", "usr/bin/expr")
rctx.symlink("/usr/bin/false", "usr/bin/false")
rctx.symlink("/usr/bin/find", "usr/bin/find")
rctx.symlink("/usr/bin/fold", "usr/bin/fold")
rctx.symlink("/usr/bin/grep", "usr/bin/grep")
rctx.symlink("/usr/bin/head", "usr/bin/head")
rctx.symlink("/usr/bin/iconv", "usr/bin/iconv")
rctx.symlink("/usr/bin/join", "usr/bin/join")
rctx.symlink("/usr/bin/kill", "usr/bin/kill")
rctx.symlink("/usr/bin/link", "usr/bin/link")
rctx.symlink("/usr/bin/ln", "usr/bin/ln")
rctx.symlink("/usr/bin/locale", "usr/bin/locale")
rctx.symlink("/usr/bin/ls", "usr/bin/ls")
rctx.symlink("/usr/bin/m4", "usr/bin/m4")
rctx.symlink("/usr/bin/mkdir", "usr/bin/mkdir")
rctx.symlink("/usr/bin/mkfifo", "usr/bin/mkfifo")
rctx.symlink("/usr/bin/more", "usr/bin/more")
rctx.symlink("/usr/bin/mv", "usr/bin/mv")
rctx.symlink("/usr/bin/nice", "usr/bin/nice")
rctx.symlink("/usr/bin/od", "usr/bin/od")
rctx.symlink("/usr/bin/paste", "usr/bin/paste")
rctx.symlink("/usr/bin/patch", "usr/bin/patch")
rctx.symlink("/usr/bin/printf", "usr/bin/printf")
rctx.symlink("/usr/bin/pwd", "usr/bin/pwd")
rctx.symlink("/usr/bin/rm", "usr/bin/rm")
rctx.symlink("/usr/bin/rmdir", "usr/bin/rmdir")
rctx.symlink("/usr/bin/sed", "usr/bin/sed")
rctx.symlink("/usr/bin/sh", "usr/bin/sh")
rctx.symlink("/usr/bin/sleep", "usr/bin/sleep")
rctx.symlink("/usr/bin/sort", "usr/bin/sort")
rctx.symlink("/usr/bin/split", "usr/bin/split")
rctx.symlink("/usr/bin/tail", "usr/bin/tail")
rctx.symlink("/usr/bin/tee", "usr/bin/tee")
rctx.symlink("/usr/bin/touch", "usr/bin/touch")
rctx.symlink("/usr/bin/tr", "usr/bin/tr")
rctx.symlink("/usr/bin/true", "usr/bin/true")
rctx.symlink("/usr/bin/uname", "usr/bin/uname")
rctx.symlink("/usr/bin/unlink", "usr/bin/unlink")
rctx.symlink("/usr/bin/wc", "usr/bin/wc")
rctx.symlink("/usr/bin/xargs", "usr/bin/xargs")
rctx.symlink("/usr/intel/pkgs/perl/5.26.1-threads/bin/perl", "usr/intel/pkgs/perl/5.26.1-threads/bin/perl")
rctx.symlink("/usr/intel/pkgs/bash/5.2.15/bin/bash", "usr/intel/pkgs/bash/5.2.15/bin/bash")
### Extra
for src, dest in EXTRA_SYMLINKS.items(): rctx.symlink(src, dest)
_BUILD_FILE = '''
load("@@//:host_deps_info.bzl", "define_targets")
load("@rules_sh//sh:sh.bzl", "sh_binaries")
package(
default_visibility = ["//visibility:public"],
)
define_targets(filegroup, sh_binaries)
'''
def _repo_impl(rctx):
create_symlinks(rctx)
rctx.file("BUILD.bazel", _BUILD_FILE, executable = False)
def make_repo():
host_deps_repo = repository_rule(
implementation = _repo_impl,
environ = [],
configure = True,
)
host_deps_repo(name = REPO_NAME)
def define_targets(filegroup, sh_binary):
### Libraries:
filegroup_ = lambda *args, **kw: filegroup(
*args,
**(kw | (
{'data': kw.get('data', []) + EXTRA_LIB_DATA_DEPS.get(kw['srcs'][0], []) }
if 'srcs' in kw else {}
)),
)
filegroup_(
name = "ld-2",
srcs = [ ":lib64/ld-2.22.so" ] + [
# Symlinks:
":lib64/ld-linux-x86-64.so.2",
],
data = [],
)
filegroup_(
name = "libc-2",
srcs = [ ":lib64/libc-2.22.so" ] + [
# Symlinks:
":lib64/libc.so.6",
],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
],
)
filegroup_(
name = "libtinfo",
srcs = [ ":lib64/libtinfo.so.5.9" ] + [
# Symlinks:
":lib64/libtinfo.so.5",
],
data = [
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libreadline",
srcs = [ ":lib64/libreadline.so.6.3" ] + [
# Symlinks:
":lib64/libreadline.so.6",
],
data = [
# `libtinfo.so.5` as `/lib64/libtinfo.so.5`
":libtinfo",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libdl-2",
srcs = [ ":lib64/libdl-2.22.so" ] + [
# Symlinks:
":lib64/libdl.so.2",
],
data = [
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
],
)
filegroup_(
name = "libm-2",
srcs = [ ":lib64/libm-2.22.so" ] + [
# Symlinks:
":lib64/libm.so.6",
],
data = [
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libpthread-2",
srcs = [ ":lib64/noelision/libpthread-2.22.so" ] + [
# Symlinks:
":lib64/noelision/libpthread.so.0",
],
data = [
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
],
)
filegroup_(
name = "libpcre",
srcs = [ ":usr/lib64/libpcre.so.1.2.13" ] + [
# Symlinks:
":usr/lib64/libpcre.so.1",
],
data = [
# `libpthread.so.0` as `/lib64/noelision/libpthread.so.0`
":libpthread-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libselinux",
srcs = [ ":lib64/libselinux.so.1" ],
data = [
# `libpcre.so.1` as `/usr/lib64/libpcre.so.1`
":libpcre",
# `libdl.so.2` as `/lib64/libdl.so.2`
":libdl-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
],
)
filegroup_(
name = "libattr",
srcs = [ ":lib64/libattr.so.1.1.0" ] + [
# Symlinks:
":lib64/libattr.so.1",
],
data = [
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libacl",
srcs = [ ":lib64/libacl.so.1.1.0" ] + [
# Symlinks:
":lib64/libacl.so.1",
],
data = [
# `libattr.so.1` as `/lib64/libattr.so.1`
":libattr",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libgmp",
srcs = [ ":usr/lib64/libgmp.so.10.1.3" ] + [
# Symlinks:
":usr/lib64/libgmp.so.10",
],
data = [
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libcap",
srcs = [ ":lib64/libcap.so.2.26" ] + [
# Symlinks:
":lib64/libcap.so.2",
],
data = [
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libnsl-2",
srcs = [ ":lib64/libnsl-2.22.so" ] + [
# Symlinks:
":lib64/libnsl.so.1",
],
data = [
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libcrypt-2",
srcs = [ ":lib64/libcrypt-2.22.so" ] + [
# Symlinks:
":lib64/libcrypt.so.1",
],
data = [
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libutil-2",
srcs = [ ":lib64/libutil-2.22.so" ] + [
# Symlinks:
":lib64/libutil.so.1",
],
data = [
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup_(
name = "libperl",
srcs = [ ":usr/intel/pkgs/perl/5.26.1-threads/lib64/5.26.1/x86_64-linux-thread-multi/CORE/libperl.so" ],
data = [
# `libpthread.so.0` as `/lib64/noelision/libpthread.so.0`
":libpthread-2",
# `libnsl.so.1` as `/lib64/libnsl.so.1`
":libnsl-2",
# `libdl.so.2` as `/lib64/libdl.so.2`
":libdl-2",
# `libm.so.6` as `/lib64/libm.so.6`
":libm-2",
# `libcrypt.so.1` as `/lib64/libcrypt.so.1`
":libcrypt-2",
# `libutil.so.1` as `/lib64/libutil.so.1`
":libutil-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
filegroup(
name = "all-libraries",
data = [
":ld-2",
":libc-2",
":libtinfo",
":libreadline",
":libdl-2",
":libm-2",
":libpthread-2",
":libpcre",
":libselinux",
":libattr",
":libacl",
":libgmp",
":libcap",
":libnsl-2",
":libcrypt-2",
":libutil-2",
":libperl",
],
)
### Binaries:
sh_binary(
name = "bin__bash",
srcs = [":bin/bash"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libreadline.so.6` as `/lib64/libreadline.so.6`
":libreadline",
# `libtinfo.so.5` as `/lib64/libtinfo.so.5`
":libtinfo",
# `libdl.so.2` as `/lib64/libdl.so.2`
":libdl-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "awk",
srcs = [":usr/bin/awk"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libdl.so.2` as `/lib64/libdl.so.2`
":libdl-2",
# `libm.so.6` as `/lib64/libm.so.6`
":libm-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "basename",
srcs = [":usr/bin/basename"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "bc",
srcs = [":usr/bin/bc"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libreadline.so.6` as `/lib64/libreadline.so.6`
":libreadline",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "cat",
srcs = [":usr/bin/cat"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "chgrp",
srcs = [":usr/bin/chgrp"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "chmod",
srcs = [":usr/bin/chmod"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "chown",
srcs = [":usr/bin/chown"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "cp",
srcs = [":usr/bin/cp"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libselinux.so.1` as `/lib64/libselinux.so.1`
":libselinux",
# `libacl.so.1` as `/lib64/libacl.so.1`
":libacl",
# `libattr.so.1` as `/lib64/libattr.so.1`
":libattr",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "cut",
srcs = [":usr/bin/cut"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "diff",
srcs = [":usr/bin/diff"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "dirname",
srcs = [":usr/bin/dirname"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "echo",
srcs = [":usr/bin/echo"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "env",
srcs = [":usr/bin/env"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "expand",
srcs = [":usr/bin/expand"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "expr",
srcs = [":usr/bin/expr"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libgmp.so.10` as `/usr/lib64/libgmp.so.10`
":libgmp",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "false",
srcs = [":usr/bin/false"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "find",
srcs = [":usr/bin/find"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libselinux.so.1` as `/lib64/libselinux.so.1`
":libselinux",
# `libm.so.6` as `/lib64/libm.so.6`
":libm-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "fold",
srcs = [":usr/bin/fold"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "grep",
srcs = [":usr/bin/grep"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libpcre.so.1` as `/usr/lib64/libpcre.so.1`
":libpcre",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "head",
srcs = [":usr/bin/head"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "iconv",
srcs = [":usr/bin/iconv"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "join",
srcs = [":usr/bin/join"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "kill",
srcs = [":usr/bin/kill"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "link",
srcs = [":usr/bin/link"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "ln",
srcs = [":usr/bin/ln"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "locale",
srcs = [":usr/bin/locale"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "ls",
srcs = [":usr/bin/ls"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libselinux.so.1` as `/lib64/libselinux.so.1`
":libselinux",
# `libcap.so.2` as `/lib64/libcap.so.2`
":libcap",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "m4",
srcs = [":usr/bin/m4"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "mkdir",
srcs = [":usr/bin/mkdir"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libselinux.so.1` as `/lib64/libselinux.so.1`
":libselinux",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "mkfifo",
srcs = [":usr/bin/mkfifo"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libselinux.so.1` as `/lib64/libselinux.so.1`
":libselinux",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "more",
srcs = [":usr/bin/more"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libtinfo.so.5` as `/lib64/libtinfo.so.5`
":libtinfo",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "mv",
srcs = [":usr/bin/mv"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libselinux.so.1` as `/lib64/libselinux.so.1`
":libselinux",
# `libacl.so.1` as `/lib64/libacl.so.1`
":libacl",
# `libattr.so.1` as `/lib64/libattr.so.1`
":libattr",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "nice",
srcs = [":usr/bin/nice"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "od",
srcs = [":usr/bin/od"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "paste",
srcs = [":usr/bin/paste"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "patch",
srcs = [":usr/bin/patch"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "printf",
srcs = [":usr/bin/printf"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "pwd",
srcs = [":usr/bin/pwd"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "rm",
srcs = [":usr/bin/rm"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "rmdir",
srcs = [":usr/bin/rmdir"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "sed",
srcs = [":usr/bin/sed"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "sh",
srcs = [":usr/bin/sh"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libreadline.so.6` as `/lib64/libreadline.so.6`
":libreadline",
# `libtinfo.so.5` as `/lib64/libtinfo.so.5`
":libtinfo",
# `libdl.so.2` as `/lib64/libdl.so.2`
":libdl-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "sleep",
srcs = [":usr/bin/sleep"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "sort",
srcs = [":usr/bin/sort"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libpthread.so.0` as `/lib64/noelision/libpthread.so.0`
":libpthread-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "split",
srcs = [":usr/bin/split"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "tail",
srcs = [":usr/bin/tail"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "tee",
srcs = [":usr/bin/tee"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "touch",
srcs = [":usr/bin/touch"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "tr",
srcs = [":usr/bin/tr"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "true",
srcs = [":usr/bin/true"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "uname",
srcs = [":usr/bin/uname"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "unlink",
srcs = [":usr/bin/unlink"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "wc",
srcs = [":usr/bin/wc"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "xargs",
srcs = [":usr/bin/xargs"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "perl",
srcs = [":usr/intel/pkgs/perl/5.26.1-threads/bin/perl"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libperl.so` as `/usr/intel/pkgs/perl/5.26.1-threads/lib64/5.26.1/x86_64-linux-thread-multi/CORE/libperl.so`
":libperl",
# `libpthread.so.0` as `/lib64/noelision/libpthread.so.0`
":libpthread-2",
# `libnsl.so.1` as `/lib64/libnsl.so.1`
":libnsl-2",
# `libdl.so.2` as `/lib64/libdl.so.2`
":libdl-2",
# `libm.so.6` as `/lib64/libm.so.6`
":libm-2",
# `libcrypt.so.1` as `/lib64/libcrypt.so.1`
":libcrypt-2",
# `libutil.so.1` as `/lib64/libutil.so.1`
":libutil-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "usr__intel__pkgs__bash__5.2.15__bin__bash",
srcs = [":usr/intel/pkgs/bash/5.2.15/bin/bash"],
data = [
# `ld-linux-x86-64.so.2` as `/lib64/ld-linux-x86-64.so.2`
":ld-2",
# `libtinfo.so.5` as `/lib64/libtinfo.so.5`
":libtinfo",
# `libdl.so.2` as `/lib64/libdl.so.2`
":libdl-2",
# `libc.so.6` as `/lib64/libc.so.6`
":libc-2",
],
)
sh_binary(
name = "all-binaries",
deps = [
":bin__bash",
":awk",
":basename",
":bc",
":cat",
":chgrp",
":chmod",
":chown",
":cp",
":cut",
":diff",
":dirname",
":echo",
":env",
":expand",
":expr",
":false",
":find",
":fold",
":grep",
":head",
":iconv",
":join",
":kill",
":link",
":ln",
":locale",
":ls",
":m4",
":mkdir",
":mkfifo",
":more",
":mv",
":nice",
":od",
":paste",
":patch",
":printf",
":pwd",
":rm",
":rmdir",
":sed",
":sh",
":sleep",
":sort",
":split",
":tail",
":tee",
":touch",
":tr",
":true",
":uname",
":unlink",
":wc",
":xargs",
":perl",
":usr__intel__pkgs__bash__5.2.15__bin__bash",
]
)
### Runnable target to regenerate:
make_regenerate_script(
name = "regenerate",
script = "@@//:update-shared-libs.py",
binaries = BINARIES,
extra_binaries = EXTRA_BINARIES,
repo_name = REPO_NAME,
package = "",
file_path = "host_deps_info.bzl",
script_package = "",
script_file_path = "update-shared-libs.py",
)
Raw
MODULE.bazel
module(name = "hermetic-sandbox-test")
bazel_dep(name = "rules_sh", version = "0.3.0")
use_extension("@rules_sh//bzlmod:extensions.bzl", "sh_configure").local_posix_config(
enable = False,
)
bazel_dep(name = "rules_perl", version = "0.1.0")
################################################################################
# Overrides:
# See: https://github.com/tweag/rules_sh/pull/50
git_override(
module_name = "rules_sh",
remote = "https://github.com/rrbutani/rules_sh.git",
commit = "b10905cef18292c371ca6968b0358e18e3fea8fc",
)
# See: https://github.com/bazelbuild/rules_perl/pull/55
# (not yet in the BCR)
git_override(
module_name = "rules_perl",
remote = "https://github.com/bazelbuild/rules_perl.git",
commit = "d458b41dd15a086721dcf663317f2c121bba8984",
# patches, # TODO: patch out the default toolchain registration
)
Raw
update-shared-libs.py
#!/usr/bin/env nix-shell
#! nix-shell -i python3.11 -p python311 pax-utils delta
# TODO: move to readme
"""
Generates a `.bzl` file that creates a Bazel repository describing the specified
host binaries and their shared object dependencies.
Note that we do not invoke this script as part of a repo rule in Bazel though
we _could_; the idea is that the description of the parts of the host system
that we depend upon shouldn't change often and should not change without us
noticing. This script exists because writing this description is tedious.
However, we should be auditing what this file emits.
## Usage
(you can run this without `nix`; you'll need to provide `lddtree` on $PATH
yourself though)
"""
import argparse
from collections import UserDict
from dataclasses import dataclass
import functools
import os.path
from pathlib import Path
import shutil
import subprocess
import sys
from tempfile import NamedTemporaryFile
from typing import Dict, Iterable, List, Optional, Self, Set, Tuple
parser = argparse.ArgumentParser(usage = (
"Generate a .bzl file, detailing host binaries and their shared " +
"object dependencies."
))
parser.add_argument(
'host_binaries', nargs = '+',
help = "list of host binaries to include in the generated repository",
)
parser.add_argument(
'--extra-binary', default = [], action = 'append',
help = "binary to include in the generated repository but to not include in the generated list of binaries -- subsequent regenerates will not automatically include this binary unless its addition is arranged somehow (i.e. `EXTRA_BINARIES`, command line argument)"
)
parser.add_argument(
'-n', "--repo-name", default = "host_deps",
help = "name of the repository the .bzl file should create",
)
parser.add_argument(
'-r', "--path-to-main-repo", default = None,
help = "path to the root of the main repository; this is where we'll put the generated .bzl file -- if None this will be inferred from the path to this script (presumed to live in the main repo)"
)
parser.add_argument(
'-p', "--package", default = "",
help = "package in the specified repository to emit the generated .bzl file to",
)
parser.add_argument(
'-f', "--file-path", default = "host_deps_info.bzl",
help = "path of the .bzl file to generate under `package`",
)
parser.add_argument(
"--package-within-main-repo-for-script", default = "build/bazel/utils/host_deps",
help = "package within the main repo where this file resides",
)
parser.add_argument(
"--file-path-within-package-for-script", default = "update_shared_libs.py",
help = "path to this script within the package it lives in",
)
parser.add_argument(
'-e', '--emit-on-stdout', default = False, action = argparse.BooleanOptionalAction,
help = "dump the generated .bzl file to stdout instead of replacing the file"
)
parser.add_argument(
'-d', "--show-diff", default = False, action = 'store_true',
help = "display what has changed between the existing copy of the .bzl file and what this script generated",
)
@functools.cache
def find_lddtree() -> Path:
if res := shutil.which("lddtree"):
return Path(res)
else:
raise Exception("unable to find `lddtree` on `$PATH`!")
@dataclass
class DynamicExecutable:
path: Path
deps: List[Tuple[str, Path, Path]] # (shortname, path -- *not* realpath, realpath)
def add_dep(self, shortname: str, path: Path, dep: 'SharedObject'):
self.deps.append((shortname, path, dep.path))
@dataclass
class SharedObject(DynamicExecutable):
aliases: Set[Path] # symlinks that point to this shared object
# in this class `.path` is the normalized realpath
# path here may be a symlink; we'll resolve it to a realpath (and record all
# the symlinks we encounter on the way)
def __init__(self, path: Path | str, deps: List[Tuple[str, Path]] = None):
if isinstance(path, str): path = Path(path)
orig_path = path
# defaults are "early bound"; we don't want instances of SharedObject to
# share the same list instance for `deps`
if deps is None: deps = []
assert path.exists()
aliases = []
while path.is_symlink():
aliases.append(path)
link = path.readlink()
if link.is_absolute():
path = link
else:
path = Path(os.path.normpath(os.path.join(path.parent, link)))
if (exp := orig_path.resolve()) != path:
print(f"warning: expected `{exp}`, got `{path}`; have aliases: `{aliases}`... there's likely a symlink within the path (should be okay)", file = sys.stderr)
self.path = path
self.aliases = set(aliases)
self.deps = deps
def basename(self) -> str: return self.path.name
def merge(self, other: Self) -> Self:
assert (
self.path == other.path and
self.deps == other.deps
), f"cannot merge; this: {self}, that: {other}"
res = SharedObject.__new__(SharedObject)
res.path = self.path
res.deps = self.deps
res.aliases = self.aliases.union(other.aliases)
return res
@dataclass
class Binary(DynamicExecutable): pass
class SharedObjectStore(UserDict[Path, SharedObject]):
def __setitem__(self, key: Path, item: SharedObject) -> None:
if key in self.data:
item = self.data[key].merge(item)
return super().__setitem__(key, item)
def add(self, item: SharedObject): self.__setitem__(item.path, item)
def __str__(self) -> str:
out = []
p = lambda i: out.append(i)
for path, so in self.data.items():
p(f"{str(path)} => {{")
assert path == so.path
if so.aliases:
p(f" aka:")
for alias in so.aliases:
p(f" → {alias}")
if so.deps:
p(f" deps:")
for shortname, _path, abs_path in so.deps:
p(f" → {shortname:30} at {str(abs_path):>40}")
p("}\n")
return "\n".join(out)
def run_lddtree(path: Path, store: SharedObjectStore) -> Binary:
print(f"Processing binary `{str(path)}`", file=sys.stderr)
try:
p = subprocess.run(
[find_lddtree(), "--all", path],
check=True,
capture_output=True,
)
except subprocess.CalledProcessError as ex:
err = ex.stderr.decode("utf-8")
print(f"Error when invoking `lddtree`: {err}", file = sys.stderr)
raise ex
first, *deps = p.stdout.decode("utf-8").splitlines()
# assuming the interpreter depends on nothing
bin_path, interpreter_path = first.split(" (interpreter => ")
interpreter_path = Path(interpreter_path[:-1])
assert bin_path == path
# walk through the dep tree:
stack = [Binary(Path(bin_path), [])]
interpreter = (
# in case a shared object is passed in, instead of a binary (i.e. there
# won't be an interpreter path; lddtree prints `None`)
[(1, interpreter_path.name, interpreter_path)]
if not str(interpreter_path) == "None" #
else []
)
deps = [(d.count(" "), *d.lstrip().split(" => ")) for d in deps]
for indent_level, shortname, so_path in interpreter + deps:
if indent_level > (depth := len(stack)):
assert False, f"can't skip indent levels: {depth} -> {indent_level}"
# Note: we defer adding to `store` until we've popped the shared object
# so that it has all its deps before its added to the store.
#
# This is important for the merging logic to work (we assert that the
# list of deps is the same...)
while len(stack) > indent_level: store.add(stack.pop())
so = SharedObject(so_path)
stack[-1].add_dep(shortname, so_path, so)
stack.append(so)
# Add any remaining shared objects to the store:
binary, *remaining = stack
for so in remaining: store.add(so)
return binary
# Loosely mimics the output of `lddtree`; for debugging.
def tree(
root: DynamicExecutable,
store: SharedObjectStore,
indent_level = 0,
shortname = None,
path = None,
):
p = lambda o: print(o, end = "")
p(" " * indent_level)
if not indent_level == 0: p("")
if shortname: p(f"{shortname} => ")
if path:
p(path)
if path != root.path: p(f" ({root.path})")
else:
p(root.path)
p("\n")
for (shortname, path, realpath) in root.deps:
tree(store[realpath], store, indent_level + 1, shortname, path)
def check_for_collisions(bins: Dict[str, Binary], store: SharedObjectStore):
# Ensure there are no collisions:
flattened_paths: Dict[Path, DynamicExecutable] = {}
def put(dict: Dict[Path, DynamicExecutable], src: DynamicExecutable, path: Path):
if path in dict:
other = dict[path]
raise ValueError(f"`{path}` from `{src}` collides with item at `{other.path}` (from {other})")
else:
dict[path] = src
for bin_path, bin in bins.items(): put(flattened_paths, bin, bin_path)
for so_path, so in store.items():
for p in [so_path] + list(so.aliases):
put(flattened_paths, so, p)
# Yields ["lib64__libc.so.6.2", "libc.so.6.2", "libc.so.6", "libc.so", "libc"]
# for `/lib64/libc.so.6.2`
def potential_labels(p: Path) -> Iterable[str]:
yield "__".join(filter(lambda p: p != "/", p.parts))
curr = p.name
while (s := Path(curr).stem) != curr:
yield curr
curr = s
yield curr
Potentials = List[str]
def assign_labels(bins: Dict[str, Binary], store: SharedObjectStore) -> Dict[str, DynamicExecutable]:
out = {}
def put(dict: Dict[str, DynamicExecutable], new: DynamicExecutable):
get_potential = lambda p: list(potential_labels(p))[::-1]
potentials: Potentials = get_potential(new.path)
# As we try to place `new` in the dict, we may collide with other
# entries; these entries are _displaced_.
#
# We'll use progressively longer labels for `new` and all the displaced
# entries until we find a mapping that has a unique label for all the
# entries.
#
# Note that if we end up using the absolute path fallback label for any
# of the displaced entries we will do so for _all_ the displaced
# entries; it would be confusing for there to be labels like
# `:libc.so.6` and `:lib64__libc.so` instead of `:usr__lib__libc.so.6.2`
# and `lib64__libc.so`.
displaced: List[Tuple[DynamicExecutable, Potentials]] = [
(new, potentials)
]
level: int
for i, _ in enumerate(potentials):
use_last_for_all = any(map(lambda pair: i >= len(pots := pair[1]) - 1, displaced))
if use_last_for_all: i = -1
mapping = set(pots[i] for _, pots in displaced)
mapping_is_unique = (len(displaced)) == len(mapping)
mapping_has_no_overlap = all(map(lambda p: p not in dict, mapping))
if mapping_is_unique and mapping_has_no_overlap:
# all is well, we can use this level of labels
level = i
break
else:
# displace anything up to and including the current level and
# then try again:
for _, pots in displaced:
# note: we do the full sweep every time because
# `use_last_for_all` means that we can "jump" over elements
for label in pots[0:i] + [pots[i]]:
if label in dict:
entry = dict[label]
del dict[label]
displaced.append((entry, get_potential(entry.path)))
else:
# unable to find valid mapping!
raise ValueError(f"unable to find valid mapping for: {displaced}")
# place the entries:
for entry, pots in displaced:
label = pots[level]
assert label not in dict
dict[label] = entry
# Binaries go first:
for bin in bins.values(): put(out, bin)
for so in store.values(): put(out, so)
return out
# Optional, you don't have to use this (can just invoke `create_symlinks` from
# your own repo rule).
REPO_RULE = r"""
_BUILD_FILE = '''
load("@@//{package}:{file_path}", "define_targets")
load("@rules_sh//sh:sh.bzl", "sh_binaries")
package(
default_visibility = ["//visibility:public"],
)
define_targets(filegroup, sh_binaries)
'''
def _repo_impl(rctx):
create_symlinks(rctx)
rctx.file("BUILD.bazel", _BUILD_FILE, executable = False)
def make_repo():
host_deps_repo = repository_rule(
implementation = _repo_impl,
environ = [],
configure = True,
)
host_deps_repo(name = REPO_NAME)
"""
def make_repo_rule_function(
bins: Dict[str, Binary],
store: SharedObjectStore,
args: argparse.Namespace,
) -> str:
out = ["def create_symlinks(rctx):"]
p = lambda lines: out.extend((" " + line).rstrip() for line in lines.splitlines())
def make_symlinks(paths: Iterable[Path]) -> Iterable[str]:
for path in paths:
assert path.is_absolute()
src = str(path)
dest = src[1:] # assumes unix...
yield f"""rctx.symlink("{src}", "{dest}")\n"""
p("### Libraries")
lib_paths = [ p for so in store.values() for p in [so.path] + list(so.aliases) ]
for line in make_symlinks(lib_paths): p(line)
p("\n\n")
p("### Binaries")
for line in make_symlinks(map(lambda b: b.path, bins.values())): p(line)
p("\n\n")
p("### Extra")
p("for src, dest in EXTRA_SYMLINKS.items(): rctx.symlink(src, dest)")
p("\n\n")
# Repo rule:
repo_rule = REPO_RULE.format(
package = args.package,
file_path = args.file_path,
)
return "\n".join(out) + repo_rule
SELF_INVOKE_RULE = r"""
def _mk_regen_script(ctx):
script = ctx.executable.script
script_runfiles = ctx.attr.script[DefaultInfo].default_runfiles
args = [
"'./{}'".format(script.path),
"--repo-name '{}'".format(ctx.attr.repo_name),
"--package '{}'".format(ctx.attr.package),
"--file-path '{}'".format(ctx.attr.file_path),
"--package-within-main-repo-for-script '{}'".format(ctx.attr.script_package),
"--file-path-within-package-for-script '{}'".format(ctx.attr.script_file_path),
"--show-diff",
] + [
"'{}'".format(bin) for bin in ctx.attr.binaries
] + [
"--extra-binary '{}'".format(bin) for bin in ctx.attr.extra_binaries
] + [ '"${@}"' ]
out = "#!/usr/bin/env bash\n"
out += "exec {}".format(" ".join(args))
update_script = ctx.actions.declare_file(ctx.label.name)
ctx.actions.write(update_script, out, is_executable = True)
runfiles = ctx.runfiles([ctx.executable.script]).merge(script_runfiles)
return [
DefaultInfo(executable = update_script, runfiles = runfiles)
]
make_regenerate_script = rule(
implementation = _mk_regen_script,
attrs = {
"script": attr.label(
executable = True,
cfg = "exec",
allow_single_file = True,
),
"binaries": attr.string_list(allow_empty = True),
"extra_binaries": attr.string_list(allow_empty = True),
"repo_name": attr.string(),
"package": attr.string(),
"file_path": attr.string(),
"script_package": attr.string(),
"script_file_path": attr.string(),
},
executable = True,
)
"""
def make_build_file_function(
bins: Dict[str, Binary],
store: SharedObjectStore,
labels: Dict[str, DynamicExecutable],
args: argparse.Namespace,
) -> str:
# invert the labels map:
labels: Dict[Path, str] = {
entry.path: label for label, entry in labels.items()
}
assert len(labels) == len(bins) + len(store)
out = ["def define_targets(filegroup, sh_binary):"]
p = lambda lines: out.extend((" " + line).rstrip() for line in lines.splitlines())
def repo_relative_path(path: Path) -> str:
assert path.is_absolute()
return ":" + str(path)[1:] # assumes unix...
def make_dep_list(item: DynamicExecutable, labels: Dict[Path, str]) -> str:
deps = "[]"
if item.deps:
deps = "[\n"
for i, (shortname, path, abspath) in enumerate(item.deps):
if i: deps += "\n"
dep_label = labels[abspath]
deps += f""" # `{shortname}` as `{path}`\n"""
deps += f""" ":{dep_label}",\n"""
deps += " ]"
return deps
def make_label_list(items: Iterable[DynamicExecutable], labels: Dict[Path, str]) -> str:
all = "[\n"
for item in items:
label = labels[item.path]
all += f""" ":{label}",\n"""
all += " ]"
return all
# Libraries first:
p("### Libraries:")
p("filegroup_ = lambda *args, **kw: filegroup(")
p(" *args,")
p(" **(kw | (")
p(" {'data': kw.get('data', []) + EXTRA_LIB_DATA_DEPS.get(kw['srcs'][0], []) }")
p(" if 'srcs' in kw else {}")
p(" )),")
p(")")
p("")
for shared_object in store.values():
label = labels[shared_object.path]
src = repo_relative_path(shared_object.path)
aliases = ""
if shared_object.aliases:
aliases = " + [\n"
aliases += " # Symlinks:\n"
for alias in shared_object.aliases:
aliases += f""" "{repo_relative_path(alias)}",\n"""
aliases += " ]"
deps = make_dep_list(shared_object, labels)
p(f"""\
filegroup_(
name = "{label}",
srcs = [ "{src}" ]{aliases},
data = {deps},
)
""")
# emit "all-libraries" filegroup:
all_libraries = make_label_list(store.values(), labels)
p(f"""\
filegroup(
name = "all-libraries",
data = {all_libraries},
)
""")
p("\n\n")
# Binaries next:
p("### Binaries:")
# note: we don't distinguish between aliases when listing the deps of a
# binary
#
# i.e. if `/bin/ls` depends on `/lib64/libc.so` but there's also a
# `/lib64/libc.so.6` and a `/lib64/libc.so.6.2` that form a symlink chain,
# we'll create one filegroup for `libc` with all of these files and list the
# entire filegroup as a dep of `/bin/ls`
#
# this is technically wasteful (the sandbox will bind mount in three files
# instead of one) but I think it's probably fine for now (TODO(build)?)
#
# to do the "right thing" here we'd need to track the symlink chains instead
# of just having a flat list of aliases
for bin in bins.values():
label = labels[bin.path]
src = repo_relative_path(bin.path)
shared_object_deps = make_dep_list(bin, labels)
p(f"""\
sh_binary(
name = "{label}",
srcs = ["{src}"],
data = {shared_object_deps},
)
""")
# "all-binaries":
all_binaries = make_label_list(bins.values(), labels)
p(f"""\
sh_binary(
name = "all-binaries",
deps = {all_binaries}
)
""")
# Finally, a target to reinvoke this script to regenerate the file:
p("\n\n")
p("### Runnable target to regenerate:")
p(f"""\
make_regenerate_script(
name = "regenerate",
script = "@@//{args.package_within_main_repo_for_script}:{args.file_path_within_package_for_script}",
binaries = BINARIES,
extra_binaries = EXTRA_BINARIES,
repo_name = REPO_NAME,
package = "{args.package}",
file_path = "{args.file_path}",
script_package = "{args.package_within_main_repo_for_script}",
script_file_path = "{args.file_path_within_package_for_script}",
)
""")
return "\n".join(out)
DIVIDER = "#" * 80
BEGIN_PRESERVED = "# begin: preserved"
END_PRESERVED = "# end: preserved"
def extract_preserved_section(path: Path) -> Optional[str]:
if path.exists:
current = path.read_text().splitlines()
divider = "#" * 80
if BEGIN_PRESERVED in current and END_PRESERVED in current:
begin = current.index(BEGIN_PRESERVED)
end = current.index(END_PRESERVED)
if begin < end and current[begin - 1] == DIVIDER and current[end + 1] == DIVIDER:
return "\n".join(current[begin - 1 : end + 2])
return None
BLANK_PRESERVED_SECTION = f"""\
{DIVIDER}
{BEGIN_PRESERVED}
# Preserved Section
#
# The contents of this section are preserved verbatim.
# Extra binaries to pass to `update_shared_libs.py`.
#
# These will not land in `BINARIES` in the regenerated version of this file; use
# this for binary paths that, for example, come from non-constant starlark
# constructs.
EXTRA_BINARIES = []
# Extra symlinks for the repository rule to create.
#
# Map from source path to (repo relative) dest path.
EXTRA_SYMLINKS = {{}}
# Extra data dependencies to add to the library filegroups.
#
# Map from the absolute realpath label of the library to a list of labels.
EXTRA_LIB_DATA_DEPS = {{}}
{END_PRESERVED}
{DIVIDER}
"""
def process_output(args: argparse.Namespace, output: str):
# Find output path:
base_path: Path
if args.path_to_main_repo:
base_path = Path(args.path_to_main_repo)
else:
# we're assuming this script is in the main repo...
#
# we use `resolve` to resolve symlinks (i.e. if argv0 is a symlink to
# this file that's sitting in the runfiles dir of some bazel target)
base_path = Path(sys.argv[0]).resolve()
# remove the package path and the file path within the package
repo_relative_script_path = os.path.join(
args.package_within_main_repo_for_script,
args.file_path_within_package_for_script,
)
assert str(base_path).endswith(repo_relative_script_path)
base_path = Path(str(base_path).removesuffix(repo_relative_script_path))
# tack on the package and file path:
base_path = Path(os.path.join(
base_path,
args.package,
args.file_path,
))
# If the file already exists, copy its preserved section:
pres = prev if (prev := extract_preserved_section(base_path)) else BLANK_PRESERVED_SECTION
print(output)
output = output.replace("{PRESERVED_SECTION}", pres, 1)
# diff, if applicable:
if args.show_diff and base_path.exists():
# write out to temp file:
with NamedTemporaryFile() as tmp:
tmp.write(output.encode('utf-8'))
if delta := shutil.which("delta"):
subprocess.run([delta, "-s", base_path, tmp.name])
elif colordiff := shutil.which("colordiff"):
subprocess.run([colordiff, "-y", base_path, tmp.name])
elif diff := shutil.which("diff"):
print([diff, "-y", base_path, tmp])
subprocess.run([diff, "-y", base_path, tmp.name])
else:
print("warning: skipping diff; no suitable diff tool found", file = sys.stderr)
# emit:
if args.emit_on_stdout:
print(output, file = sys.stdout)
else:
with open(base_path, 'w') as f:
f.write(output)
print(f"Wrote out to `{base_path}`", file = sys.stderr)
def main(args: argparse.Namespace) -> int:
binaries = args.host_binaries + args.extra_binary
repo_name = args.repo_name
store = SharedObjectStore()
bins = {}
# Process the binaries:
for bin in binaries: bins[bin] = run_lddtree(bin, store)
# Check for collisions, assign labels:
check_for_collisions(bins, store)
labels: Dict[str, DynamicExecutable] = assign_labels(bins, store)
binaries = "\n ".join(f"""\"{bin}\",""" for bin in args.host_binaries)
output = f"""\
# Generated by `update_shared_libs.py`; prefer rerunning the script to modifying
# this file manually.
#
# Run as: `bazel run @{repo_name}//:regenerate`
#
# You can also pass `-- <extra binaries to add>` to the above invocation.
# Modifications to this list *will* persist when you run `:regenerate`.
#
# However, comments and starlark constructs will *not* be preserved. See
# `EXTRA_BINARIES` below if this is required.
BINARIES = [
{binaries}
]
REPO_NAME = "{repo_name}"
{{PRESERVED_SECTION}}
{SELF_INVOKE_RULE}
"""
output += "\n\n"
output += make_repo_rule_function(bins, store, args)
output += "\n\n"
output += make_build_file_function(bins, store, labels, args)
output += "\n\n"
process_output(args, output)
if __name__ == "__main__": main(parser.parse_args())
Raw
WORKSPACE.bzlmod
workspace(name = "hermetic-sandbox-test")
load(":bin_repo.bzl", "bin_repo")
bin_repo(
name = "binaries",
path = "/usr/bin",
cmds = [
"echo",
"ldd",
"ls",
],
)
# Bazel expects bash to be at this path (i.e. `/bin/bash`) however it isn't
# automatically mounted into the sandbox.
#
# genrules use this "built in" bash path (can be overriden with `$SH_BINARY` or
# `--shell_executable`)
bin_repo(
name = "host_bash",
path = "/bin",
cmds = [ "bash" ]
)
# note: on nixos... this should be fine as long as `/nix/store` is mounted into
# the sandbox?
new_local_repository(
name = "host_perl",
build_file_content = """
package(
default_visibility = ["//visibility:public"],
)
filegroup(
name = "bin",
srcs = glob(["bin/*"]),
)
filegroup(
name = "core",
srcs = glob(["lib*/*/x86_64-linux-thread-multi/CORE/**/*"]),
)
filegroup(
name = "core_headers",
srcs = glob(["lib*/*/x86_64-linux-thread-multi/CORE/*.h"]),
)
exports_files(
glob(
[
"bin/*",
"lib64/module/r0/**/*",
"lib64/*/x86_64-linux-thread-multi/**/*",
],
exclude = ["lib64/module/r0/man/**"],
),
)
""",
path = "/usr/intel/pkgs/perl/5.26.1-threads/",
)
################################################################################
load("host_deps_info.bzl", "make_repo")
make_repo()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment