Skip to content

Instantly share code, notes, and snippets.

@ExpHP

ExpHP/_example__crates.toml

Last active July 30, 2019 17:52
Show Gist options
  • Save ExpHP/ab6efce00ae19924ae627ebf9aca7d0c to your computer and use it in GitHub Desktop.
Save ExpHP/ab6efce00ae19924ae627ebf9aca7d0c to your computer and use it in GitHub Desktop.
Download ZIP
crates.py
Raw
_example__crates.toml
# This is crates.d/crates.toml
# Master config file for crates in this workspace.
#
# This is a homegrown solution (I'm sorry) to a number of
# annoying problems with using workspaces. Cargo.toml files
# in this workspace are generated from source files that
# contain lines like this:
#
# !!rsp2-assert-close
# !!itertools
#
# When you run the following command in the workspace root:
#
# ./crates gen
#
# Cargo.toml files are created with those lines expanded to e.g.
#
# rsp2-assert-close = { path = "../../util/assert-close" }
# itertools = "0.7"
#
# based on metadata in this file.
#--------------------------------------------
# Which crate represents the workspace?
root = "rsp2"
#--------------------------------------------
# Specification of all crates in the workspace and their
# paths relative to the root. The 'crates' script uses this
# information to generate the necessary relative paths
# to include when one crate in the workspace depends on
# another.
[crates]
rsp2 = "."
rsp2-lammps-wrap = "src/io/lammps"
rsp2-tasks = "src/tasks"
rsp2-minimize = "src/minimize"
rsp2-assert-close = "src/util/assert-close"
rsp2-structure = "src/structure"
rsp2-structure-io = "src/io/structure"
rsp2-slice-math = "src/util/slice-math"
rsp2-util-macros = "src/util/macros"
rsp2-fs-util = "src/util/fs"
rsp2-clap = "src/util/clap"
rsp2-tasks-config = "src/tasks/config"
rsp2-array-types = "src/util/array-types"
rsp2-linalg = "src/linalg"
rsp2-integration-test = "src/util/integration-test"
rsp2-soa-ops = "src/util/soa-ops"
rsp2-newtype-indices = "src/util/newtype-indices"
rsp2-python = "src/python"
rsp2-potentials = "src/potentials"
rsp2-dftbplus = "src/io/dftbplus"
rsp2c-unfold = "scripts/unfold_lib"
rsp2-phonopy-io = "src/io/phonopy"
rsp2-sparse = "src/util/sparse"
rsp2-dynmat = "src/dynmat"
#!!INSERTION POINT FOR NEW CRATES!!#
# (do not edit the above line!)
#--------------------------------------------
# Versions for external dependencies, so that we don't have
# to look a whole bunch of them up every single time we make
# a new crate in the workspace.
#
# This also makes it easier to migrate the entire workspace
# to a new version of a dependency.
[preferred-versions]
num-traits = "0.2.3"
num-complex = "0.2.1"
rand = "0.3"
serde = { version = "1.0.91", features = ["rc"] }
slice-of-array = "0.2.1"
# (...many, many entries omitted!)
#--------------------------------------------
# Always optimize some crates when they appear as dependencies.
#
# Choose crates that contain monomorphized instances of code that is critical
# to efficiency. (`pub` generic functions will generally not benefit from the
# feature).
#
# This depends on a cargo nightly feature, so support must be explicitly
# enabled for it in .crates.d/config.toml
[always-optimize]
rsp2-array-types = 3
Raw
_example__rsp2-array-types.Cargo.toml
# This is crates.d/rsp2-array-types.Cargo.toml
[package]
!!CRATE-NAME-LINE
version = "0.1.0"
authors = ["Michael Lamparski <[email protected]>"]
description = "Provides mathematical vector and matrix types used throughout rsp2."
!!LICENSE-PERMISSIVE
!!DEFAULT-EDITION
[lib]
path = "lib.rs"
[dependencies]
!!slice-of-array
!!num-traits
!!serde { optional = true, features = ["derive"] }
!!rand
!!num-complex
[dev-dependencies]
!!rsp2-assert-close
[features]
default = []
!!NIGHTLY-FEATURE-LINE
# FIXME Once namespaced-features are stabilized, this feature will be renamed to "serde".
# see https://github.com/rust-lang/cargo/issues/1286 (problem and proposal)
# https://github.com/rust-lang/cargo/issues/5565 (tracking issue, of sorts)
serde-support = ["serde"]
Raw
crates.py
#!/usr/bin/env python3
# pylint: disable=cell-var-from-loop
# This file is licensed under WTFPL 2.0.
# I honestly do not care what you do with it.
from contextlib import contextmanager
from pathlib import Path
from argparse import ArgumentParser
import os
import sys
import warnings
import string
import pytoml as toml
import typing as tp
# See crates.d/crates.toml for what this script is all about.
#
# For TOML file modification, this script relies mostly on plain
# text searching and substitution; this is partly because I doubt
# there's any good round-trip parsers for TOML in python,
# and I hate to destroy comments and other fine details.
CARGO_TOMLS_DIR = Path('crates.d')
MASTER_CFG_PATH = Path('crates.d/crates.toml')
MASTER_USER_CFG_PATH = Path('crates.d/config.toml')
MASTER_USER_CFG_EXAMPLE_PATH = Path('crates.d/config.example.toml')
GIT_HOOKS_DIR = Path('crates.d/git-hooks')
# something long enough for people to grep and find this script
INSERTION_POINT_TEXT = "#!!INSERTION POINT FOR NEW CRATES!!#"
def main():
parser = ArgumentParser(description='manage Cargo.tomls in sp2')
class no_command:
@staticmethod
def __init__(*_junk): pass
@staticmethod
def run(_args, fail): fail("no subcommand specified")
parser.set_defaults(klass=no_command)
subs = parser.add_subparsers(title='subcommands')
for klass in ALL_COMMANDS:
sub = subs.add_parser(klass.cmd, **klass.parser_kw)
sub.set_defaults(klass=klass)
klass.configure(sub)
args = parser.parse_args()
args.klass.run(args, fail=parser.error)
ALL_COMMANDS = []
def register(klass):
ALL_COMMANDS.append(klass)
return klass
#------------
thru = lambda f: lambda g: lambda *a, **kw: f(g(*a, **kw)) # pylint: disable=E0602
#------------
KIND_SPECIAL = object()
KIND_INTERNAL_DEP = object()
KIND_EXTERNAL_DEP = object()
@register
class Gen:
cmd = 'gen'
parser_kw = dict(
description=f'Regenerate Cargo.tomls from the source files in {CARGO_TOMLS_DIR}',
)
@staticmethod
def configure(sub):
sub.add_argument(
'--vcs', action='store_true',
help='temporarily override some user config settings to avoid '
'undue changes from being checked into VCS')
@staticmethod
def run(args, fail):
Gen._do_it(args, fail)
# several other commands invoke 'gen'. This is kind of a dumb
# hack to allow them to do so.
@staticmethod
def run_like_function(fail):
# HACK - get the default arguments as an argparse.Namespace
p = ArgumentParser()
Gen.configure(p)
default_args = p.parse_args([])
Gen._do_it(default_args, fail)
@staticmethod
def _do_it(args, fail):
with pushd(search_for_root(fail)):
data = read_master_config(fail)
user_conf = read_master_user_config(fail)
crates = data.pop('crates')
# catch some dumb user errors before we get too committed
for (name, path) in crates.items():
if not path.is_dir():
fail('[crates."{}"]: "{}": no such directory'.format(name, path))
src = root_rel_src_toml_path(name)
if not src.is_file(): # pylint: disable=E1101
fail('{}: no such file'.format(src))
# FIXME vvv this is waaaaaay too application-specific.
binary_shim_pairs = get_binary_shim_pairs()
make_binary_shims(binary_shim_pairs)
# FIXME ^^^ this is waaaaaay too application-specific.
lookup_get = make_macro_lookup(args.vcs, fail)
if user_conf.simulated_workspace.enabled:
simulated_workspace_gen_symlinks(crates, user_conf.simulated_workspace, fail)
# generate modified forms of each Cargo.toml
for (name, path) in crates.items():
src_path = Path(root_rel_src_toml_path(name))
dest_path = path / 'Cargo.toml'
with open(src_path) as f:
lines = list(f)
with open(dest_path, 'w') as f:
write = lambda *args: print(*args, file=f)
write("# DO NOT EDIT THIS FILE BY HAND!!")
write("# It has been autogenerated from", src_path)
write("#")
write("# To update it, run './crates gen' in the workspace root")
write()
for (i, line) in enumerate(lines):
line = line.rstrip('\n')
invocation = MacroInvocation.try_parse(line, fail=fail)
if invocation is None:
write(line)
continue
macro = lookup_get(invocation.name)
if macro is None:
s = f"{src_path}:{i}: unknown macro: {format_macro(invocation.name)}"
fail(s)
write(macro.expand(
# Optional inline-TOML supplied after the macro name
argument=invocation.argument,
# These are the keyword args available to callbacks:
# The directory for the Cargo.toml that we are generating.
cur_dir=path,
# The name of the crate whose Cargo.toml we are generating.
cur_name=name,
# The name of the macro we are calling
macro_name=invocation.name,
fail=fail,
))
def make_macro_lookup(vcs_safe, fail):
with pushd(search_for_root(fail)):
data = read_master_config(fail)
crates = data.pop('crates')
preferred_versions = data.pop('preferred-versions')
opt_levels = data.pop('always-optimize')
user_conf = read_master_user_config(fail)
root_name = get_root_package(fail)
class MaybeItsAWorkspaceMacro(NoArgMacro):
def expand_noarg(self, **_ignored):
# This is unfortunate. The Cargo.toml files should not depend on
# config, but there is no other way to disable a workspace.
if user_conf.simulated_workspace.enabled and not vcs_safe:
# This message is intended to be seen from a git diff.
return '''
# !!! CAUTION !!!
# The workspace table has been removed for the 'simulated-workspace'
# feature of 'crates'. Please do not check this into VCS!!!
#
# To correct it, do the following to temporarily override the
# simulated-workspace setting before committing:
#
# ./crates gen --vcs
#
#[workspace]
'''.strip()
else:
return '''
[workspace]
exclude = ["./scripts"]
'''.strip()
class CrateNameLineMacro(NoArgMacro):
def expand_noarg(self, *, cur_name, **_ignored):
return f'name = "{cur_name}"'
binary_shim_pairs = get_binary_shim_pairs()
# for defining new macros
lookup_put = lambda k, v: lookup_d.__setitem__(canonicalize_crate(k), v)
lookup_get = lambda k: lookup_d.get(canonicalize_crate(k))
# build a dash-insensitive lookup table of callbacks
lookup_d = {
# name = "crate-name"
'CRATE-NAME-LINE': CrateNameLineMacro(),
# [workspace]. Or not.
'MAYBE-ITS-A-WORKSPACE': MaybeItsAWorkspaceMacro(),
# licenses
'LICENSE-PERMISSIVE': ConstantMacro('license = "MIT OR Apache 2.0"'),
'LICENSE-GPL': ConstantMacro('license = "GPL-3.0"'),
# Edition
'DEFAULT-EDITION': ConstantMacro('edition = "2018"'),
# FIXME this is waaaaaay too application-specific.
'BINARY-SHIM-BIN-ENTRIES': BinaryShimsMacro(binary_shim_pairs),
# FIXME this is waaaaaay too application-specific.
'NIGHTLY-FEATURE-LINE': NightlyFeaturesMacro(lookup_get, crates),
# FIXME this is waaaaaay too application-specific.
'MAYBE-CARGO-FEATURES': CargoFeaturesMacro(user_conf, vcs_safe),
'MAYBE-OPTIMIZE-SOME-DEPS': OptimizeDepsMacro(lookup_get, user_conf, opt_levels, vcs_safe),
}
for (name, path) in crates.items():
lookup_put(name, InternalDepMacro(name, path))
for (name, rhs) in preferred_versions.items():
lookup_put(name, ExternalDepMacro(name, rhs))
return lookup_get
class Macro:
def expand(self,
argument: tp.Optional[tp.Union[int, float, str, dict, list]],
*,
cur_name: str,
macro_name: str,
cur_dir,
fail):
raise NotImplementedError
# A list of the internal dep crate names that would be added if this
# macro were expanded in the proper location.
def internal_deps(self):
return []
# A list of the external dep crate names that would be added if this
# macro were expanded in the proper location.
def external_deps(self):
return []
class NoArgMacro(Macro):
def expand(self, argument, *, cur_name, cur_dir, macro_name, fail):
if argument is not None:
fail(f'macro {macro_name} takes no arguments!')
return self.expand_noarg(cur_name=cur_name, cur_dir=cur_dir, macro_name=macro_name, fail=fail)
def expand_noarg(self, *, cur_name, cur_dir, macro_name, fail):
raise NotImplementedError
class ConstantMacro(NoArgMacro):
def __init__(self, value: str):
self.value = value
def expand_noarg(self, **_ignored):
return self.value
class InternalDepMacro(Macro):
def __init__(self, dep_name: str, dep_dir):
self.dep_name = dep_name
self.dep_dir = dep_dir
def expand(self, argument, *, cur_dir, fail, **_ignored):
# Don't use Path for this because:
# - It has no normpath equivalent that doesn't follow links (wtf?)
# - Its relpath equivalent is too strict
rel = os.path.normpath(os.path.relpath(self.dep_dir, cur_dir))
rhs = { "path": rel }
if argument is not None:
rhs = extend_dependency_toml(rhs, argument, fail=fail, dep_name=self.dep_name)
return f'{self.dep_name} = {inline_toml(rhs)}'
def internal_deps(self):
return [self.dep_name]
class ExternalDepMacro(Macro):
def __init__(self, dep_name: str, rhs: tp.Union[dict, str]):
self.dep_name = dep_name
self.rhs = rhs
def expand(self, argument, fail, **_ignored):
if argument is not None:
if not isinstance(argument, dict):
fail("argument to dep macros must be dict, not {repr(argument)}")
rhs = extend_dependency_toml(self.rhs, argument, fail=fail, dep_name=self.dep_name)
else:
rhs = self.rhs
return f'{self.dep_name} = {inline_toml(rhs)}'
def external_deps(self):
return [self.dep_name]
def extend_dependency_toml(dest, src, *, dep_name, fail):
if isinstance(dest, str):
if "version" in src:
that_version = src['version']
fail(f"two versions specified for {dep_name}: {repr(dest)} and {repr(that_version)}")
dest = { "version": dest }
if not isinstance(dest, dict):
fail(f"Expected a table or version string for {dep_name}, not {repr(dest)}")
dest = dict(dest)
for (k, v) in src.items():
if k == "features":
dest[k] = sorted(dest.get(k, []) + v)
else:
dest[k] = v
return dest
# a makeshift toml.dumps that uses inline tables
def inline_toml(x):
if isinstance(x, dict):
items = ', '.join(f'{k} = {inline_toml(v)}' for (k, v) in x.items())
return f'{{ {items} }}'
elif isinstance(x, list):
items = ', '.join(inline_toml(v) for v in x)
return f'[{items}]'
# types that only have an inline representation
elif isinstance(x, (str, float, int, bool)):
# I'm not 100% sure if python str() makes a valid toml str
# so dumb dumb hack dumb hack hack dumb dumb dumb
s = toml.dumps({'a': x}).strip()
assert s.startswith('a')
s = s[1:].strip()
assert s.startswith('=')
s = s[1:].strip()
return s
else:
raise TypeError(x)
# dumb hack to make pytoml parse a value written in inline syntax
def parse_inline_toml(s):
if '\n' in s or '\r' in s:
raise ValueError(f"More than one line in inline TOML: {repr(s)}")
return toml.loads(f"a = {s}")['a']
class MacroInvocation:
def __init__(self, name: str, argument=None):
self.name = name.replace('_', '-')
self.argument = argument
# If a line is a valid macro invocation, parse it.
# If it is clearly not a macro, return `None`.
# If it is dubious looking, fail.
#
# Intended grammar: (current implementation is sloppy/overly permissive)
# [ WHITESPACE ]*
# "!!" [ WHITESPACE ]* KEBAB_CASE_IDENTIFIER
# [ WHITESPACE [ WHITESPACE ]* INLINE_TOML_VALUE ]?
# [ WHITESPACE ]*
#
@classmethod
def try_parse(cls, line, fail) -> tp.Optional["MacroInvocation"]:
line = line.strip()
if line[:2] != "!!":
return None
line = line[2:]
parts = line.split(None, 1)
if len(parts) == 0:
fail("!! with no macro name")
elif len(parts) == 1:
name, = parts
return cls(name)
elif len(parts) == 2:
name, argument = parts
return cls(name, parse_inline_toml(argument))
assert False, "unreachable"
def format_macro(name):
return '!!' + name
# crates are generally compared as dash-insensitive
def canonicalize_crate(crate_name):
return crate_name.replace('_', '-')
# For simulated workspaces, make all target directories symlinks to
# the root package's target, to help reduce duplicate build
# artefacts even if we are not a true workspace.
#
# Also link together configuration directories for subcrates.
def simulated_workspace_gen_symlinks(crates, conf, fail):
root_package_name = get_root_package(fail)
# prepare the root crate first
if conf.shared_target:
ensure_not_symlink('target')
Path('target').mkdir(parents=True, exist_ok=True)
if conf.shared_dot_idea:
(Path('.subcrate-cfg') / '.idea').mkdir(parents=True, exist_ok=True) # pylint: disable=E1101
if conf.shared_dot_vscode:
(Path('.subcrate-cfg') / '.vscode').mkdir(parents=True, exist_ok=True) # pylint: disable=E1101
# link the rest
for (name, path) in crates.items():
if name != root_package_name:
if conf.shared_target:
maybe_ensure_target_link('.', path)
if conf.shared_dot_idea:
maybe_ensure_target_link('.subcrate-cfg', path, filename='.idea')
if conf.shared_dot_vscode:
maybe_ensure_target_link('.subcrate-cfg', path, filename='.vscode')
# tries to have a crate's target symlink back to the root crate's target,
# but may fail. (no big deal)
#
# The logic here was really intended for just the target directory,
# but 'name' can be specified to reuse this for files where the logic
# for target is probably good enough
# (the theme is: "a symlink to a directory that an IDE is very likely
# to attempt to regenerate the moment it vanishes")
def maybe_ensure_target_link(root_path, crate_path, filename='target'):
path = Path(crate_path) / filename
dest = os.path.relpath(Path(root_path) / filename, crate_path)
# don't remove existing links unless we have to.
if path.is_symlink() and readlink(path) == str(dest):
return
# As soon as we destroy the existing contents, we are in a race
# against the RLS to recreate the directory the way WE want it.
rm_rf(path)
# gogogogogogogogo!!!!!!
try: symlink(dest, path)
except FileExistsError:
# Damn, the RLS probably beat us.
# Let it win; maybe we'll have better luck next time.
pass
def ensure_not_symlink(path):
if os.path.islink(path):
os.unlink(path)
#------------
CARGO_TOML_TEMPLATE = '''
[package]
!!CRATE-NAME-LINE
version = "0.1.0"
authors = ["Michael Lamparski <[email protected]>"]
{license}
!!DEFAULT-EDITION
[lib]
path = "lib.rs"
[dependencies]
[features]
!!NIGHTLY-FEATURE-LINE
'''[1:]
LICENSE_COMMENT_PERMISSIVE = '''
/* ************************************************************************ **
** This file is part of rsp2, and is licensed under EITHER the MIT license **
** or the Apache 2.0 license, at your option. **
** **
** http://www.apache.org/licenses/LICENSE-2.0 **
** http://opensource.org/licenses/MIT **
** **
** Be aware that not all of rsp2 is provided under this permissive license, **
** and that the project as a whole is licensed under the GPL 3.0. **
** ************************************************************************ */
'''[1:]
LICENSE_COMMENT_GPL = '''
/* ********************************************************************** **
** This file is part of rsp2. **
** **
** rsp2 is free software: you can redistribute it and/or modify it under **
** the terms of the GNU General Public License as published by the Free **
** Software Foundation, either version 3 of the License, or (at your **
** option) any later version. **
** **
** http://www.gnu.org/licenses/ **
** **
** Do note that, while the whole of rsp2 is licensed under the GPL, many **
** parts of it are licensed under more permissive terms. **
** ********************************************************************** */
'''[1:]
@register
class New:
cmd = 'new'
parser_kw = {}
@staticmethod
def configure(sub):
sub.add_argument('--license', required=True, choices=['permissive', 'gpl'])
sub.add_argument('CRATENAME')
sub.add_argument('DIR')
@staticmethod
def run(args, fail):
license = args.license
if not is_root('.'):
# (I'm too lazy to work out the correct relative paths
# to be written to crates.toml)
fail("The 'new' subcommand must be run from the root.")
name = args.CRATENAME
path = Path(args.DIR)
if path.exists():
fail(f'"{path}": path already exists')
if (path / root_rel_src_toml_path(name)).exists():
fail(f'crate "{name}" already exists')
# Insert a line to crates.toml and the root package Cargo.toml
# without destroying formatting
# (do this now for earlier exits on error)
root_name = get_root_package(fail=fail)
root_path = root_rel_src_toml_path(root_name)
with open(root_path) as f:
root_lines = list(f)
root_lines = master_config_textual_add(root_path, root_lines, f'!!{name}\n', fail=fail)
with open(MASTER_CFG_PATH) as f:
master_lines = list(f)
master_lines = master_config_textual_add(MASTER_CFG_PATH, master_lines, f'{name} = "{path}"\n', fail=fail)
# make the directory
path.mkdir()
with open(root_rel_src_toml_path(name), 'w') as f:
toml_license = {
'permissive': "!!LICENSE-PERMISSIVE",
'gpl': "!!LICENSE-GPL",
}[license]
f.write(CARGO_TOML_TEMPLATE.format(license=toml_license))
with open(path / 'lib.rs', 'w') as f:
f.write({
'permissive': LICENSE_COMMENT_PERMISSIVE,
'gpl': LICENSE_COMMENT_GPL,
}[license])
with open_tmp(root_path) as f:
f.writelines(root_lines)
with open_tmp(MASTER_CFG_PATH) as f:
f.writelines(master_lines)
# regenerate
Gen.run_like_function(fail)
#------------
@register
class Mv:
cmd = 'mv'
parser_kw = {}
@staticmethod
def configure(sub):
sub.add_argument('CURDIR')
sub.add_argument('DESTDIR')
@staticmethod
def run(args, fail):
do_mv_or_rm(curdir=args.CURDIR, destdir=args.DESTDIR, fail=fail)
@register
class Rm:
cmd = 'rm'
parser_kw = {}
@staticmethod
def configure(sub):
sub.add_argument('DIR')
@staticmethod
def run(args, fail):
do_mv_or_rm(curdir=args.DIR, destdir=None, fail=fail)
# Shared implementation for mv and rm.
# When distdir is None, this deletes.
def do_mv_or_rm(curdir, destdir, fail):
curdir = Path(curdir)
if destdir is not None:
destdir = Path(destdir)
if not is_root('.'):
# (I'm too lazy to work out the correct relative paths
# to be written to crates.toml)
cmd_name = 'rm' if destdir is None else 'mv'
fail(f"The '{cmd_name}' subcommand must be run from the root.")
root = Path('.')
if destdir is not None and destdir.exists():
fail(f'"{destdir}": path already exists')
if not curdir.exists():
fail(f'"{curdir}": path does not exist')
crate = crate_name_from_path(curdir)
# Edit the line in crates.toml without destroying formatting
# (do this now for earlier exits on error)
with open(MASTER_CFG_PATH) as f:
lines = list(f)
lines = master_config_textual_set_path(lines, crate, destdir, fail=fail)
if destdir is None: # rm
rm_rf(str(curdir))
unlink(root / root_rel_src_toml_path(crate))
else: # mv
curdir.rename(destdir)
with open_tmp(MASTER_CFG_PATH) as f:
f.writelines(lines)
# regenerate
Gen.run_like_function(fail)
#------------
@register
class InstallHooks:
cmd = 'install-git-hooks'
parser_kw = {}
@staticmethod
def configure(sub):
pass
@staticmethod
def run(args, fail):
with pushd(search_for_root(fail)):
nothing_to_do = True
error = False
for name in os.listdir(GIT_HOOKS_DIR):
dest = os.path.join('.git/hooks', name)
src = os.path.join(GIT_HOOKS_DIR, name)
dest_rel_src = os.path.relpath(src, start=os.path.dirname(dest))
if os.path.lexists(dest):
if any(x() for x in [
(lambda: not os.path.exists(dest)),
(lambda: not os.path.exists(src)),
(lambda: not os.path.samefile(os.path.realpath(dest), os.path.realpath(src))),
]):
warnings.warn(f'Not overwriting {dest}')
nothing_to_do = False
error = True
continue
else:
nothing_to_do = False
print(f'Installing {name} hook', file=sys.stderr)
os.symlink(dest_rel_src, dest)
if nothing_to_do:
print(f'Already up to date!', file=sys.stderr)
if error:
sys.exit(1)
@register
class Test:
cmd = 'test'
parser_kw = dict(
description=
"Replacement for 'cargo test --all' when the 'simulated-workspace'"
" option is enabled. You won't believe how it works."
)
@staticmethod
def configure(sub):
pass
@staticmethod
def run(_args, fail):
from subprocess import Popen
# regenerate
Gen.run_like_function(fail)
def do_cargo_test_all(**kw):
code = Popen(['cargo', 'test', '--all'], **kw).wait()
if code:
sys.exit(code)
with pushd(search_for_root(fail)):
if read_master_user_config(fail).simulated_workspace.enabled:
# do the insane thing
# Create a replica that looks exactly like our project,
# except that it truly is a workspace.
TEST_ROOT = '.secret-test-dir'
simulated_workspace_ensure_test_replica_exists(TEST_ROOT)
do_cargo_test_all(cwd=TEST_ROOT)
# keep TEST_ROOT around for build artefacts
else:
# do something reasonable instead.
do_cargo_test_all()
def simulated_workspace_ensure_test_replica_exists(test_root):
import shutil
test_root = Path(test_root)
if not test_root.exists():
test_root.mkdir()
for path in map(Path, ['src', 'tests', 'examples']):
test_path = test_root / path
effective_path = test_root / '..' / path
if effective_path.exists() and not test_path.exists():
test_path.symlink_to('..' / path)
test_cargo = test_root / 'Cargo.toml'
shutil.copyfile('Cargo.toml', test_cargo)
with open(test_cargo, 'a') as f:
f.write('\n')
f.write('[workspace]\n')
#------------
@register
class Dot:
cmd = 'dot'
parser_kw = dict(
description="Show dependency graph."
)
@staticmethod
def configure(sub):
sub.add_argument('--root', action='store_true', dest='root')
sub.add_argument('--no-root', action='store_false', dest='root')
sub.set_defaults(root=False)
sub.add_argument('--no-leaves', action='store_false', dest='leaves')
sub.add_argument('--leaves', action='store_true', dest='leaves')
sub.set_defaults(leaves=True)
@staticmethod
def run(args, fail):
import matplotlib.pyplot as plt
from networkx.drawing.nx_pydot import write_dot
# regenerate
Gen.run_like_function(fail)
crates = get_internal_crates_and_root_rel_paths(fail)
g = get_internal_dep_graph(crates, fail)
if not args.root:
g.remove_node(get_root_package(fail))
if not args.leaves:
leaves = [node for node in g if g.out_degree(node) == 0]
for node in leaves:
g.remove_node(node)
write_dot(g, sys.stdout)
plt.show()
def get_internal_dep_graph(crates, fail):
import networkx as nx
with pushd(search_for_root(fail)):
g = nx.DiGraph()
for (crate, path) in crates.items():
with open(path / 'Cargo.toml') as f:
d = toml.load(f)
for dep in d['dependencies']:
if dep in crates:
g.add_edge(crate, dep)
return g
#------------
@register
class Edit:
cmd = 'edit'
parser_kw = dict(
description="Edit a crate's Cargo.toml."
)
@staticmethod
def configure(sub):
sub.add_argument('CRATE_OR_PATH', default='.',
help="Accepts a crate name or a path to a crate directory."
" You can also do 'edit crates' to edit crates.toml,"
" or 'edit config' to edit config.toml.")
@staticmethod
def run(args, fail):
import subprocess
import shutil
root = search_for_root(fail)
if args.CRATE_OR_PATH == 'crates':
toml_path = root / MASTER_CFG_PATH
elif args.CRATE_OR_PATH == 'config':
toml_path = root / MASTER_USER_CFG_PATH
if not toml_path.exists():
shutil.copyfile(root / MASTER_USER_CFG_EXAMPLE_PATH, toml_path)
print(f"Created new config file at '{toml_path}'.", file=sys.stderr)
else:
if args.CRATE_OR_PATH in get_internal_crate_names(fail):
crate = args.CRATE_OR_PATH
else:
crate = crate_name_from_path(args.CRATE_OR_PATH)
toml_path = root / root_rel_src_toml_path(crate)
editor = os.environ.get('EDITOR') or 'vim'
if subprocess.run([editor, toml_path]).returncode != 0:
print(f"Failed to edit '{toml_path}'!", file=sys.stderr)
sys.exit(1)
# regenerate
Gen.run_like_function(fail)
#------------
@register
class UnusedDeps:
cmd = 'unused'
parser_kw = dict(
description="List unused external crate dependencies."
)
@staticmethod
def configure(sub):
pass
@staticmethod
def run(_args, fail):
with pushd(search_for_root(fail)):
data = read_master_config(fail)
our_crates = list(data.pop('crates'))
unused_deps = set(map(canonicalize_crate, data.pop('preferred-versions')))
for crate in our_crates:
with open(root_rel_src_toml_path(crate)) as f:
lines = list(f)
to_be_removed = []
for line in lines:
invocation = MacroInvocation.try_parse(line, fail=fail)
if invocation is not None:
to_be_removed.append(canonicalize_crate(invocation.name))
unused_deps -= set(to_be_removed)
for dep in unused_deps:
print(dep)
#------------
class UserConfig:
def __init__(self, d):
d = dict(d)
simulated_workspace = d.pop('simulated-workspace', {})
nightly_cargo = d.pop('nightly-cargo', {})
for key in d:
warnings.warn(f"Unrecognized user-config section: {key}")
self.simulated_workspace = CfgSimulatedWorkspace(simulated_workspace)
self.nightly_cargo = CfgNightlyCargo(nightly_cargo)
@classmethod
def default(cls):
return cls({})
class CfgSimulatedWorkspace:
def __init__(self, d):
d = dict(d)
self.enabled = d.pop('enable', False)
self.shared_target = d.pop('shared-target', True)
self.shared_dot_idea = d.pop('shared-dot-idea', True)
self.shared_dot_vscode = d.pop('shared-dot-vscode', False)
for key in d:
warnings.warn(f'Unrecognized user-config key: simulated-workspace.{key}')
class CfgNightlyCargo:
def __init__(self, d):
d = dict(d)
self.always_optimize_some_deps = d.pop('always-optimize-some-deps', False)
for key in d:
warnings.warn(f'Unrecognized user-config key: nightly-cargo.{key}')
#------------
def read_master_config(fail):
root = search_for_root(fail)
with open(root / MASTER_CFG_PATH) as f:
d = toml.load(f)
d['crates'] = {k: Path(v) for (k, v) in d['crates'].items()}
return d
def read_master_user_config(fail):
root = search_for_root(fail)
try:
with open(root / MASTER_USER_CFG_PATH) as f:
d = toml.load(f)
return UserConfig(d)
except FileNotFoundError:
return UserConfig.default()
def get_internal_crates_and_root_rel_paths(fail):
# There used to be logic in here that branched on whether each
# value in [crates] was a str or a dict... but I think there is
# already other code that assumes the values are all paths?
return read_master_config(fail)['crates']
def get_internal_crate_names(fail):
return set(get_internal_crates_and_root_rel_paths(fail))
def get_root_package(fail):
return read_master_config(fail)['root']
def crate_name_from_path(path):
with open(Path(path) / "Cargo.toml") as f:
return toml.load(f)['package']['name']
def is_root(path):
return (Path(path) / MASTER_CFG_PATH).exists()
def search_for_root(fail):
d = Path('.')
for _ in range(20):
if is_root(d):
return d.resolve()
d = d / '..'
fail('Cannot find workspace root!')
# set a path for one of our crates, by plaintext search
def master_config_textual_add(path, lines, line, fail):
lines = list(lines)
stripped = [x.strip() for x in lines]
try: i = stripped.index(INSERTION_POINT_TEXT)
except ValueError:
fail(f'''
Failed to find the new crate insertion point in '{path}'
It should look like the following:
{INSERTION_POINT_TEXT}
'''[:-1])
assert False, "unreachable"
lines.insert(i, line)
return lines
# set a path for one of our crates, by plaintext search.
# 'path = None' deletes
def master_config_textual_set_path(lines, name, path, fail):
lines = list(lines)
name = canonicalize_crate(name)
for (i,line) in enumerate(lines):
if canonicalize_crate(line.strip()).startswith(name):
# expect exactly two, paired quotes
quotes = list(find_all_indices_of_any("\"'", line))
if not (len(quotes) == 2 and line[quotes[0]] == line[quotes[1]]):
fail(f"""
{MASTER_CFG_PATH}:{i}: This line looks funny, not sure how to edit it.
It should ideally look like:
{name} = 'current/path/to/{name}'
But at least make sure there are exactly two quotes of the same style.
"""[1:])
if path is None:
del lines[i]
else:
lines[i] = f'{lines[i][:quotes[0]+1]}{path}{lines[i][quotes[1]:]}'
return lines
else: # pylint: disable=useless-else-on-loop
fail(f"{MASTER_CFG_PATH}: Failed to find line for {name}.")
#------------
# FIXME this is waaaaaay too application-specific.
# Maybe .crates.d could have some simplistic plugin files or something for crap like this.
def get_binary_shim_pairs():
with open('src/tasks/entry_points.rs') as f:
lines = list(f)
identifier_chars = set(''.join([
string.ascii_lowercase,
string.ascii_uppercase,
string.digits,
'_',
]))
pairs = []
for cur, nxt in zip(lines, lines[1:]):
heuristic = "CRATES" # for better debugging
cur = cur.strip()
nxt = nxt.strip()
if heuristic in cur:
prefix = "// %% CRATES: binary: "
suffix = " %%"
assert cur.startswith(prefix)
assert cur.endswith(suffix)
bin = cur[len(prefix):-len(suffix)]
prefix = "pub fn "
stop = nxt.index("(")
assert nxt.startswith(prefix)
func = nxt[len(prefix):stop].strip()
assert set(func).issubset(identifier_chars)
pairs.append((bin, func))
return sorted(pairs)
def make_binary_shims(pairs):
shims_dir = Path("src/binary-shims")
shims_dir.mkdir(exist_ok = True)
for (bin, func) in pairs:
with open(shims_dir / f"{bin}.rs", mode='w') as f:
assert not (set("\\\"") & set(bin))
print(f"""\
// This file was autogenerated by `crates gen`. Do not edit!
fn main() {{
let version = rsp2::version::get();
rsp2_tasks::entry_points::{func}("{bin}", version);
}}
""", end='', file=f)
class BinaryShimsMacro(NoArgMacro):
def __init__(self, pairs):
self.pairs = pairs
def expand_noarg(self, **_ignored):
entries = []
for (bin, _func) in self.pairs:
lines = [
f"[[bin]]\n",
f'name = "{bin}"\n',
f'path = "src/binary-shims/{bin}.rs"\n',
]
entries.append(''.join(lines))
return '\n'.join(entries)
class NightlyFeaturesMacro(NoArgMacro):
def __init__(self, lookup_get, crates):
self.lookup_get = lookup_get
self.crates = crates
def expand_noarg(self, *, cur_name, fail, **_ignored):
deps = read_dep_names_from_macros(cur_name, self.lookup_get, fail=fail, external=False)
def recursive_feature_line(feature, base_list=()):
toml_strs = [f'"{f}"' for f in base_list]
toml_strs += sorted([f'"{d}/{feature}"' for d in deps])
toml_list = '[' + ', '.join(toml_strs) + ']'
return f"{feature} = {toml_list}"
nightly = recursive_feature_line("nightly", ['beta'])
beta = recursive_feature_line("beta")
return f'{nightly}\n{beta}'
class CargoFeaturesMacro(NoArgMacro):
def __init__(self, config, vcs_safe):
self.features = []
if not vcs_safe:
if config.nightly_cargo.always_optimize_some_deps:
self.features.append('profile-overrides')
def expand_noarg(self, *_bleh, **_ignored):
if not self.features:
return ''
return f'''
# !!! CAUTION !!!
# Nightly cargo features are enabled in the user config of 'crates'.
# Please do not check this change into VCS!!!
#
# To correct it, do the following to temporarily override the
# cargo-nightly settings before committing:
#
# ./crates gen --vcs
#
cargo-features = {repr(self.features)}
'''.strip()
class OptimizeDepsMacro(NoArgMacro):
def __init__(self, lookup_get, config, crate_opt_levels, vcs_safe):
self.lookup_get = lookup_get
self.enabled = config.nightly_cargo.always_optimize_some_deps and not vcs_safe
self.crate_opt_levels = crate_opt_levels
def expand_noarg(self, *, cur_name, fail, **_ignored):
if not self.enabled:
return ''
deps = read_dep_names_from_macros(cur_name, self.lookup_get, fail=fail)
out = []
for dep in deps:
if dep in self.crate_opt_levels:
out.append(f'''
[profile.dev.overrides."{dep}"]
opt-level = {self.crate_opt_levels[dep]}
'''.strip())
return '\n\n'.join(out)
# HACK:
# To figure out the list of crates that a crate depends on during macro
# expansion, we directly re-parse the template Cargo.toml files and look for
# macros that generate deps.
#
# It'd be much nicer to get this information from the expanded TOML,
# but there is a chicken-and-egg problem there as we are currently in
# the middle of macro expansion!
def read_dep_names_from_macros(crate_name, lookup_get, fail, external=True, internal=True):
deps = []
with open(root_rel_src_toml_path(crate_name)) as f:
for line in f:
invocation = MacroInvocation.try_parse(line, fail=fail)
if invocation is None:
continue
macro = lookup_get(invocation.name)
if internal: deps.extend(macro.internal_deps())
if external: deps.extend(macro.external_deps())
return sorted(deps)
#------------
def root_rel_src_toml_path(crate_name):
return Path(CARGO_TOMLS_DIR) / f'{crate_name}.Cargo.toml'
# Variant of open(path, 'w') that writes to a temp file,
# then overwrites the requested file only once the context manager
# is exited without an exception
@contextmanager
def open_tmp(path, mode='w'):
import tempfile
with tempfile.NamedTemporaryFile(mode=mode, delete=False) as f:
try: yield f
except:
# THIS NEVER HAPPENED
os.unlink(f.name)
raise
f.flush()
# exiting the 'with' closes (but doesn't delete) the temp file
# Make the new file official.
#
# Writing Cargo.toml seems to cause rebuilds of any crate
# that has a build.rs, so try to avoid it if we can.
replace_if_different(path, f.name)
@contextmanager
def pushd(path):
old = os.path.abspath('.')
os.chdir(path)
try: yield None
finally: os.chdir(old)
def replace_if_different(current, candidate):
import shutil
with open(current) as f:
a = f.read()
with open(candidate) as f:
b = f.read()
if a != b:
shutil.move(candidate, current)
else:
os.unlink(candidate)
def find_all_indices_of_any(needles, haystack):
needles = set(needles)
for (i, x) in enumerate(haystack):
if x in needles:
yield i
def rm_rf(path):
import shutil
path = Path(path)
try:
# note: cannot call rmtree on a symlink because lolidunno
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
else:
unlink(path)
except FileNotFoundError:
pass
iife = lambda f: f()
#------------
# CLion thinks these functions can't accept pathlib.Path.
def unlink(path):
return os.unlink(str(path))
def symlink(a, b):
return os.symlink(str(a), str(b))
def listdir(path):
return os.listdir(str(path))
def readlink(path):
return os.readlink(str(path))
#------------
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment