Profpatsch · May 6, 2020 10:13
diff --git a/build-purescript-tmp.nix b/build-purescript-tmp.nix
 { pkgs }:
 let
  lib = pkgs.lib;
 in
 # list of additional packages to add to the packageset
 additionalPackages:

 # source path of the package (derivation or (filtered) nix path)
 assert (lib.all (pkg: pkg ? src) additionalPackages);
 # dhall file of that package
 assert (lib.all (pkg: pkg ? dhall-file) additionalPackages);

 let
  lib = pkgs.lib;
  runCommandLocal = import ../nix/lib/run-command-local.nix { inherit (pkgs) runCommand; };
  packagesDhall = ./packages.dhall;

  # As introduced by https://github.com/purescript/purescript/commit/8a14cc0f23a6e749c852dd202df91edb2cd8df08
  # a cache-db.json file is a mapping from module name to file compilation time
  # and content hashes. This command merges the json objects and is thus
  # the mappend on these cache dbs.
  merge-purs-cache-db-json = pkgs.writers.writePython3 "merge-purs-cache-db-json" {} ''
    import sys
    import json

    res = {}
    for i in sys.argv[1:]:
        with open(i) as f:
            # right-based merge of dicts
            res = {**res, **json.load(f)}

    print(json.dumps(res))
  '';

  # make all references to `"dependency"` a reference to `self.dependency`
  # instead, where `self` is the version of the packageset with all dependencies
  # already resolving correctly.
  #
  # buildPackage: a function:
  #  { name: String
  #  , version: String
  #  , dependencies: List a
  #  , purescriptGlobs: List PurescriptGlob
  #  , src: Drv }
  # -> a
  # if you insert Drv for a, you get out a set of derivations.
  # A PurescriptGlob is a string like "src/**/*.purs" that is passed
  # to the purescript compiler to glob for source files.
  #
  # flatPackageAttrset:
  # an attrset mapping from the name of a package
  # to the packages name, version, src and dependencies
  # where `dependencies` is a list of package names as strings.
  transformedSpago = buildPackage: flatPackageAttrset:
    lib.fix (self:
      let depStringToSymbol = str:
            # nix allows you to reference attributes of an attrset by string name.
            # So we are able to map the strings to actual symbols.
            self.${str};
          convertPkg = name: {version, src, dependencies, purescriptGlobs}:
            buildPackage {
              inherit name version src purescriptGlobs;
              dependencies = map depStringToSymbol dependencies;
            };
      in lib.mapAttrs convertPkg flatPackageAttrset
    );

  # functions that can be used as `buildPackage` in `transformedSpago`
  builders =
    let
      # just returns the tree (dag really) of all inputs
      inputs = lib.id;

      # `buildPackage` function that compiles a set of purescript pakages.
      # The result is a set of package names to
      # { src                # source code of the package
      # , compiled           # compiled output of the package
      # , transitiveSrc      # list of the sources
      #                      # of all transitive dependencies
      # , transitiveCompiled # list of the compiled outputs
      #                      # of all transitive dependencies
      # }
      compileAll = {name, version, src, dependencies, purescriptGlobs}:
        let
          mergeByName = builtins.foldl' lib.mergeAttrs {};
          # We (abuse?) the fact, that each packages has a unique name
          # to collect all transitive dependencies.
          transitiveSrc = mergeByName
            ([{ ${name} = { inherit src purescriptGlobs; }; }]
            ++ (map (d: d.transitiveSrc) dependencies));
          transitiveCompiled = mergeByName
            (map (d: d.transitiveCompiled) dependencies);

          # ask the compiler to compile some source code.
          # transitiveSources is a list of { src, purescriptGlob }
          # the script takes $1, the output path.
          purescript-compile = pkgName: transitiveSources:
            pkgs.writers.writeDash "purescript-compile-${pkgName}" (''
              # we disable parallel compilation by setting the purescript
              # runtime to -N1 so it only runs on one thread. nix already
              # parallelizes the builds for us. This leads to a speedup of
              # ~100% for compiling big package sets.
              ${pkgs.purescript}/bin/purs \
                compile \
                +RTS -N1 -RTS \
                --output "$1" \
                --verbose-errors \
              ''
              + (lib.escapeShellArgs
                  # purs needs the source code of all transitive dependencies
                  (lib.concatMap
                    # globs are patterns like "src/**/*.purs" that are passed
                    # to the compiler (not expanded by the shell) to avoid
                    # breaching the system argv length limit.
                    ({ src, purescriptGlobs }:
                        map (glob: "${src}/${glob}") purescriptGlobs)
                    (transitiveSources))
            ));

          compiled = pkgs.runCommand "${name}-${version}" {
            outputs = [ "out" "cachedb" ];
          } ''
            mkdir $out
            # TODO: should be able to give these inputs to purs directly
            ${# this is the cache of all dependencies already compiled
              # so we don’t have to rebuild them
              lib.concatStringsSep "\n"
               (lib.mapAttrsToList (_: compiled: ''ln -s "${compiled}"/* $out'')
                 transitiveCompiled)}

            # the cache db file contains the timestamps and hashes
            # of the files just compiled. The compiler uses these
            # to determine whether to recompile a file.
            # Since our storepaths all have the same timestamp (1970-01-01)
            # nothing should be recompiled. But the file has to be there anyways.
            ${merge-purs-cache-db-json} ${(lib.escapeShellArgs
                (map (dep: dep.compiled.cachedb) dependencies))} \
                   > $out/cache-db.json

            ${purescript-compile "${name}-${version}"
              # purs needs the source code of all transitive dependencies;
              # transitiveSrc already includes our own
              (lib.mapAttrsToList (_: v: v) transitiveSrc)
             } \
               $out

            # move the combined cache-db to its output
            mv $out/cache-db.json $cachedb

            # TODO: hack to remove the other compiled modules again;
            # everything that is a symlink was added by us above
            for dir in $out/*; do
              test -L "$dir" && rm "$dir" || true
            done
          '';
        in {
          inherit src compiled transitiveSrc;
          transitiveCompiled = transitiveCompiled // { ${name} = compiled; };
        };

      # All transitive dependencies of a package as a “set”
      # (attrset with `null` as values)
      transitiveDependencies = {name, version, src, dependencies}:
        lib.foldl' lib.mergeAttrs { ${name} = null; } dependencies;
      # Create a package set that can be used from psc-package
      # by linking it to `.psc-package/$name` in the project folder
      # and setting the `"set"` field in `psc-package.json` to $name.
      pscPackageSet = srcDeps:
        let
          # convert dhall package set to a json file
          spagoJson = pkgs.runCommand "spago-packages.json" {} ''
            ${pkgs.dhall-json}/bin/dhall-to-json \
              <<< "${packagesDhall}" \
              > $out
          '';
        in pkgs.runCommand "spago" {} ''

          # this is the package list psc-package references
          mkdir -p $out/.set
          ln -s '${spagoJson}' $out/.set/packages.json

          # now we link in all packages
          ${lib.concatStringsSep "\n"
              ( # TODO: escaping?
                lib.mapAttrsToList (name: {version, src}: ''
                  mkdir -p "$out/${name}"
                  ln -s '${src}' "$out/${name}/${version}"
                '') srcDeps)}
        '';


    in { inherit inputs pscPackageSet compileAll transitiveDependencies; };

  # read dhall file in as JSON, then import as nix expression.
  # The dhall file must not try to import from non-local URLs!
  readDhallFileAsJson = dhallType: file:
    let
      convert = runCommandLocal "dhall-to-json" {} ''
        printf '%s' ${lib.escapeShellArg "${file} : ${dhallType}"} \
          | ${pkgs.dhall-json}/bin/dhall-to-json \
          > $out
      '';
    in builtins.fromJSON (builtins.readFile convert);

  # Additional dependencies that were passed as a list at the very top.
  additionalDeps = lib.listToAttrs
    (map
      ({ src, dhall-file }:
        let file = readDhallFileAsJson ''
            { dependencies : List Text
            , name : Text
            , version : Text
            , sources : List Text }
          '' dhall-file;
        in {
          inherit (file) name;
          value = {
            inherit (file) dependencies version;
            purescriptGlobs = file.sources;
            inherit src;
          };
        })
      additionalPackages);

  # The flat attrset of packages as returned by spago.dhall.
  # We need name, version, dependencies and src for each package.
  # version and dependencies are in json file we generated
  # from spago.dhall.
  # The source derivations are created by spago2nix and
  # exposed as `inputs` from its generated nix file,
  # so we just use those instead of writing our own fetcher.
  flat =
    let
      allPackages = builtins.fromJSON (builtins.readFile ./spago-dependencies.json);
      transitiveDeps = lib.mapAttrs
        (name: src:
           allPackages.${name} // {
             inherit src;
             # spago dependencies have their purs files in src,
             # so we can use this generic glob for all of them.
             purescriptGlobs = [ "src/**/*.purs" ];
           })
        # only maps over the transitive deps we need
        # to enrich them with depnendency information
        (import ./spago-packages.nix { inherit pkgs; }).inputs;

      allDeps = transitiveDeps // additionalDeps;
    in allDeps;

  # Final purescript packageset with derivations.
  # This is an attrset from package name to compiled packages.
  # See docstring of `compileAll` how each value looks.
  # You can get the compilation output of e.g. `control`
  # with `packages.control.compiled`.
  packages =
    transformedSpago
      builders.compileAll
      flat;

 in packages
	{ pkgs }:
	let
	lib = pkgs.lib;
	in
	# list of additional packages to add to the packageset
	additionalPackages:

	# source path of the package (derivation or (filtered) nix path)
	assert (lib.all (pkg: pkg ? src) additionalPackages);
	# dhall file of that package
	assert (lib.all (pkg: pkg ? dhall-file) additionalPackages);

	let
	lib = pkgs.lib;
	runCommandLocal = import ../nix/lib/run-command-local.nix { inherit (pkgs) runCommand; };
	packagesDhall = ./packages.dhall;

	# As introduced by https://github.com/purescript/purescript/commit/8a14cc0f23a6e749c852dd202df91edb2cd8df08
	# a cache-db.json file is a mapping from module name to file compilation time
	# and content hashes. This command merges the json objects and is thus
	# the mappend on these cache dbs.
	merge-purs-cache-db-json = pkgs.writers.writePython3 "merge-purs-cache-db-json" {} ''
	import sys
	import json

	res = {}
	for i in sys.argv[1:]:
	with open(i) as f:
	# right-based merge of dicts
	res = {res, json.load(f)}

	print(json.dumps(res))
	'';

	# make all references to `"dependency"` a reference to `self.dependency`
	# instead, where `self` is the version of the packageset with all dependencies
	# already resolving correctly.
	#
	# buildPackage: a function:
	# { name: String
	# , version: String
	# , dependencies: List a
	# , purescriptGlobs: List PurescriptGlob
	# , src: Drv }
	# -> a
	# if you insert Drv for a, you get out a set of derivations.
	# A PurescriptGlob is a string like "src/*/.purs" that is passed
	# to the purescript compiler to glob for source files.
	#
	# flatPackageAttrset:
	# an attrset mapping from the name of a package
	# to the packages name, version, src and dependencies
	# where `dependencies` is a list of package names as strings.
	transformedSpago = buildPackage: flatPackageAttrset:
	lib.fix (self:
	let depStringToSymbol = str:
	# nix allows you to reference attributes of an attrset by string name.
	# So we are able to map the strings to actual symbols.
	self.${str};
	convertPkg = name: {version, src, dependencies, purescriptGlobs}:
	buildPackage {
	inherit name version src purescriptGlobs;
	dependencies = map depStringToSymbol dependencies;
	};
	in lib.mapAttrs convertPkg flatPackageAttrset
	);

	# functions that can be used as `buildPackage` in `transformedSpago`
	builders =
	let
	# just returns the tree (dag really) of all inputs
	inputs = lib.id;

	# `buildPackage` function that compiles a set of purescript pakages.
	# The result is a set of package names to
	# { src # source code of the package
	# , compiled # compiled output of the package
	# , transitiveSrc # list of the sources
	# # of all transitive dependencies
	# , transitiveCompiled # list of the compiled outputs
	# # of all transitive dependencies
	# }
	compileAll = {name, version, src, dependencies, purescriptGlobs}:
	let
	mergeByName = builtins.foldl' lib.mergeAttrs {};
	# We (abuse?) the fact, that each packages has a unique name
	# to collect all transitive dependencies.
	transitiveSrc = mergeByName
	([{ ${name} = { inherit src purescriptGlobs; }; }]
	++ (map (d: d.transitiveSrc) dependencies));
	transitiveCompiled = mergeByName
	(map (d: d.transitiveCompiled) dependencies);

	# ask the compiler to compile some source code.
	# transitiveSources is a list of { src, purescriptGlob }
	# the script takes $1, the output path.
	purescript-compile = pkgName: transitiveSources:
	pkgs.writers.writeDash "purescript-compile-${pkgName}" (''
	# we disable parallel compilation by setting the purescript
	# runtime to -N1 so it only runs on one thread. nix already
	# parallelizes the builds for us. This leads to a speedup of
	# ~100% for compiling big package sets.
	${pkgs.purescript}/bin/purs \
	compile \
	+RTS -N1 -RTS \
	--output "$1" \
	--verbose-errors \
	''
	+ (lib.escapeShellArgs
	# purs needs the source code of all transitive dependencies
	(lib.concatMap
	# globs are patterns like "src/*/.purs" that are passed
	# to the compiler (not expanded by the shell) to avoid
	# breaching the system argv length limit.
	({ src, purescriptGlobs }:
	map (glob: "${src}/${glob}") purescriptGlobs)
	(transitiveSources))
	));

	compiled = pkgs.runCommand "${name}-${version}" {
	outputs = [ "out" "cachedb" ];
	} ''
	mkdir $out
	# TODO: should be able to give these inputs to purs directly
	${# this is the cache of all dependencies already compiled
	# so we don’t have to rebuild them
	lib.concatStringsSep "\n"
	(lib.mapAttrsToList (_: compiled: ''ln -s "${compiled}"/* $out'')
	transitiveCompiled)}

	# the cache db file contains the timestamps and hashes
	# of the files just compiled. The compiler uses these
	# to determine whether to recompile a file.
	# Since our storepaths all have the same timestamp (1970-01-01)
	# nothing should be recompiled. But the file has to be there anyways.
	${merge-purs-cache-db-json} ${(lib.escapeShellArgs
	(map (dep: dep.compiled.cachedb) dependencies))} \
	> $out/cache-db.json

	${purescript-compile "${name}-${version}"
	# purs needs the source code of all transitive dependencies;
	# transitiveSrc already includes our own
	(lib.mapAttrsToList (_: v: v) transitiveSrc)
	} \
	$out

	# move the combined cache-db to its output
	mv $out/cache-db.json $cachedb

	# TODO: hack to remove the other compiled modules again;
	# everything that is a symlink was added by us above
	for dir in $out/*; do
	test -L "$dir" && rm "$dir" \|\| true
	done
	'';
	in {
	inherit src compiled transitiveSrc;
	transitiveCompiled = transitiveCompiled // { ${name} = compiled; };
	};

	# All transitive dependencies of a package as a “set”
	# (attrset with `null` as values)
	transitiveDependencies = {name, version, src, dependencies}:
	lib.foldl' lib.mergeAttrs { ${name} = null; } dependencies;
	# Create a package set that can be used from psc-package
	# by linking it to `.psc-package/$name` in the project folder
	# and setting the `"set"` field in `psc-package.json` to $name.
	pscPackageSet = srcDeps:
	let
	# convert dhall package set to a json file
	spagoJson = pkgs.runCommand "spago-packages.json" {} ''
	${pkgs.dhall-json}/bin/dhall-to-json \
	<<< "${packagesDhall}" \
	> $out
	'';
	in pkgs.runCommand "spago" {} ''

	# this is the package list psc-package references
	mkdir -p $out/.set
	ln -s '${spagoJson}' $out/.set/packages.json

	# now we link in all packages
	${lib.concatStringsSep "\n"
	( # TODO: escaping?
	lib.mapAttrsToList (name: {version, src}: ''
	mkdir -p "$out/${name}"
	ln -s '${src}' "$out/${name}/${version}"
	'') srcDeps)}
	'';


	in { inherit inputs pscPackageSet compileAll transitiveDependencies; };

	# read dhall file in as JSON, then import as nix expression.
	# The dhall file must not try to import from non-local URLs!
	readDhallFileAsJson = dhallType: file:
	let
	convert = runCommandLocal "dhall-to-json" {} ''
	printf '%s' ${lib.escapeShellArg "${file} : ${dhallType}"} \
	\| ${pkgs.dhall-json}/bin/dhall-to-json \
	> $out
	'';
	in builtins.fromJSON (builtins.readFile convert);

	# Additional dependencies that were passed as a list at the very top.
	additionalDeps = lib.listToAttrs
	(map
	({ src, dhall-file }:
	let file = readDhallFileAsJson ''
	{ dependencies : List Text
	, name : Text
	, version : Text
	, sources : List Text }
	'' dhall-file;
	in {
	inherit (file) name;
	value = {
	inherit (file) dependencies version;
	purescriptGlobs = file.sources;
	inherit src;
	};
	})
	additionalPackages);

	# The flat attrset of packages as returned by spago.dhall.
	# We need name, version, dependencies and src for each package.
	# version and dependencies are in json file we generated
	# from spago.dhall.
	# The source derivations are created by spago2nix and
	# exposed as `inputs` from its generated nix file,
	# so we just use those instead of writing our own fetcher.
	flat =
	let
	allPackages = builtins.fromJSON (builtins.readFile ./spago-dependencies.json);
	transitiveDeps = lib.mapAttrs
	(name: src:
	allPackages.${name} // {
	inherit src;
	# spago dependencies have their purs files in src,
	# so we can use this generic glob for all of them.
	purescriptGlobs = [ "src/*/.purs" ];
	})
	# only maps over the transitive deps we need
	# to enrich them with depnendency information
	(import ./spago-packages.nix { inherit pkgs; }).inputs;

	allDeps = transitiveDeps // additionalDeps;
	in allDeps;

	# Final purescript packageset with derivations.
	# This is an attrset from package name to compiled packages.
	# See docstring of `compileAll` how each value looks.
	# You can get the compilation output of e.g. `control`
	# with `packages.control.compiled`.
	packages =
	transformedSpago
	builders.compileAll
	flat;

	in packages