build-purescript-tmp.nix

{ pkgs }:
let
  lib = pkgs.lib;
in
# list of additional packages to add to the packageset
additionalPackages:

# source path of the package (derivation or (filtered) nix path)
assert (lib.all (pkg: pkg ? src) additionalPackages);
# dhall file of that package
assert (lib.all (pkg: pkg ? dhall-file) additionalPackages);

let
  lib = pkgs.lib;
  runCommandLocal = import ../nix/lib/run-command-local.nix { inherit (pkgs) runCommand; };
  packagesDhall = ./packages.dhall;

  # As introduced by https://github.com/purescript/purescript/commit/8a14cc0f23a6e749c852dd202df91edb2cd8df08
  # a cache-db.json file is a mapping from module name to file compilation time
  # and content hashes. This command merges the json objects and is thus
  # the mappend on these cache dbs.
  merge-purs-cache-db-json = pkgs.writers.writePython3 "merge-purs-cache-db-json" {} ''
    import sys
    import json

    res = {}
    for i in sys.argv[1:]:
        with open(i) as f:
            # right-based merge of dicts
            res = {**res, **json.load(f)}

    print(json.dumps(res))
  '';

  # make all references to `"dependency"` a reference to `self.dependency`
  # instead, where `self` is the version of the packageset with all dependencies
  # already resolving correctly.
  #
  # buildPackage: a function:
  #  { name: String
  #  , version: String
  #  , dependencies: List a
  #  , purescriptGlobs: List PurescriptGlob
  #  , src: Drv }
  # -> a
  # if you insert Drv for a, you get out a set of derivations.
  # A PurescriptGlob is a string like "src/**/*.purs" that is passed
  # to the purescript compiler to glob for source files.
  #
  # flatPackageAttrset:
  # an attrset mapping from the name of a package
  # to the packages name, version, src and dependencies
  # where `dependencies` is a list of package names as strings.
  transformedSpago = buildPackage: flatPackageAttrset:
    lib.fix (self:
      let depStringToSymbol = str:
            # nix allows you to reference attributes of an attrset by string name.
            # So we are able to map the strings to actual symbols.
            self.${str};
          convertPkg = name: {version, src, dependencies, purescriptGlobs}:
            buildPackage {
              inherit name version src purescriptGlobs;
              dependencies = map depStringToSymbol dependencies;
            };
      in lib.mapAttrs convertPkg flatPackageAttrset
    );

  # functions that can be used as `buildPackage` in `transformedSpago`
  builders =
    let
      # just returns the tree (dag really) of all inputs
      inputs = lib.id;

      # `buildPackage` function that compiles a set of purescript pakages.
      # The result is a set of package names to
      # { src                # source code of the package
      # , compiled           # compiled output of the package
      # , transitiveSrc      # list of the sources
      #                      # of all transitive dependencies
      # , transitiveCompiled # list of the compiled outputs
      #                      # of all transitive dependencies
      # }
      compileAll = {name, version, src, dependencies, purescriptGlobs}:
        let
          mergeByName = builtins.foldl' lib.mergeAttrs {};
          # We (abuse?) the fact, that each packages has a unique name
          # to collect all transitive dependencies.
          transitiveSrc = mergeByName
            ([{ ${name} = { inherit src purescriptGlobs; }; }]
            ++ (map (d: d.transitiveSrc) dependencies));
          transitiveCompiled = mergeByName
            (map (d: d.transitiveCompiled) dependencies);

          # ask the compiler to compile some source code.
          # transitiveSources is a list of { src, purescriptGlob }
          # the script takes $1, the output path.
          purescript-compile = pkgName: transitiveSources:
            pkgs.writers.writeDash "purescript-compile-${pkgName}" (''
              # we disable parallel compilation by setting the purescript
              # runtime to -N1 so it only runs on one thread. nix already
              # parallelizes the builds for us. This leads to a speedup of
              # ~100% for compiling big package sets.
              ${pkgs.purescript}/bin/purs \
                compile \
                +RTS -N1 -RTS \
                --output "$1" \
                --verbose-errors \
              ''
              + (lib.escapeShellArgs
                  # purs needs the source code of all transitive dependencies
                  (lib.concatMap
                    # globs are patterns like "src/**/*.purs" that are passed
                    # to the compiler (not expanded by the shell) to avoid
                    # breaching the system argv length limit.
                    ({ src, purescriptGlobs }:
                        map (glob: "${src}/${glob}") purescriptGlobs)
                    (transitiveSources))
            ));

          compiled = pkgs.runCommand "${name}-${version}" {
            outputs = [ "out" "cachedb" ];
          } ''
            mkdir $out
            # TODO: should be able to give these inputs to purs directly
            ${# this is the cache of all dependencies already compiled
              # so we don’t have to rebuild them
              lib.concatStringsSep "\n"
               (lib.mapAttrsToList (_: compiled: ''ln -s "${compiled}"/* $out'')
                 transitiveCompiled)}

            # the cache db file contains the timestamps and hashes
            # of the files just compiled. The compiler uses these
            # to determine whether to recompile a file.
            # Since our storepaths all have the same timestamp (1970-01-01)
            # nothing should be recompiled. But the file has to be there anyways.
            ${merge-purs-cache-db-json} ${(lib.escapeShellArgs
                (map (dep: dep.compiled.cachedb) dependencies))} \
                   > $out/cache-db.json

            ${purescript-compile "${name}-${version}"
              # purs needs the source code of all transitive dependencies;
              # transitiveSrc already includes our own
              (lib.mapAttrsToList (_: v: v) transitiveSrc)
             } \
               $out

            # move the combined cache-db to its output
            mv $out/cache-db.json $cachedb

            # TODO: hack to remove the other compiled modules again;
            # everything that is a symlink was added by us above
            for dir in $out/*; do
              test -L "$dir" && rm "$dir" || true
            done
          '';
        in {
          inherit src compiled transitiveSrc;
          transitiveCompiled = transitiveCompiled // { ${name} = compiled; };
        };

      # All transitive dependencies of a package as a “set”
      # (attrset with `null` as values)
      transitiveDependencies = {name, version, src, dependencies}:
        lib.foldl' lib.mergeAttrs { ${name} = null; } dependencies;
      # Create a package set that can be used from psc-package
      # by linking it to `.psc-package/$name` in the project folder
      # and setting the `"set"` field in `psc-package.json` to $name.
      pscPackageSet = srcDeps:
        let
          # convert dhall package set to a json file
          spagoJson = pkgs.runCommand "spago-packages.json" {} ''
            ${pkgs.dhall-json}/bin/dhall-to-json \
              <<< "${packagesDhall}" \
              > $out
          '';
        in pkgs.runCommand "spago" {} ''

          # this is the package list psc-package references
          mkdir -p $out/.set
          ln -s '${spagoJson}' $out/.set/packages.json

          # now we link in all packages
          ${lib.concatStringsSep "\n"
              ( # TODO: escaping?
                lib.mapAttrsToList (name: {version, src}: ''
                  mkdir -p "$out/${name}"
                  ln -s '${src}' "$out/${name}/${version}"
                '') srcDeps)}
        '';


    in { inherit inputs pscPackageSet compileAll transitiveDependencies; };

  # read dhall file in as JSON, then import as nix expression.
  # The dhall file must not try to import from non-local URLs!
  readDhallFileAsJson = dhallType: file:
    let
      convert = runCommandLocal "dhall-to-json" {} ''
        printf '%s' ${lib.escapeShellArg "${file} : ${dhallType}"} \
          | ${pkgs.dhall-json}/bin/dhall-to-json \
          > $out
      '';
    in builtins.fromJSON (builtins.readFile convert);

  # Additional dependencies that were passed as a list at the very top.
  additionalDeps = lib.listToAttrs
    (map
      ({ src, dhall-file }:
        let file = readDhallFileAsJson ''
            { dependencies : List Text
            , name : Text
            , version : Text
            , sources : List Text }
          '' dhall-file;
        in {
          inherit (file) name;
          value = {
            inherit (file) dependencies version;
            purescriptGlobs = file.sources;
            inherit src;
          };
        })
      additionalPackages);

  # The flat attrset of packages as returned by spago.dhall.
  # We need name, version, dependencies and src for each package.
  # version and dependencies are in json file we generated
  # from spago.dhall.
  # The source derivations are created by spago2nix and
  # exposed as `inputs` from its generated nix file,
  # so we just use those instead of writing our own fetcher.
  flat =
    let
      allPackages = builtins.fromJSON (builtins.readFile ./spago-dependencies.json);
      transitiveDeps = lib.mapAttrs
        (name: src:
           allPackages.${name} // {
             inherit src;
             # spago dependencies have their purs files in src,
             # so we can use this generic glob for all of them.
             purescriptGlobs = [ "src/**/*.purs" ];
           })
        # only maps over the transitive deps we need
        # to enrich them with depnendency information
        (import ./spago-packages.nix { inherit pkgs; }).inputs;

      allDeps = transitiveDeps // additionalDeps;
    in allDeps;

  # Final purescript packageset with derivations.
  # This is an attrset from package name to compiled packages.
  # See docstring of `compileAll` how each value looks.
  # You can get the compilation output of e.g. `control`
  # with `packages.control.compiled`.
  packages =
    transformedSpago
      builders.compileAll
      flat;

in packages