Nix flakes for Python projects and development

flake.nix

{
  description = "Flake to manage python environment";

  inputs = {
    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
    mach-nix.url = "mach-nix/3.5.0";
    flake-utils.url = "github:numtide/flake-utils";
  };

  outputs = inputs@ { self, nixpkgs, mach-nix, flake-utils, ... }:
    let
      python = "python38";
      pypiDataRev = "master";
      pypiDataSha256 = "0arndci48iczc7ingkyarcjcq2vdpygh2rw3rm01lcdy2z4wv2pl";
    in
    flake-utils.lib.eachDefaultSystem (system:
      let
        pkgs = nixpkgs.legacyPackages.${system};
        mach = import mach-nix { inherit pkgs python pypiDataRev pypiDataSha256; };
        lib = pkgs.lib;
        lib_req = import ./lib.nix { inherit pkgs lib; };

        # pyDevEnv = (pkgs.${python}.withPackages
        #   (ps: with ps; [ pip black pylint pyflakes isort ]));

        pyEnv = mach.mkPython {
          requirements = lib_req.requirements ./requirements/dev.txt;
        };
      in
      {
        devShells.default = pkgs.mkShellNoCC {
          packages = with pkgs; [
            pyEnv
          ];
        };
      }
    );
}

jupyterlab-flake.nix

{
  description = "JupyterLab with Python and R";

  inputs = {
    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
    jupyterWith.url = "github:tweag/jupyterWith";
    flake-utils.url = "github:numtide/flake-utils";
    devshell.url = "github:numtide/devshell";
  };

  outputs = { self, nixpkgs, flake-utils, jupyterWith, ... }@inputs:
    flake-utils.lib.eachDefaultSystem
      (system:
        let
          pkgs = import nixpkgs {
            system = system;
            # overlays = [ inputs.devshell.overlay ];
            overlays = nixpkgs.lib.attrValues jupyterWith.overlays ++ [
              inputs.devshell.overlay
              (final: prev: {
                python39 = prev.python39.override {
                  packageOverrides = (pfinal: pprev: {
                    pyopenssl = pprev.pyopenssl.overrideAttrs (old: {
                      meta = old.meta // { broken = false; };
                    });
                  });
                };
                python310 = prev.python310.override {
                  packageOverrides = (pfinal: pprev: {
                    pyopenssl = pprev.pyopenssl.overrideAttrs (old: {
                      meta = old.meta // { broken = false; };
                    });
                  });
                };
              })
            ];
          };
          RStudio-with-packages = pkgs.rstudioWrapper.override {
            packages = with pkgs.rPackages; [ ggplot2 dplyr xts ];
          };
          R-with-my-packages = pkgs.rWrapper.override {
            packages = with pkgs.rPackages; [
              tidyverse
              magrittr
              purrr
              arrow
              digest
              stringi
              here
            ];
          };
          iR = pkgs.kernels.iRWith {
            name = "nixpkgs";
            # Libraries to be available to the kernel.
            packages = p: with p; [
              tidyverse
            ];
          };
          iPython = pkgs.kernels.iPythonWith {
            name = "Python-env";
            packages = p: with p; [ sympy numpy ];
            ignoreCollisions = true;
          };
          jupyterEnvironment = pkgs.jupyterlabWith {
            kernels = [ iPython ];
          };
          pythonEnvironment = pkgs.python39.withPackages (p: with p; [
            boto3
          ]);
        in
        rec
        {
          apps.jupyterlab = {
            type = "app";
            program = "${jupyterEnvironment}/bin/jupyter-lab";
          };
          defaultApp = apps.jupyterlab;
          # devShell = jupyterEnvironment.env;
          # devShells.default = import ./shell.nix { inherit pkgs; };
          devShell = pkgs.devshell.mkShell {
            packages = [
              # RStudio-with-my-packages
              jupyterEnvironment
              R-with-my-packages
              # pythonEnvironment
            ];
          };
        }
      );
}

lib.nix

{ pkgs, lib }:

let
  lines = file: lib.splitString "\n" (builtins.readFile file);

  # Very simple test to look for lines starting with -r
  isRequire = lib.hasPrefix "-r ";

  # Very simple way to extract filename from a -r line
  requireFile = dir: file: "${dir}/${lib.removePrefix "-r " file}";

  # Return list of lines from a requirements file, resursively including
  # files included using -r
  requires = file:
    let part = lib.partition isRequire (lines file); # right = -r lines, wrong = rest
    in
    part.wrong ++ (
      if builtins.length part.right == 0
      then [ ]
      else builtins.concatMap (x: requires (requireFile (builtins.dirOf file) x)) part.right
    );
in
{
  # Helper for mach-nix requirements. It looks for includes of the form "-r filename.txt" and recursively
  # inlines them instead as mach-nix doesn't currently support this feature.
  requirements = file: builtins.concatStringsSep "\n" (requires file);
}

mach-build-private-py-package.nix

{
  description = "Flake to manage python environment";

  inputs = {
    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
    mach-nix.url = "mach-nix/3.5.0";
    flake-utils.url = "github:numtide/flake-utils";
  };

  outputs = inputs@ { self, nixpkgs, mach-nix, flake-utils, ... }:
    let
      python = "python39";
      pypiDataRev = "master";
      pypiDataSha256 = "0arndci48iczc7ingkyarcjcq2vdpygh2rw3rm01lcdy2z4wv2pl";
    in
    flake-utils.lib.eachDefaultSystem (system:
      let
        pkgs = nixpkgs.legacyPackages.${system};
        mach = import mach-nix { inherit pkgs python pypiDataRev pypiDataSha256; };

        pyDevEnv = (pkgs.${python}.withPackages
          (ps: with ps; [ pip black pylint pyflakes isort ]));

        # NOTE: just for demonstration
        private-package = mach.buildPythonPackage {
          src = builtins.fetchGit {
            url = "git+ssh://[email protected]/github-user/private-package";
            ref = "master";
            rev = "5bf4f23e84a8ba280b6ebb4979e1a6f366f8c291";
          };
          pname = "private-package";
          version = "0.6.1";
          requirements = ''
            boto3>=1.17.2
            botocore>=1.20.2
            certifi>=2020.12.5
            chardet>=4.0.0
            Deprecated>=1.2.11
            gitdb>=4.0.5
            GitPython>=3.1.13
            idna>=2.10
            isort>=5.7.0
            jmespath>=0.10.0
            pep8>=1.7.1
            PyGithub>=1.54.1
            PyJWT>=1.7.1
            python-dateutil>=2.8.1
            requests>=2.25.1
            s3transfer>=0.3.4
            s3path>=0.3.4
            six>=1.15.0
            smmap>=3.0.5
            urllib3>=1.26.3
            wrapt>=1.12.1
          '';
        };

        pyEnv = mach.mkPython {
          requirements = builtins.readFile ./requirements.txt;
          packagesExtra = [
            private-package
          ];
        };
      in
      {
        devShells.default = pkgs.mkShellNoCC {
          packages = with pkgs; [
            pyEnv
            pyDevEnv
            treefmt
            postgresql
          ];
        };
      }
    );
}