Nixpkgs to JSON

nix_try_eval.patch

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index bc253d0a3..e6c348d99 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -867,7 +867,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va
         state.forceValue(*args[0], pos);
         attrs.insert(state.sValue, args[0]);
         attrs.alloc("success").mkBool(true);
-    } catch (AssertionError & e) {
+    } catch (EvalError & e) {
         attrs.alloc(state.sValue).mkBool(false);
         attrs.alloc("success").mkBool(false);
     }

nixpkgs_to_json_v1.nix

{
  description = "Nixpkgs to JSON";

  inputs = {
    nixpkgs_stable.url = "github:NixOS/nixpkgs/nixos-22.05";
    flake-utils.url = "github:numtide/flake-utils/v1.0.0";
  };

  outputs = { self, nixpkgs_stable, flake-utils }:
    with builtins;
    with nixpkgs_stable.lib;
    let
      params = import ./params.nix;

      pkgs = import nixpkgs_stable {
        system = "x86_64-linux";
        config.android_sdk.accept_license = true;
        #  config.allowUnfree = true;
      };
    in
    rec {
      attrCount = x: pipe x [
        (mapAttrsToList nameValuePair)
        length
      ];

      getMeta = attr: acc:
        if isAttrs attr then
        # some attr.meta can be a boolean (wtf)
          if attr ? meta && isAttrs attr.meta then
            let meta = attr.meta;
            in
            {
              path = acc;
              name = if attr ? name then attr.name else lists.last acc;
              meta = (meta
                //
                # sometimes it has a list of lists of platforms
                (if meta ? platforms then
                  { platforms = lists.flatten meta.platforms; }
                else
                  { })
                //
                # ignore maintainers that do not have email
                # like `["roblabla"]`
                (if meta ? maintainers && meta.maintainers ? name then
                  { }
                else
                  { maintainers = [ ]; }
                )
                //
                # normalize .homepage to list
                (if meta ? homepage && !isList meta.homepage then
                  { homepage = [ meta.homepage ]; }
                else
                  { }
                )
              );
            }
          else
          # remove the attr, transforming it on a list of path/meta
            pipe attr [
              (mapAttrsToList
                (name: value:
                  # test if its possible to evaluate
                  # ((match "androidndkPkgs.*" name) == null) && ((debug.traceVal name) != false) && (tryEval (typeOf value == "set")).value
                  # NOTE: cuda is cursed ¯\_(ツ)_/¯
                  if (lists.all (x: x != name) (acc ++ [ "cuda" ]))
                    && (tryEval (isAttrs value)).value
                    && (attrCount value) < 200
                  then
                    pipe (getMeta value (acc ++ [ name ])) [
                      # flatten packages inside packages
                      lists.flatten
                    ]
                  else
                    [ ]
                ))
            ]
        else
          [ ];

      attrToJSON = pipe pkgs [
        (mapAttrsToList nameValuePair)
        (x: lists.sublist params.start params.count x)
        listToAttrs
        (x: getMeta x [ "pkgs" ])
        # flatten packages inside packages
        lists.flatten
        toJSON
      ];
    };
}

nixpkgs_to_json_v2.nix

{
  description = "Nixpkgs to JSON";

  inputs = {
    nixpkgs_stable.url = "github:NixOS/nixpkgs/nixos-22.11";
    nixpkgs_unstable.url = "github:NixOS/nixpkgs/nixos-unstable";
    flake-utils.url = "github:numtide/flake-utils/v1.0.0";
  };

  outputs = { self, nixpkgs_stable, nixpkgs_unstable, flake-utils }:
    with builtins;
    with nixpkgs_stable.lib;
    let
      params = import ./params.nix;

      config = {
        android_sdk.accept_license = true;
        allowUnfree = true;
        allowBroken = true;
        allowUnsupportedSystem = true;
        permittedInsecurePackages = true;
        input-fonts.acceptLicense = true;
        joypixels.acceptLicense = true;
      };

      pkgs_stable = import nixpkgs_stable {
        system = "x86_64-linux";
        config = config;
      };

      pkgs_unstable = import nixpkgs_unstable {
        system = "x86_64-linux";
        config = config;
      };

      pkgs = if params.version == "unstable" then pkgs_unstable else pkgs_stable;

      ignore_list = [
        # not necessary to evaluate
        #"drvAttrs"
        #"inputDerivation"

        "lib" # functions
        "pkgsCross" # cross-compiled packages
        "config" # options
        "nixosTests" # nixpkgs tests
        "tests" # functions tests
        "stdenv" # it has meta and a lot of packages have stdenv internally
      ];

      # how much packages per batch
      # can cause stack overflows!
      offset = params.offset;
      size = params.size;
    in
    rec {
      #
      # API
      #

      get_pkgs = pkgs_unstable;

      # divide and conquer for `findSuperPackagesPath`
      getSuperPackagesPath =
        let
          attr_path = getAttr params.attr_path pkgs;
        in
        pipe attr_path [
          #(x: (debug.traceVal x))
          (mapAttrsToList nameValuePair)
          (lists.sublist (offset * size) size)
          listToAttrs
          (x: findSuperPackagesPath x (params.attr_path))
          lists.flatten
          toJSON
        ];

      # divide and conquer for `findPackagesMeta`
      getPackagesMeta =
        let
          attr_path = getAttr params.attr_path pkgs;
        in
        pipe attr_path [
          (mapAttrsToList nameValuePair)
          (lists.sublist (offset * size) size)
          listToAttrs
          (x: findPackagesMeta x (params.attr_path))
          lists.flatten
          toJSON
        ];

      #
      # Support
      #

      # counts how much attributes a pkg node has
      # -> attrCount {...pkgs...}
      attrCount = x: pipe x [
        (mapAttrsToList nameValuePair)
        length
      ];

      # traverses the pkgs nodes using a list of names
      # -> getAttr [ "pkgs" "ArchiSteamFarm" ] {...pkgs...}
      getAttr = list: attr:
        lists.last
          (foldl'
            (acc: x: (catAttrs x acc))
            [ attr ] # current pkg node
            list # attr_path list
          );

      # GENERIC FUNCTION
      # traverses the pkgs nodes until it find the meta attr
      travelPackages = attr: acc: fn_found: fn_else:
        if isAttrs attr && !(isOption attr) then
          if attr ? meta
            # some attr.meta can be a boolean or string
            # and to avoid cases where the meta attrs can't be
            # evaluated we only get what has platforms
            && (tryEval (isAttrs attr.meta)).value
            && (tryEval (attr.meta.platforms)).success then
            fn_found attr
          else
            pipe attr [
              (mapAttrsToList
                (name: value:
                  if (lists.all (x: x != name) (acc ++ ignore_list))
                    #&& ((debug.traceVal (acc ++ [ name ])) != false)
                    #
                    # if it got rejected by the first if and has meta attr
                    # then its completely broken!!! (and highly explosive)
                    && !(tryEval (isAttrs attr.meta)).value
                    && (tryEval (isAttrs value)).value # only attrs
                    && !(hasAttr name value) # ignore if recursive attr
                  then
                    fn_else name value
                  else
                    [ ]
                ))
            ]
        else
          [ ];

      normalizeMeta = meta: (
        meta
        //
        # sometimes it has a list of lists of platforms
        { platforms = lists.flatten meta.platforms; }
        //
        # ignore maintainers that do not have email
        # like `["roblabla"]`
        (if meta ? maintainers && (lists.any isString meta.maintainers) then
          { maintainers = [ ]; }
        else
          { }
        )
        //
        # normalize .homepage to list
        (if meta ? homepage && !isList meta.homepage then
          { homepage = [ meta.homepage ]; }
        else
          { }
        )
        //
        (if meta ? license then
          {
            licenses = pipe meta [
              (m:
                (if isList m.license then m.license else [ m.license ])
                ++ (if m ? licenses then m.licenses else [ ])
              )
              lists.flatten
              (map (x: (
                if isString x then { shortName = x; } else x
              )
              ))
            ];
          }
        else
          { }
        )

      );

      #
      # Traversers
      #

      # find pkgs nodes that have more than 200 attributes
      # returning results like: {"r":["pkgs","ArchiSteamFarm"]}
      # to be used with findPackagesMeta
      findSuperPackagesPath = attr: acc:
        travelPackages attr acc
          # fn_found
          (attr: [ ])
          # fn_else
          (name: value:
            if (attrCount value) > size then
              { r = acc ++ [ name ]; }
            else
              pipe value [
                (x: findSuperPackagesPath x (acc ++ [ name ]))
                lists.flatten
              ]
          );

      # find pkgs nodes with meta attributes
      # using `normalizeMeta` on the pkgs metadata
      findPackagesMeta = attr: acc:
        travelPackages attr acc
          # fn_found
          (attr:
            {
              path = acc;
              version =
                if (tryEval attr.version).success
                  && attr.version != "" then
                  attr.version
                else
                  if (tryEval attr.meta.version).success
                    && attr.meta.version != "" then
                    attr.meta.version
                  else
                    if (tryEval attr.meta.name).success
                      && attr.meta.name != "" then
                      attr.meta.name
                    else
                      lists.last acc;
              meta =
                if (tryEval attr.meta).success then
                  normalizeMeta attr.meta
                else
                  { };
            }
          )
          # fn_else
          (name: value:
            if (attrCount value) > size then
              [ ]
            else
              pipe value [
                (x: findPackagesMeta x (acc ++ [ name ]))
                lists.flatten
              ]
          );
    };
}

nixpkgs_to_json_v3.nix

# nix-eval-jobs --meta --flake path:priv/nix_eval2#get
{
  description = "Nixpkgs to JSON";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11";
  };

  outputs = { self, nixpkgs, }:
    with builtins;
    with nixpkgs.lib;
    let
      config = {
        android_sdk.accept_license = true;
        allowUnfree = true;
        allowBroken = true;
        allowUnsupportedSystem = true;
        permittedInsecurePackages = true;
        input-fonts.acceptLicense = true;
        joypixels.acceptLicense = true;
        segger-jlink.acceptLicense = true;
      };

      pkgs = import nixpkgs {
        system = "x86_64-linux";
        config = config;
      };

      ignore_list = [
        "lib" # functions
        "pkgsCross" # cross-compiled packages
        "config" # options
        "nixosTests" # nixpkgs tests
        "tests" # functions tests
        "stdenv" # it has meta and a lot of packages have stdenv internally
        "iosSdkPkgs"
      ];
    in
    rec {
      #
      # API
      #

      get = pkgs;

      getSize = pipe pkgs [
        (mapAttrsToList nameValuePair)
        length
        toJSON
      ];

      getMidSize = { size ? 100, offset ? 0 }:
        pipe
          pkgs
          [
            (mapAttrsToList nameValuePair)
            (lists.sublist (offset * size) size)
            listToAttrs
            (x: findPackagesMeta x [ ])
            lists.flatten
            length
            # WORKAROUND: Nix does not accept `--arg` on flakes[1], so we use
            # nix-eval-jobs, and them we get the size using a derivation name
            # [1] - https://github.com/NixOS/nix/issues/3843#issuecomment-661710951
            (x: derivation {
              name = toJSON x;
              builder = ":";
              system = "x86_64-linux";
            })
          ];

      getSlice = { size ? 100, offset ? 0, mid_size ? 1, mid_offset ? 0 }:
        pipe
          pkgs
          [
            (mapAttrsToList nameValuePair)
            # WORKAROUND: Nix can't evaluate every package, even with
            # nix-eval-jobs, so we evaluate one by one :)
            (lists.sublist (offset * size) size)
            listToAttrs
            (x: findPackagesMeta x [ ])
            lists.flatten
            # WORKAROUND: Nix can't handle big amounts of data, and some
            # grouped packages are too big for it, so we need to split the split
            (lists.sublist (mid_offset * mid_size) mid_size)
            (foldl (attrsets.recursiveUpdate) { })
          ];

      #
      # Support
      #

      # GENERIC FUNCTION
      # traverses the pkgs nodes until it find the meta attr
      travelPackages = attr: acc: fn_found: fn_else:
        if isAttrs attr && !(isOption attr) then
          if attrsets.isDerivation attr
          #&& ((debug.traceVal acc) != false) 
          then
          #{name = acc; value = attr;}
            fn_found (attrsets.setAttrByPath acc attr)
          else
            pipe attr [
              (mapAttrsToList
                (name: value:
                  if (lists.all (x: x != name) (acc ++ ignore_list))
                    && !((length acc) > 1)
                    && !(strings.hasPrefix "androidndkPkgs" name)
                    #&& ((debug.traceVal (acc ++ [ name ])) != false)
                    #
                    # if it got rejected by the first if and has meta attr
                    # then its completely broken!!! (and highly explosive)
                    #&& !(tryEval (attr ? meta && isAttrs attr.meta)).value
                    && (tryEval (isAttrs value)).value # only attrs
                    #&& ((debug.traceVal (acc ++ [ name ])) != false)
                    && !(tryEval (hasAttr name value)).value # ignore if recursive attr
                  #&& ((debug.traceVal (acc ++ [ name ])) != false)
                  then
                    fn_else name value
                  else
                    [ ]
                ))
            ]
        else
          [ ];

      #
      # Traversers
      #

      # find pkgs nodes with meta attributes
      # using `normalizeMeta` on the pkgs metadata
      findPackagesMeta = attr: acc:
        travelPackages attr acc
          # fn_found
          (attr: attr)
          # fn_else
          (name: value:
            pipe value [
              (x: findPackagesMeta x (acc ++ [ name ]))
              lists.flatten
            ]
          );
    };
}