Detect licenses from sources of derivation with ninka for derivations without meta.license info

check.nix

#? nix-instantiate check.nix | xargs --max-procs 4 -n 1 -- nix-store -k -r --option use-binary-caches false
#? find /nix/store -name *ninka-licenses-builder* | xargs -n1 -I@ nix-store --delete @
#? find /nix/store -name '*ninka-licenses*' | xargs -n1 -I@ -- nix-store --delete @
#? find /nix/store -type d -name *ninka-licenses* | xargs -n1 -I@ -- sh -c 'nix-store --dump @ > `basename @`.nar'
with builtins;
with {
  inherit (import ./nixpkgs/pkgs/top-level/all-packages.nix {
    config = { allowBroken = true; allowUnfree = true; };
  }) stdenv pkgs;
}; let
  ninkaLicensesBuilder = stdenv.mkDerivation {
    name = "ninka-licenses-builder";
    propagatedBuildInputs = [ stdenv pkgs.unzip pkgs.ninka ];
    phases = [ "buildPhase" "installPhase" ];
    buildPhase = ''
      cat > builder.sh <<EOF
        nativeBuildInputs="${stdenv} ${pkgs.unzip} ${pkgs.ninka}"
        phases="unpackPhase buildPhase installPhase"
        source $stdenv/setup
        buildPhase() {
          find . -type f \
            | xargs -n1 -I@ ninka.pl -fd '@' \
            > ninka.licenses
        }
        installPhase() {
          mkdir \$out
          cp ninka.licenses \$out
        }
        genericBuild
EOF
    '';
    installPhase = ''
      mkdir -p $out/bin
      chmod 755 builder.sh
      cp builder.sh $out/bin
    '';
  };

  fixLicensesBuilder = stdenv.mkDerivation {
    name = "fix-licenses-builder";
    propagatedBuildInputs = [ stdenv ];
    phases = [ "buildPhase" "installPhase" ];
    buildPhase = ''
      cat > builder.sh <<EOF
        nativeBuildInputs="${stdenv}"
        phases="unpackPhase buildPhase installPhase"
        source $stdenv/setup
        buildPhase() {
          licenses=\`cat ninka.licenses \
            | cut -d ';' -f 2 \
            | sed -r -e 's/([^,]+)(,\1)*/\1/g' \
                  -e 's/,/\n/g' \
                  -e 's/^\s*(UNKNOWN|NONE|SeeFile|Exception)\s*\$//g' \
                  -e '/^\$/d' \
            | sort -u \
            | xargs\`
          metaFile=\`echo \$metaPosition | cut -d : -f 1\`
          metaLine=\`echo \$metaPosition | cut -d : -f 2\`
          cat > fix-license.sh <<EOD
            cd ./nixpkgs
            nixlicenses=()
            nixlicense=""
            mapLicense() {
              nixlicense=\\\`echo \\\$1 | sed -f ../licenses.sed\\\`
            }
            addLicense() {
              nixlicenses=(\\\`printf "%s\n" \\\''${nixlicenses[@]} \\\$1 | sort -u | xargs\\\`)
            }
            for license in \$licenses; do
              mapLicense \\\$license
              if [[ \\\$nixlicense =~ missingLicenses ]] ; then
                echo "s/^\\\$license\\\\\$//;t;" | xclip -i
                \\\$EDITOR ../licenses.sed
                mapLicense \\\$license
                xclip -i <<EOP
  \\\$nixlicense = spdx {
    spdxId = "\\\$license";
    fullName = "";
  };
EOP
                \\\$EDITOR ./lib/licenses.nix
                git commit -e -a -m "licenses: add \\\$nixlicense" -m "ninka detected \\\$license"
              fi
              addLicense \\\$nixlicense
            done
            if [[ ! \\\''${nixlicenses[@]} = "" && "\$autoFix" = "1" ]] ; then
              #echo '#############################################'
              #echo '#' \$pkgName \$metaFile \$lastMetaLine
              #echo '#############################################'
              sed -n "\$metaLine p" \$metaFile
              #echo "license = with stdenv.lib.licenses; [\\\''${nixlicenses[@]} ];" | xclip -i
              #\\\$EDITOR +\$metaLine \$metaFile
              #git commit -e -a -m "\$pkgName: fix missing license" -m "ninka detected \$licenses"
            fi
EOD
        }
        installPhase() {
          mkdir -p \$out/bin
          chmod 755 fix-license.sh
          cp fix-license.sh \$out/bin
        }
        genericBuild
EOF
    '';
    installPhase = ''
      mkdir -p $out/bin
      chmod 755 builder.sh
      cp builder.sh $out/bin
    '';
  };

  pkgKeys = attrNames pkgs;

  hasName = hasAttr "name";
  isPkgAttrs = x: isAttrs x && hasName x;
  hasPkgMeta = x: isPkgAttrs x && hasAttr "meta" x;
  hasPkgMetaAttr = n: x: hasPkgMeta x && hasAttr n x.meta;
  hasPkgMetaLicense = x: hasPkgMetaAttr "license" x;
  hasPkgMetaPosition = x: hasPkgMetaAttr "position" x;

  # maybe TODO handle assertions with providing sane values instead of catching error
  getPkg = n: (tryEval (getAttr n pkgs)).value;

  getPkgMetaAttr = n: x: if hasPkgMetaAttr n x
                           then getAttr n x.meta
                           else null;

  getPkgMetaPosition = x: getPkgMetaAttr "position" x;

  fileFromMetaPos = x: substring 0 (sub (stringLength x) 3) x;
  metaAttrNames = x: filter (n: n != "position") (attrNames x.meta);
  posMeta = x: n: unsafeGetAttrPos n x.meta;
  posMetaAttrs = x: map (posMeta x) (metaAttrNames x);
  isPosMetaSafe = x: all (y: y.file == (fileFromMetaPos (getPkgMetaPosition x))) (posMetaAttrs x);
  metaLines = x: sort lessThan (map (x: x.line) (posMetaAttrs x));
  last = x: if ((length x) > 0) then elemAt x (length x - 1) else null;

  ninkaLicensesForPkg = x: stdenv.mkDerivation {
    name = "${x.name}-ninka-licenses";
    builder = "${ninkaLicensesBuilder}/bin/builder.sh";
    src = if (hasAttr "src" x) then (getAttr "src" x) else "";
    srcs = if (hasAttr "srcs" x) then (getAttr "srcs" x) else [];
    srcDrv = if (hasAttr "srcDrv" x) then (getAttr "srcDrv" x) else null;
  };

  fixLicensesForPkg = x: n: stdenv.mkDerivation {
    name = "${x.name}-fix-licenses";
    builder = "${fixLicensesBuilder}/bin/builder.sh";
    src = (ninkaLicensesForPkg x);
    metaPosition = x.meta.position;
    lastMetaLine = last (metaLines x);
    autoFix = (isPosMetaSafe x);
    pkgName = n;
  };

  buildLicenseDrv = x: n: if hasPkgMetaPosition x && !hasPkgMetaLicense x
                        then fixLicensesForPkg x n
                        else null;

  missingLicensePkgName = n: buildLicenseDrv (getPkg n) n;
in
  filter (x: !isNull x) (map missingLicensePkgName pkgKeys)

licenses.sed

s/^ClassPathException$/classpathException/;t;
s/^WxException$/wxException/;t;
s/^SameTermsAs$//;t;
s/^SameAsPerl$/gpl1 gpl2 gpl3 artistic1Perl/;t;
s/^GPLv1orArtistic$/gpl1 artistic1 artistic2/;t;
s/^MIToldwithoutSellCMUVariant$/mit/;t;
s/^GPL2orBSD3$/gpl2 bsd3/;t;
s/^sunRPC$//;t;
s/^SimpleLic$/free/;t;
s/^simpleLicense1$/free/;t;
s/^ApacheV2orLGPLgeneric$/asl20 lgpl2 lgpl21 lgpl3/;t;
s/^NPLv1_1$/npl11/;t;
s/^LesserGPLv2$/lgpl2/;t;
s/^LesserGPLnoVersion$/lgpl2 lgpl21 lgpl3/;t;
s/^GPL2orOpenIB$/gpl2/;t;
s/^SimpleOnlyKeepCopyright$/free/;t;
s/^SunSimpleLic$//;t;
s/^LesserGPLv3+$/lgpl3/;t;
s/^EPLv1$/epl10/;t;
s/^GPLv1+$/gpl1 gpl2 gpl3/;t;
s/^Apachev1.0$/asl1/;t;
s/^orGPLv2+orLGPLv2.1+$/gpl2 gpl3 lgpl21 lgpl3/;t;
s/^boostV1Ref$/boost/;t;
s/^ZLIBref$/zlib/;t;
s/^ZLIB$/zlib/;t;
s/^X11mit$/x11/;t;
s/^X11$/x11/;t;
s/^MPLv1_1$/mpl11/;t;
s/^MPL1_1andLGPLv2_1$/mpl11 lgpl21/;t;
s/^MIToldwithoutSellandNoDocumentationRequi$/mit/;t;
s/^MIToldwithoutSell$/mit/;t;
s/^MITmodern$/mit/;t;
s/^MITX11BSDvar$/x11/;t;
s/^LibraryGPLv2+$/lgpl2 lgpl21 lgpl3/;t;
s/^LesserGPLv2.1$/lgpl21/;t;
s/^GPLv2+orLGPLv2.1$/gpl2 gpl3 lgpl21/;t;
s/^GPLv2$/gpl2/;t;
s/^FreeType$/ftl/;t;
s/^Exception$//;t;
s/^BeerWareVer42$/beerware/;t;
s/^BSD4$/bsdOriginal/;t;
s/^BSD3NoWarranty$/bsd3/;t;
s/^BSD2EndorseInsteadOfBinary$/bsd2/;t;
s/^publicDomain$/publicDomain/;t;
s/^BSD2$/bsd2/;t;
s/^BisonException$/bisonException22/;t;
s/^spdxBSD4$/bsdOriginal/;t;
s/^MITold$/mit/;t;
s/^MITX11noNotice$/x11/;t;
s/^MITX11NoSellNoDocDocBSDvar$/x11/;t;
s/^Apachev1.1$/asl11/;t;
s/^spdxBSD2$/bsd2/;t;
s/^orLGPLVer2.1$/lgpl21/;t;
s/^orGPLv3$/gpl3/;t;
s/^openSSLvar1$/openssl/;t;
s/^LibraryGPLv2$/lgpl2/;t;
s/^MITX11$/x11/;t;
s/^LinkException$//;t;
s/^GPLnoVersion$/gpl1 gpl2 gpl3/;t;
s/^CDDLic$/cddl/;t;
s/^autoConfException$/autoconfException2 autoconfException3/;t;
s/^SeeFile$//;t;
s/^MITX11simple$/x11/;t;
s/^FSFUnlimited$/fsful/;t;
s/^Apachev2$/asl20/;t;
s/^spdxBSD3$/bsd3/;t;
s/^BSD3$/bsd3/;t;
s/^LesserGPLv2+$/lgpl2 lgpl21 lgpl3/;t;
s/^LesserGPLv2.1+$/lgpl21 lgpl3/;t;
s/^GPLv3+$/gpl3/;t;
s/^GPLv2+$/gpl2 gpl3/;t;
s/\(.*\)/missingLicense-\1/;