mk-vendor-file.diff

--- pkgs/applications/networking/browsers/chromium-git/mk-vendor-file.py	2019-12-30 16:30:56.947645764 -0500
+++ /home/danielrf/NixDroid/apks/chromium/mk-vendor-file.py	2019-12-30 15:18:29.068764104 -0500
@@ -1,5 +1,6 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -i python -p python2 nix git nix-prefetch-git
+#!nix-shell -i python -p python2 nix git nix-prefetch-git cipd -I nixpkgs=../../pkgs.nix
+# TODO: Include cipd above
 
 from __future__ import print_function
 
@@ -12,9 +13,9 @@
 import subprocess
 import sys
 
-BASEDIR = "/mnt/media/chromium"
+BASEDIR = "/tmp/z"
 
-SKIP_DEPS = [ "src" ]
+SKIP_DEPS = [ "src" "src/tools/luci-go" ]
 
 def hash_path(path):
     sha256 = subprocess.check_output(["nix", "hash-path", "--base32", "--type", "sha256", path]).strip()
@@ -32,6 +33,12 @@
         "--fetch-submodules"])
     return hash_path(path)
 
+def checkout_cipd(package, version, path):
+    os.mkdir(path)
+    subprocess.check_call(["cipd", "init", path])
+    subprocess.check_call(["cipd", "install", "-root", path, package, version])
+    return hash_path(path)
+
 def nix_str_git(path, dep):
     return '''  %(path)-90s = fetchgit { url = %(url)-128s; rev = "%(rev)s"; sha256 = "%(sha256)s"; };\n''' % {
         "path": '"' + path + '"',
@@ -40,6 +47,22 @@
         "sha256": dep["sha256"],
     }
 
+def nix_str_cipd(path, dep):
+    def _fetchcipd_str(p):
+        return 'fetchcipd { package = "%(package)s"; version = "%(version)s"; sha256 = "%(sha256)s"; }' % p
+
+    if len(dep['packages']) == 1:
+        src_str = _fetchcipd_str(dep['packages'][0])
+    else:
+        nix_paths = '    \n'.join([ "(%s)" % (_fetchcipd_str(p),) for p in dep['packages'] ])
+        src_str = '''
+            symlinkJoin { name = "cipd-joined"; paths = [
+            %s
+            ]; }
+        ''' % nix_paths
+
+    return "  %-90s = %s;\n" %  ('"' + path + '"', src_str)
+
 def make_vendor_file(chromium_version, target_os):
     topdir = os.path.join(BASEDIR, chromium_version)
     if not os.path.isdir(topdir):
@@ -157,21 +180,45 @@
                     }
 
                 elif fields['dep_type'] == "cipd":
-                    pass # Left unimplemented in nixpkgs for simplicity. Ping danielfullmer if it is needed
+                    packages = []
+                    for p in fields['packages']:
+                        package, version = p['package'], p['version']
+                        dirname = (package + '_' + version).replace('/', '_').replace(':', '') # TODO: Better path normalization
+                        memoized_path = os.path.join(BASEDIR, dirname)
+
+                        if os.path.exists(memoized_path + ".sha256"): # memoize hash
+                            sha256 = open(memoized_path + ".sha256").read()
+                        else:
+                            shutil.rmtree(memoized_path, ignore_errors=True)
+                            sha256 = checkout_cipd(package, version, memoized_path)
+                            open(memoized_path + ".sha256", "w").write(sha256)
+
+                        packages.append({
+                            "package": package,
+                            "version": version,
+                            "sha256": sha256,
+                        })
+
+                    deps[path] = {
+                        "packages": packages,
+                        "dep_type": "cipd",
+                    }
 
                 else:
                     raise ValueError("Unrecognized dep_type", fields['dep_type'])
 
     with open('vendor-%s.nix' % chromium_version, 'w') as vendor_nix:
         vendor_nix.write("# GENERATED BY 'mk-vendor-file.py %s' for %s\n" % (chromium_version, ", ".join(target_os)))
-        vendor_nix.write("{fetchgit, fetchurl, runCommand}:\n");
+        vendor_nix.write("{fetchgit, fetchcipd, fetchurl, runCommand, symlinkJoin}:\n");
         vendor_nix.write("{\n");
 
         for path, dep in sorted(deps.iteritems()):
             if dep['dep_type'] == "git":
                 vendor_nix.write(nix_str_git(path, dep))
+            if dep['dep_type'] == "cipd":
+                vendor_nix.write(nix_str_cipd(path, dep))
 
-        # Some additional non-git sources
+        # Some additional non-git/cipd sources
         for path, name in [("src/third_party/node/node_modules", "chromium-nodejs"),
                         ("src/third_party/test_fonts/test_fonts", "chromium-fonts")]:
             sha1 = open(os.path.join(topdir, path + ".tar.gz.sha1")).read().strip()
@@ -186,6 +233,45 @@
 '';
 ''' % { "path": path, "name": name, "sha1": sha1 })
 
+        # condition: checkout_android or checkout_linux
+        # TODO: Memoize
+        gs_url = open(os.path.join(topdir, 'src/chrome/android/profiles/newest.txt')).read().strip()
+        GS_HTTP_URL = "https://storage.googleapis.com/"
+        gz_prefix = "gs://"
+        if gs_url.startswith(gz_prefix):
+            url = GS_HTTP_URL + gs_url[len(gz_prefix):]
+        else:
+            url = GS_HTTP_URL + "chromeos-prebuilt/afdo-job/llvm/" + gs_url
+        sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", url]).strip()
+        path = "src/chrome/android/profiles/afdo.prof"
+        vendor_nix.write(
+'''
+"%(path)s" = runCommand "download_afdo_profile" {} ''
+    bzip2 -d -c ${fetchurl {
+                url  = "%(url)s";
+                sha256 = "%(sha256)s";
+            }} > $out
+'';
+''' % { "path": path, "url": url, "sha256": sha256 })
+
+        local_scope = {}
+        global_scope = {"__file__": "update.py"}
+        exec(open(os.path.join(topdir, "src/tools/clang/scripts/update.py")).read(), local_scope, global_scope) # TODO: Safety?
+        url = '%s/Linux_x64/clang-%s.tgz' % (global_scope['CDS_URL'], global_scope['PACKAGE_VERSION'])
+        sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", url]).strip()
+        path = "src/third_party/llvm-build/Release+Asserts"
+        vendor_nix.write(
+'''
+"%(path)s" = runCommand "download_upstream_clang" {} ''
+    mkdir $out
+    tar xf ${fetchurl {
+                url  = "%(url)s";
+                sha256 = "%(sha256)s";
+            }} -C $out
+'';
+''' % { "path": path, "url": url, "sha256": sha256 })
+
+
         vendor_nix.write("}\n")
 
 def main():