demo for running check-clang on swarming (tested on mac only)

llvm-gn-swarming.patch

Mostly superceded by https://github.com/nico/llvm-project/commit/c43875075d15a58e674792b45b0e96d721296aed

commit 02c27a95ecc0555ba30c363321cc62459e469bff
Author: Nico Weber <[email protected]>
Date:   Fri Jun 15 16:54:38 2018 -0400

    demo for running check-clang on swarming (tested on mac only)
    
    Use like so:
        # Instant:
        gn desc out/gn //clang/test:check-clang runtime_deps | \
            gn/to_isolate.py > out/gn/check-clang.isolate
    
        # Taks about 30s
        time ~/src/chrome/src/tools/swarming_client/isolate.py archive \
            -s out/gn/check-clang.isolated -I isolateserver.appspot.com
    
        # Takes about 6s, prints URL to see progress
        time ~/src/chrome/src/tools/swarming_client/swarming.py trigger \
            -I isolateserver.appspot.com -S chromium-swarm.appspot.com \
            -d pool Chrome -d os Mac \
            -s 82d8725b1705ae79625cbde63a77506fc450a5b4  # From archive stdout
    
    To make this work:
    - use gn data and data_deps so that runtime_deps prints required files
    - make lit config files use relative paths
    - change lit internals to convert those to absolute paths at test time
    - swarming provides inputs as read-only files; tweak one test that
      didn't expect that

diff --git a/clang/lib/Headers/BUILD.gn b/clang/lib/Headers/BUILD.gn
index 71caae72645..a88d75bead2 100644
--- a/clang/lib/Headers/BUILD.gn
+++ b/clang/lib/Headers/BUILD.gn
@@ -151,7 +151,7 @@ copy("arm_headers") {
 }
 
 copy("Headers") {
-  deps = [
+  data_deps = [
     ":arm_headers",
   ]
   sources = files
diff --git a/clang/test/BUILD.gn b/clang/test/BUILD.gn
index eceea2fbf91..ebfe34a4ec0 100644
--- a/clang/test/BUILD.gn
+++ b/clang/test/BUILD.gn
@@ -26,24 +26,26 @@ template("write_lit_config") {
       "@LIT_SITE_CFG_IN_FOOTER@=import lit.llvm; lit.llvm.initialize(lit_config, config)",
       "@CLANG_ANALYZER_WITH_Z3@=0",
       "@CLANG_BINARY_DIR@=" +
-          rebase_path(get_label_info("//clang", "target_out_dir")),
+          rebase_path(get_label_info("//clang", "target_out_dir"),
+                      root_build_dir),
       "@CLANG_BUILD_EXAMPLES@=0",
       "@CLANG_DEFAULT_CXX_STDLIB@=",
-      "@CLANG_SOURCE_DIR@=" + rebase_path("//clang"),
-      "@CLANG_TOOLS_DIR@=" + rebase_path("$root_out_dir/bin"),
+      "@CLANG_SOURCE_DIR@=" + rebase_path("//clang", root_build_dir),
+      "@CLANG_TOOLS_DIR@=" + rebase_path("$root_out_dir/bin", root_build_dir),
       "@CMAKE_CXX_COMPILER@=c++",  # FIXME: better
       "@ENABLE_SHARED@=0",
       "@ENABLE_BACKTRACES@=1",
       "@HOST_ARCH@=x86_64",  # FIXME: better
       "@LLVM_BUILD_MODE@=.",
       "@LLVM_BINARY_DIR@=" +
-          rebase_path(get_label_info("//llvm", "target_out_dir")),
+          rebase_path(get_label_info("//llvm", "target_out_dir"),
+                      root_build_dir),
       "@LLVM_HOST_TRIPLE@=$llvm_host_triple",
       "@LLVM_LIBS_DIR@=",  # needed only for shared builds
       "@LLVM_LIT_TOOLS_DIR@=",  # Intentionally empty, matches cmake build.
       "@LLVM_PLUGIN_EXT@=",  # FIXME
-      "@LLVM_SOURCE_DIR@=" + rebase_path("//llvm"),  # FIXME: assumes monorepo
-      "@LLVM_TOOLS_DIR@=" + rebase_path("$root_out_dir/bin"),
+      "@LLVM_SOURCE_DIR@=" + rebase_path("//llvm", root_build_dir),  # FIXME: assumes monorepo
+      "@LLVM_TOOLS_DIR@=" + rebase_path("$root_out_dir/bin", root_build_dir),
       "@LLVM_USE_SANITIZER@=",
       "@PYTHON_EXECUTABLE@=$python_path",
       "@SHLIBDIR@=",  # FIXME!
@@ -118,7 +120,19 @@ action("check-clang") {
 # //:default depends on it, so that ninja's default target builds all
 # prerequisites for check-clang but doesn't run check-clang itself.
 group("test") {
-  deps = [
+  data = [
+    "./",
+
+    # Read by several test/Index tests.
+    "../bindings/xml/comment-xml-schema.rng",
+
+    # Read by test/Misc/pragma-attribute-supported-attributes-list.test
+    "../include/clang/Basic/Attr.td",
+    "../include/clang/Basic/AttrDocs.td",
+    "../include/clang/Basic/DeclNodes.td",
+    "../include/clang/Basic/StmtNodes.td",
+  ]
+  data_deps = [
     ":lit_site_cfg",
     ":lit_unit_site_cfg",
     "//clang/lib/Headers",
@@ -153,13 +167,13 @@ group("test") {
     "//llvm/utils/not",
   ]
   if (clang_enable_arcmt) {
-    deps += [
+    data_deps += [
       "//clang/tools/arcmt-test",
       "//clang/tools/c-arcmt-test",
     ]
   }
   if (clang_enable_static_analyzer) {
-    deps += [
+    data_deps += [
       "//clang/tools/clang-check",
       "//clang/tools/clang-func-mapping",
     ]
diff --git a/clang/test/Modules/crash-vfs-headermaps.m b/clang/test/Modules/crash-vfs-headermaps.m
index 4f88f3ba119..c187038b22e 100644
--- a/clang/test/Modules/crash-vfs-headermaps.m
+++ b/clang/test/Modules/crash-vfs-headermaps.m
@@ -9,6 +9,9 @@
 // RUN: rm -rf %t
 // RUN: mkdir -p %t/m
 // RUN: cp -a %S/../Preprocessor/Inputs/headermap-rel %t/i
+// If the Inputs directory is marked readonly, `cp -a` will copy the readonly
+// bit. Explicitly make the file writable before writing to it.
+// RUN: chmod +w %t/i/Foo.framework/Headers/Foo.h
 // RUN: echo '// Foo.h' > %t/i/Foo.framework/Headers/Foo.h
 
 // RUN: not env FORCE_CLANG_DIAGNOSTICS_CRASH= TMPDIR=%t TEMP=%t TMP=%t \
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index f5cc72bdf51..b43134623d3 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -1,5 +1,6 @@
 # -*- Python -*-
 
+import errno
 import os
 import platform
 import re
@@ -37,7 +38,12 @@ config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt', 'deb
 config.test_source_root = os.path.dirname(__file__)
 
 # test_exec_root: The root path where tests should be run.
-config.test_exec_root = os.path.join(config.clang_obj_root, 'test')
+config.test_exec_root = os.path.join(os.path.abspath(config.clang_obj_root), 'test')
+try:
+    os.makedirs(config.test_exec_root)
+except OSError as err:
+    if err.errno != errno.EEXIST:
+        raise
 
 llvm_config.use_default_substitutions()
 
diff --git a/clang/tools/driver/BUILD.gn b/clang/tools/driver/BUILD.gn
index 2585661a91c..2ef8adbd93b 100644
--- a/clang/tools/driver/BUILD.gn
+++ b/clang/tools/driver/BUILD.gn
@@ -17,7 +17,7 @@ symlinks = [
 ]
 foreach(target, symlinks) {
   symlink_or_copy(target[0]) {
-    deps = [
+    data_deps = [
       ":clang",
     ]
     source = "clang"
@@ -27,9 +27,9 @@ foreach(target, symlinks) {
 
 # //:clang depends on this symlink target, see comment in //BUILD.gn.
 group("symlinks") {
-  deps = []
+  data_deps = []
   foreach(target, symlinks) {
-    deps += [ ":${target[0]}" ]
+    data_deps += [ ":${target[0]}" ]
   }
 }
 
diff --git a/clang/unittests/BUILD.gn b/clang/unittests/BUILD.gn
index 218b0c6b788..397052e75ff 100644
--- a/clang/unittests/BUILD.gn
+++ b/clang/unittests/BUILD.gn
@@ -1,7 +1,7 @@
 import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
 
 group("unittests") {
-  deps = [
+  data_deps = [
     "AST:ASTTests",
     "ASTMatchers:ASTMatchersTests",
     "ASTMatchers/Dynamic:DynamicASTMatchersTests",
@@ -17,7 +17,7 @@ group("unittests") {
     "Tooling:ToolingTests",
   ]
   if (clang_enable_static_analyzer) {
-    deps += [
+    data_deps += [
       "Analysis:ClangAnalysisTests",
       "Frontend:FrontendTests",
       "StaticAnalyzer:StaticAnalysisTests",
@@ -28,7 +28,7 @@ group("unittests") {
     # to failures, mostly in libclang.VirtualFileOverlay_*.
     # FIXME: Also, the executable can't find libclang.dll since that's
     # in a different directory.
-    deps += [ "libclang:libclangTests" ]
+    data_deps += [ "libclang:libclangTests" ]
   }
   testonly = true
 }
diff --git a/clang/unittests/libclang/BUILD.gn b/clang/unittests/libclang/BUILD.gn
index ce2a0a2477a..352a2f20014 100644
--- a/clang/unittests/libclang/BUILD.gn
+++ b/clang/unittests/libclang/BUILD.gn
@@ -11,7 +11,8 @@ executable("libclangTests") {
     "LibclangTest.cpp",
   ]
   if (host_os == "mac") {
-    ldflags = [ "-Wl,-rpath," + rebase_path("$root_out_dir/lib") ]
+    # Get back from obj/clang/unittests/libclang
+    ldflags = [ "-Wl,-rpath,@loader_path/../../../../lib" ]
   }
   testonly = true
 }
diff --git a/clang/utils/TableGen/tablegen.gni b/clang/utils/TableGen/tablegen.gni
index 7cbe8206e18..598675c5b8f 100644
--- a/clang/utils/TableGen/tablegen.gni
+++ b/clang/utils/TableGen/tablegen.gni
@@ -37,7 +37,11 @@ template("tablegen") {
   }
 
   action(target_name) {
-    forward_variables_from(invoker, [ "visibility" ])
+    forward_variables_from(invoker,
+                           [
+                             "data",
+                             "visibility",
+                           ])
 
     # FIXME: In cross builds, this should depend on the host binary.
     tblgen_target = "//clang/utils/TableGen:clang-tblgen"
diff --git a/gn/symlink.gni b/gn/symlink.gni
index 9b2b86e4d78..fc497fc66ef 100644
--- a/gn/symlink.gni
+++ b/gn/symlink.gni
@@ -4,7 +4,11 @@
 #   output: Where to create the symlink.
 template("symlink_or_copy") {
   action(target_name) {
-    forward_variables_from(invoker, [ "deps" ])
+    forward_variables_from(invoker,
+                           [
+                             "data_deps",
+                             "deps",
+                           ])
 
     # Make a stamp file the output to work around
     # https://github.com/ninja-build/ninja/issues/1186
@@ -13,6 +17,9 @@ template("symlink_or_copy") {
     outputs = [
       stamp,
     ]
+    data = [
+      invoker.output,
+    ]
     script = "//gn/symlink.py"
     args = [
       "--stamp",
diff --git a/gn/to_isolate.py b/gn/to_isolate.py
new file mode 100755
index 00000000000..34ca5c8ba70
--- /dev/null
+++ b/gn/to_isolate.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+"""to_isolate.py takes gn desc runtime_deps output and writes an isolate file.
+
+Example usage:
+    gn desc out/gn //clang/test:check-clang runtime_deps | \
+        python gn/to_isolate.py > out/gn/check-clang.isolate
+"""
+
+import pprint
+import fileinput
+
+
+def main():
+    files = [s.rstrip() for s in fileinput.input()]
+
+    # FIXME: flexible command?
+    cmd = ['python', 'llvm-lit', '-sv',
+        '--param', 'clang_site_config=gen/clang/test/lit.site.cfg.py',
+        '--param', 'clang_unit_site_config=gen/clang/test/Unit/lit.site.cfg.py',
+        '../../clang/test']
+
+    isolate_dict = {
+        'variables': {
+            'command': cmd,
+            'files': files,
+        }
+    }
+    print pprint.pformat(isolate_dict)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/llvm/utils/TableGen/tablegen.gni b/llvm/utils/TableGen/tablegen.gni
index 022f4294e05..8f8c47b4f81 100644
--- a/llvm/utils/TableGen/tablegen.gni
+++ b/llvm/utils/TableGen/tablegen.gni
@@ -34,7 +34,11 @@ template("tablegen") {
   }
 
   action(target_name) {
-    forward_variables_from(invoker, [ "visibility" ])
+    forward_variables_from(invoker,
+                           [
+                             "data",
+                             "visibility",
+                           ])
 
     # FIXME: In cross builds, this should depend on the host binary.
     tblgen_target = "//llvm/utils/TableGen:llvm-tblgen"
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index 2257fb6db67..9b1a4f2da12 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -368,11 +368,11 @@ class LLVMConfig(object):
         # Tweak the PATH to include the tools dir and the scripts dir.
         # Put Clang first to avoid LLVM from overriding out-of-tree clang builds.
         possible_paths = ['clang_tools_dir', 'llvm_tools_dir']
-        paths = [getattr(self.config, pp) for pp in possible_paths
+        paths = [os.path.abspath(getattr(self.config, pp)) for pp in possible_paths
                  if getattr(self.config, pp, None)]
         self.with_environment('PATH', paths, append_path=True)
 
-        paths = [self.config.llvm_shlib_dir, self.config.llvm_libs_dir]
+        paths = [os.path.abspath(self.config.llvm_shlib_dir), os.path.abspath(self.config.llvm_libs_dir)]
         self.with_environment('LD_LIBRARY_PATH', paths, append_path=True)
 
         # Discover the 'clang' and 'clangcc' to use.
@@ -413,7 +413,7 @@ class LLVMConfig(object):
                 ('%target_itanium_abi_host_triple', ''))
 
         self.config.substitutions.append(
-            ('%src_include_dir', self.config.clang_src_dir + '/include'))
+            ('%src_include_dir', os.path.abspath(self.config.clang_src_dir) + '/include'))
 
         # FIXME: Find nicer way to prohibit this.
         self.config.substitutions.append(
@@ -448,15 +448,15 @@ class LLVMConfig(object):
 
         """
         # Tweak the PATH to include the tools dir
-        tool_dirs = [self.config.llvm_tools_dir]
-        lib_dirs = [self.config.llvm_libs_dir]
+        tool_dirs = [os.path.abspath(self.config.llvm_tools_dir)]
+        lib_dirs = [os.path.abspath(self.config.llvm_libs_dir)]
         lld_tools_dir = getattr(self.config, 'lld_tools_dir', None)
         lld_libs_dir = getattr(self.config, 'lld_libs_dir', None)
 
         if lld_tools_dir:
-            tool_dirs = tool_dirs + [lld_tools_dir]
+            tool_dirs = tool_dirs + [os.path.abspath(lld_tools_dir)]
         if lld_libs_dir:
-            lib_dirs = lib_dirs + [lld_libs_dir]
+            lib_dirs = lib_dirs + [os.path.abspath(lld_libs_dir)]
 
         self.with_environment('PATH', tool_dirs, append_path=True)
         self.with_environment('LD_LIBRARY_PATH', lib_dirs, append_path=True)
diff --git a/llvm/utils/lit/lit/util.py b/llvm/utils/lit/lit/util.py
index e20c4ab9015..b3d5f713b9d 100644
--- a/llvm/utils/lit/lit/util.py
+++ b/llvm/utils/lit/lit/util.py
@@ -213,7 +213,7 @@ def which(command, paths=None):
         for ext in pathext:
             p = os.path.join(path, command + ext)
             if os.path.exists(p) and not os.path.isdir(p):
-                return os.path.normpath(p)
+                return os.path.abspath(os.path.normpath(p))
 
     return None
 
diff --git a/llvm/utils/llvm-lit/BUILD.gn b/llvm/utils/llvm-lit/BUILD.gn
index 06a926bc74d..b097ecacab4 100644
--- a/llvm/utils/llvm-lit/BUILD.gn
+++ b/llvm/utils/llvm-lit/BUILD.gn
@@ -28,23 +28,35 @@ action("llvm-lit") {
   # FIXME: Use that instead of ; once it works, https://crbug.com/831613.
   # FIXME: Come up with a way to not repeat the lit cfg paths in 4 places.
   config_map +=
-      "map_config(\"" + rebase_path("//clang/test/lit.cfg.py") + "\", \"" +
-      rebase_path("$root_gen_dir/clang/test/lit.site.cfg.py") + "\");"
+      "map_config(\"" + rebase_path("//clang/test/lit.cfg.py", root_build_dir) +
+      "\", \"" +
+      rebase_path("$root_gen_dir/clang/test/lit.site.cfg.py", root_build_dir) +
+      "\");"
   config_map +=
-      "map_config(\"" + rebase_path("//clang/test/Unit/lit.cfg.py") + "\", \"" +
-      rebase_path("$root_gen_dir/clang/test/Unit/lit.site.cfg.py") + "\");"
+      "map_config(\"" +
+      rebase_path("//clang/test/Unit/lit.cfg.py", root_build_dir) + "\", \"" +
+      rebase_path("$root_gen_dir/clang/test/Unit/lit.site.cfg.py",
+                  root_build_dir) + "\");"
   config_map +=
-      "map_config(\"" + rebase_path("//lld/test/lit.cfg.py") + "\", \"" +
-      rebase_path("$root_gen_dir/lld/test/lit.site.cfg.py") + "\");"
+      "map_config(\"" + rebase_path("//lld/test/lit.cfg.py", root_build_dir) +
+      "\", \"" +
+      rebase_path("$root_gen_dir/lld/test/lit.site.cfg.py", root_build_dir) +
+      "\");"
+  config_map += "map_config(\"" +
+                rebase_path("//lld/test/Unit/lit.cfg.py", root_build_dir) +
+                "\", \"" +
+                rebase_path("$root_gen_dir/lld/test/Unit/lit.site.cfg.py",
+                            root_build_dir) + "\");"
   config_map +=
-      "map_config(\"" + rebase_path("//lld/test/Unit/lit.cfg.py") + "\", \"" +
-      rebase_path("$root_gen_dir/lld/test/Unit/lit.site.cfg.py") + "\");"
+      "map_config(\"" + rebase_path("//llvm/test/lit.cfg.py", root_build_dir) +
+      "\", \"" +
+      rebase_path("$root_gen_dir/llvm/test/lit.site.cfg.py", root_build_dir) +
+      "\");"
   config_map +=
-      "map_config(\"" + rebase_path("//llvm/test/lit.cfg.py") + "\", \"" +
-      rebase_path("$root_gen_dir/llvm/test/lit.site.cfg.py") + "\");"
-  config_map +=
-      "map_config(\"" + rebase_path("//llvm/test/Unit/lit.cfg.py") + "\", \"" +
-      rebase_path("$root_gen_dir/llvm/test/Unit/lit.site.cfg.py") + "\");"
+      "map_config(\"" +
+      rebase_path("//llvm/test/Unit/lit.cfg.py", root_build_dir) + "\", \"" +
+      rebase_path("$root_gen_dir/llvm/test/Unit/lit.site.cfg.py",
+                  root_build_dir) + "\");"
 
   sources = [
     input,
@@ -57,13 +69,16 @@ action("llvm-lit") {
 
     # FIXME: Do all these paths have to be absolute?
     # FIXME: Get all paths from a central gni file (or one per project)
-    "@LLVM_SOURCE_DIR@=" + rebase_path("//llvm"),  # FIXME: assumes monorepo
+    "@LLVM_SOURCE_DIR@=" + rebase_path("//llvm", root_build_dir),  # FIXME: assumes monorepo
     "@LLVM_BINARY_DIR@=" +
-        rebase_path(get_label_info("//llvm", "target_out_dir")),
+        rebase_path(get_label_info("//llvm", "target_out_dir"), root_build_dir),
     "@BUILD_MODE@=.",
     "@LLVM_LIT_CONFIG_MAP@=" + config_map,
   ]
   outputs = [
     output,
   ]
+  data = [
+    "//llvm/utils/lit/",
+  ]
 }



For swarming:

C:\src\llvm-mono>..\tim\tim python ..\chrome\src\tools\swarming_client\swarming.py run  -I isolateserver.appspot.com -S chromium-swarm.appspot.com -d pool Chrome -d os Windows-10 --cipd-package gnuwin
:chromium/llvm-build-tools/gnuwin:b1ba139629576e90ee9bed6f4606f9210952c9ee --env-prefix=PATH gnuwin -s 343039d54cdba1c443d07cd68a2ec2048238110f --shards 6

index 807360ac5b9..dca3103ddb3 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -454,6 +454,13 @@ def main_with_tmp(builtinParameters):
     else:
         run.tests.sort(key = lambda t: (not t.isEarlyTest(), t.getFullName()))

+    # Accept the widely-used GTEST sharding env vars too if set.
+    if (opts.numShards is None and opts.runShard is None and
+        os.environ.get('GTEST_SHARD_INDEX') and
+        os.environ.get('GTEST_TOTAL_SHARDS')):
+        opts.runShard = int(os.environ['GTEST_SHARD_INDEX']) + 1  # runShard is 1-based, gtest 0-based
+        opts.numShards = int(os.environ['GTEST_TOTAL_SHARDS'])
+
     # Then optionally restrict our attention to a shard of the tests.
     if (opts.numShards is not None) or (opts.runShard is not None):
         if (opts.numShards is None) or (opts.runShard is None):