Very provisional ctypes interface to Clang Incremental

README.md

Note: I'm assuming that "conda-forge" is your only conda channel (for instance, installed with Miniforge).

Step 1: download all of these files.

Step 2: install 14.0.0.rc2 of clangdev:

conda install -c conda-forge/label/llvm_rc clangdev==14.0.0.rc2 libclang==14.0.0.rc2

Step 3: update the conda_path (line 7) of CMakeLists.txt.

Step 4: build it:

cmake .
make

Step 5: in the right directory, if all the paths are set, you should be able to

python all_call_fn.py

and see meaningful output.

Step 6: install the HEAD of Awkward Array, following the developer installation instructions (either localbuild or pip install .).

Step 7: the examples in scikit-hep/awkward-1.0#1359 should work.

all_call_fn.py

# Author Vassil Vassilev
import ctypes

# Load our C wrapper.
libInterop = ctypes.CDLL("./libAArrayclangInterpreter.so")

class Cpp:
  # Responsible for finding the relevant C/C++ type.
  _get_scope = libInterop.Clang_LookupName
  _get_scope.restype = ctypes.c_size_t
  _get_scope.argtypes = [ctypes.c_char_p]

  # Responsible for calling low-level function pointers.
  _get_funcptr = libInterop.Clang_GetFunctionAddress
  _get_funcptr.restype = ctypes.c_void_p
  _get_funcptr.argtypes = [ctypes.c_size_t]

  def __getattr__(self, name):
    self.fn_name = name
    return self

  def __call__(self, *args, **kwds):
    # In real life this would normally go through the interop layer to know
    # whether to pass pointer, reference, or value of which type etc.
    proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)
    # Find the C++ function and store a handle.
    fn_handle = self._get_scope(self.fn_name.encode("ascii"))
    self._funcptr = proto(self._get_funcptr(fn_handle))
    # See the comment above. Possibly do type conversions.
    #a0 = ctypes.cast(args[0].cppobj, ctypes.POINTER(ctypes.c_void_p))
    #a1 = args[1]
    #a2 = args[2].cppobj
    return self._funcptr(*args, **kwds)

class InteractiveCppEnv:
  # Responsible for finding the relevant compiling C++ code.
  _cpp_compile = libInterop.Clang_Parse
  _cpp_compile.argtypes = [ctypes.c_char_p]

  Cpp = Cpp()

  def cpp_compile(self, arg):
    return self._cpp_compile(arg.encode("ascii"))

# Demo
if __name__ == '__main__':
  CppEnv = InteractiveCppEnv()
  CppEnv.cpp_compile(r"""\
  extern "C" int printf(const char*,...);
  int demo_call(int v) { return printf("I was called with %d\n", v); }
""")

  print("K", CppEnv.Cpp.demo_call(1))

CMakeLists.txt

# Author: Vassil Vassilev

project(aarray-example)
cmake_minimum_required(VERSION 3.10)

#conda install -c conda-forge/label/llvm_rc clangdev=14.0.0.rc2
set(conda_path "/home/jpivarski/mambaforge/envs/vassil-clang-python/")

set(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL)
set(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC)

#find_package(LLVM 13.0.1 REQUIRED CONFIG HINTS ${conda_path}/lib/cmake/llvm NO_DEFAULT_PATH)
find_package(Clang REQUIRED HINTS ${conda_path}/lib/cmake/clang NO_DEFAULT_PATH)

message(STATUS "Found LLVM version ${LLVM_VERSION}")

if (NOT LLVM_VERSION VERSION_GREATER_EQUAL 14.0)
  message(FATAL_ERROR "Error you need llvm version 14 or later")
endif()

list(APPEND CMAKE_MODULE_PATH "${LLVM_DIR}")

include(AddLLVM)

llvm_add_library(AArrayclangInterpreter SHARED
  InterpreterUtils.cpp
  LINK_LIBS
  clangBasic
  clangInterpreter)

target_include_directories(AArrayclangInterpreter PUBLIC ${CLANG_INCLUDE_DIRS})

InterpreterUtils.cpp

#include "InterpreterUtils.h"

#include "clang/AST/Mangle.h"
#include "clang/Interpreter/Interpreter.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Sema/Lookup.h"
#include "clang/Sema/TemplateDeduction.h"

#include "llvm/Support/TargetSelect.h"

#include <memory>
#include <vector>
#include <sstream>

using namespace clang;

static std::unique_ptr<clang::Interpreter> CreateInterpreter() {
  std::vector<const char *> ClangArgv = {"-Xclang", "-emit-llvm-only", "-fPIC", "-fno-rtti", "-fno-exceptions", "-O3"};
  auto CI = llvm::cantFail(IncrementalCompilerBuilder::create(ClangArgv));
  return llvm::cantFail(Interpreter::create(std::move(CI)));
}

struct LLVMInitRAII {
  LLVMInitRAII() {
    llvm::InitializeNativeTarget();
    llvm::InitializeNativeTargetAsmPrinter();
  }
  ~LLVMInitRAII() {llvm::llvm_shutdown();}
} LLVMInit;

auto Interp = CreateInterpreter().release();

static LookupResult LookupName(Sema &SemaRef, const char* Name) {
  ASTContext &C = SemaRef.getASTContext();
  DeclarationName DeclName = &C.Idents.get(Name);
  LookupResult R(SemaRef, DeclName, SourceLocation(), Sema::LookupOrdinaryName);
  SemaRef.LookupName(R, SemaRef.TUScope);
  assert(!R.empty());
  return R;
}

Decl_t Clang_LookupName(const char* Name, Decl_t Context /*=0*/) {
  return LookupName(Interp->getCompilerInstance()->getSema(), Name).getFoundDecl();
}

FnAddr_t Clang_GetFunctionAddress(Decl_t D) {
  clang::NamedDecl *ND = static_cast<clang::NamedDecl*>(D);
  clang::ASTContext &C = ND->getASTContext();
  std::unique_ptr<MangleContext> MangleC(C.createMangleContext());
  std::string mangledName;
  llvm::raw_string_ostream RawStr(mangledName);
  MangleC->mangleName(ND, RawStr);
  auto Addr = Interp->getSymbolAddress(RawStr.str());
  if (!Addr)
    return 0;
  return *Addr;
}

void Clang_Parse(const char* Code) {
  llvm::cantFail(Interp->ParseAndExecute(Code));
}

InterpreterUtils.h

typedef void* Decl_t;
typedef unsigned long FnAddr_t;

extern "C" {
  void Clang_Parse(const char* Code);
  FnAddr_t Clang_GetFunctionAddress(Decl_t D);
  Decl_t Clang_LookupName(const char* Name, Decl_t Context = 0);
}

The link to the PR should be: scikit-hep/awkward#1359

Also if some inexperienced with conda is trying this (me :)) you can use -c conda-forge to set your conda channel.

conda install -c conda-forge -c conda-forge/label/llvm_rc clangdev==14.0.0.rc2 libclang==14.0.0.rc2

I think installing a single package with -c conda-forge does not ensure that all of its dependencies are consistent with the constraints in the conda-forge channel.

What I used to do for this is set up conda to only use the conda-forge channel, no others, but what I do now is use Miniforge. Either way, the -c will no longer need to be included and the set of dependencies will be guaranteed consistent.

However, this is a quick-setup, not intended for many people, so if your setup works, then that's good enough.