Using Swift libraries in Python

_INSTRUCTIONS.md

Using Swift libraries in Python

So... this is obviously totally, 100%, like for. real. not. supported. by. Apple. …yet?

But still... I thought it was pretty badass. And, seeing how there's already a Swift buildpack for Heroku you could move some slow code into Swift can call it as a library function. But, you know, not in production or anything. That would be silly, right?

Now, having said that, the actual Python/Swift interop may have bugs. I'll leave that as an exercise to the reader.

How to get Python code calling Swift functions:

1. Get an Ubuntu 15.10 system running. These instructions may also work on other versions of Ubuntu (and Heroku uses Ubuntu server).

2. Install swift per the directions for Ubuntu 15.10 (or with heroku, just add the heroku buildback)

3. Install dependencies: sudo apt-get install python-dev

4. Wherever you put your copy of swift (which you unzipped in the installation instruction)…

   cd SWIFT-ROOT/usr/lib/swift/
   mkdir python
   touch module.map

5. Copy in the contents of the module.map file below (into YOUR-SWIFT-ROOT/usr/lib/swift/python/module.map)

6. TADA! Now you can create the sit.swift and sit.py files anywhere you want (but in the same directory with each other) and compile them like so:

# creates "libsit.so" (or on OS X, "libsit.dylib")
swiftc -emit-library sit.swift

Now this works!

$ python sit.py
Hamburger Hamburger
Bang! Bang!

Unfortunately there isn't a way to avoid the crazy mangled names in the Python code right now (that I know of). So as you create Swift functions, you'll have to find the mangled name using nm:

nm libsit.so | grep YOUR_SWIFT_FN_NAME

…again replacing .so with .dylib on OS X. Note: on OS X you get two underscores as a prefix, but you have to remove one for it to work in Python.

One final note… for some reason the mangled names are different on OS X and Linux.

module.map

framework module Python [extern_c] [system] {
  umbrella header "/usr/include/python2.7/Python.h"

  export *
  module * { export * }

  explicit module bitset { 
    header "/usr/include/python2.7/bitset.h"
    export *
  }

  explicit module bytes_methods { 
    header "/usr/include/python2.7/bytes_methods.h"
    export *
  }

  explicit module cStringIO { 
    header "/usr/include/python2.7/cStringIO.h"
    export *
  }

  explicit module datetime { 
    header "/usr/include/python2.7/datetime.h"
    export *
  }

  explicit module errcode { 
    header "/usr/include/python2.7/errcode.h"
    export *
  }

  explicit module frameobject { 
    header "/usr/include/python2.7/frameobject.h"
    export *
  }

  explicit module graminit { 
    header "/usr/include/python2.7/graminit.h"
    export *
  }

  explicit module grammar { 
    // rdar://problem/19484773
    requires !cplusplus
    header "/usr/include/python2.7/grammar.h"
    export *
  }

  explicit module longintrepr { 
    header "/usr/include/python2.7/longintrepr.h"
    export *
  }

  explicit module marshal { 
    header "/usr/include/python2.7/marshal.h"
    export *
  }

  explicit module metagrammar { 
    header "/usr/include/python2.7/metagrammar.h"
    export *
  }

  explicit module node { 
    header "/usr/include/python2.7/node.h"
    export *
  }

  explicit module opcode { 
    header "/usr/include/python2.7/opcode.h"
    export *
  }

  explicit module osdefs { 
    header "/usr/include/python2.7/osdefs.h"
    export *
  }

  explicit module parsetok { 
    // rdar://problem/19484773
    requires !cplusplus
    header "/usr/include/python2.7/parsetok.h"
    export *
  }

  explicit module pgen { 
    // rdar://problem/19484773
    requires !cplusplus
    header "/usr/include/python2.7/pgen.h"
    export *
  }

  explicit module pgenheaders { 
    header "/usr/include/python2.7/pgenheaders.h"
    export *
  }

  explicit module py_curses { 
    header "/usr/include/python2.7/py_curses.h"
    export *
  }

  explicit module pygetopt { 
    header "/usr/include/python2.7/pygetopt.h"
    export *
  }

  explicit module pythread { 
    header "/usr/include/python2.7/pythread.h"
    export *
  }

  explicit module structmember { 
    header "/usr/include/python2.7/structmember.h"
    export *
  }

  explicit module structseq { 
    header "/usr/include/python2.7/structseq.h"
    export *
  }

  explicit module timefuncs { 
    header "/usr/include/python2.7/timefuncs.h"
    export *
  }

  explicit module token { 
    header "/usr/include/python2.7/token.h"
    export *
  }

  explicit module ucnhash { 
    header "/usr/include/python2.7/ucnhash.h"
    export *
  }
  
  explicit module ncurses {
    header "/usr/include/ncurses.h" // note: same as curses.h
    export *

    explicit module dll {
      header "/usr/include/ncurses_dll.h"
      export *
    }

    explicit module unctrl {
      header "/usr/include/unctrl.h"
      export *
    }
  }

  // FIXME: true/false issues might be a compiler bug
  exclude header "/usr/include/python2.7/pymactoolbox.h"
  exclude header "/usr/include/python2.7/asdl.h"
  exclude header "/usr/include/python2.7/ast.h"
  exclude header "/usr/include/python2.7/Python-ast.h"
  exclude header "/usr/include/python2.7/symtable.h"

  // Note: missing #include here
  exclude header "/usr/include/python2.7/pyexpat.h"
  
}

sit.py

import sys
from ctypes import *


# NOTE: fn_name is copied from the output of nm (see INSTRUCTIONS.md)
if sys.platform == "darwin":
    lib_ext = "dylib"
    fn_name = "_TF3sit15mystringdoublerFGVSs20UnsafeMutablePointerVSC7_object_GS0_S1__"
else:
    lib_ext = "so"
    fn_name = "_TF3sit15mystringdoublerFGSpVSC7_object_GSpS0__"


########################################
# Calling the Simple way
########################################
libsit = PyDLL("./libsit.{0}".format(lib_ext))
prototype = PYFUNCTYPE(    
    py_object, # return type
    py_object, # args type, arg type, arg type
)

str_doubler = prototype((fn_name, libsit))

print str_doubler("Hamburger")



########################################
# Calling the Complicated Way…
########################################
def swift_fn(name, ret_type, *arg_types):
    prototype = PYFUNCTYPE(    
        ret_type,
        *arg_types
    )
    return lambda self, *args: prototype((name, self.lib))(*args)

class SwiftLib(object):
    def __init__(self):
        self.lib = PyDLL(self.lib_file)
        

class SwiftInteropTest(SwiftLib):
    lib_file = "./libsit.{0}".format(lib_ext)
    
    double_str = swift_fn(
        fn_name,
        py_object,
        py_object,
    )

lib = SwiftInteropTest()
print lib.double_str("Bang!")

sit.swift

import Python

// Some stuff to make writing Swift code easier on you :)
public typealias PythonObject = UnsafeMutablePointer<Python.PyObject>

extension String {
  func toPyString() -> PythonObject {
    let len = Array(self.utf8).count
    return PyString_FromStringAndSize(self, len)
  }
  static func fromPyString(pyString: PythonObject) -> String? {
    let cstring = PyString_AsString(pyString)
    return String.fromCString(cstring)
  }
}



// Actual function for use in python:
public func mystringdoubler(inPyStr: PythonObject) -> PythonObject {
  let instr = String.fromPyString(inPyStr)!
  return "\(instr) \(instr)".toPyString()
}

How to use Python 3 with Swift?

this also apply for macOS?

Using @_cdecl("mystringdoubler") on public func mystringdoubler(inPyStr: PythonObject) will allow you to export the symbol predictably. https://forums.swift.org/t/communicating-with-dynamically-loaded-swift-library/6769/3

An example with python3 and ctypes (tested under Centos 8)

modules.py

import ctypes

modules = ctypes.CDLL('/usr/lib64/python3.6/lib-dynload/libmodules.so')
modules.add.argtypes = (ctypes.c_int, ctypes.c_int)
modules.strlen.argtypes = (ctypes.c_char_p,)
modules.myname.argtypes = (ctypes.c_char_p,)
modules.myname.restype = ctypes.c_char_p

def add(x,y):
    result = modules.add(ctypes.c_int(x), ctypes.c_int(y))
    return int(result)

def strlen(x):
    y = ctypes.c_char_p(x.encode())
    return modules.strlen(y)

def myname(x):
    y = ctypes.c_char_p(x.encode())
    return modules.myname(y).decode()

modules.swift

import Foundation

@_cdecl("add")
public func add (x: Int, y: Int) -> Int {
	return x+y
}

@_cdecl("strlen")
public func strlen (x: UnsafePointer<CChar>) -> Int{
    let s : String = "\(String(cString: x))"
    return s.count
}

@_cdecl("myname")
public func myname (x: UnsafePointer<CChar>) -> UnsafePointer<CChar>{
    let ret : String = "My name is \(String(cString: x))"
    return UnsafePointer<CChar>(ret)
}

@_cdecl("objc_autoreleaseReturnValue")
public func objc_autoreleaseReturnValue (x: Int) -> Int {
    return 0
}

The library libmodules.so is built with the following commands:

swiftc -emit-library modules.swift
cp mv libmodules.so /usr/lib64/python3.6/lib-dynload

The python module "modules.py" must be moved in the python path:

cp modules.py /usr/local/lib/python3.6/site-packages

And now the module "modules.py" can be used like this:

python3 -c "import modules; print(modules.add(28, 42))"
70

python3 -c "import modules; print(modules.strlen('I compute the length of this string!'))"
36

python3 -c "import modules; print(modules.myname('Nobody'))"
My name is Nobody

In fact, the interesting part of this example is usage of strings parameters between Python and Swift.

In the above example, there is no need to include the Python library from Swift and to build a module.map file

In fact the workaround associated to "objc_autoreleaseReturnValue" can be removed:

modules.swift

import Foundation

@_cdecl("add")
public func add (x: Int, y: Int) -> Int {
	return x+y
}

@_cdecl("strlen")
public func strlen (x: UnsafePointer<CChar>) -> Int{
    let s : String = "\(String(cString: x))"
    return s.count
}

@_cdecl("myname")
public func myname (x: UnsafePointer<CChar>) -> UnsafePointer<CChar>{
    let ret : String = "My name is \(String(cString: x))"
    return UnsafePointer<CChar>(ret)
}

Note: swift and swiftc are now available under Centos 8. To install them:

dnf install -y epel-release
dnf install swift-lang