Proof of concept on building extensions that links against executable (as shared library) - To implement Rubinius rbx on Windows

extension.c

#include <stdio.h>
#include "interface.h"

void define_stuff()
{
  printf("here we define some stuff\n");
}

void Init_extension()
{
  printf("Hello from extension\n");
  rb_hello();
  define_stuff();
}

interface.h

#ifdef __cplusplus
extern "C" {
#endif

#ifdef RBX_WINDOWS
  #ifdef RBX_BUILDING_VM
    #define RUBY_EXTERN extern __declspec(dllexport)
  #else
    #define RUBY_EXTERN extern __declspec(dllimport)
  #endif
#else
  #define RUBY_EXTERN extern
#endif

RUBY_EXTERN void rb_hello();

#ifdef __cplusplus
}
#endif

main.c

#include <stdio.h>
#include "interface.h"

#ifdef RBX_WINDOWS
#include <windows.h>
#endif

RUBY_EXTERN void rb_hello()
{
  printf("Hello from rb_hello()\n");
}

int main(int argc, char* argv[])
{
  int i;

  printf("Hello from main()\n");

  if (argv[1])
  {
#ifdef RBX_WINDOWS
    HMODULE handle;
    FARPROC symbol;
    char *ext = "extension.so";
    char *init = "Init_extension";

    printf("Loading %s\n", ext);

    // Ruby 1.9 behavior (first look into system, then local path)
    handle = LoadLibraryEx((LPSTR)ext, NULL, LOAD_WITH_ALTERED_SEARCH_PATH);

    if (!handle)
    {
      printf("Error loading %s\n", ext);
      return -1;
    }

    printf("Getting address of %s\n", init);
    symbol = GetProcAddress(handle, init);

    printf("Invoking %s\n", init);
    symbol();

    printf("Releasing %s\n", ext);
    FreeLibrary(handle);
#endif
  }

  return 0;
}

Rakefile

require 'rake/clean'

CLEAN.include '*.{a,o,def,exe,so}'

file 'vm.exe' => ['main.c', 'interface.h'] do |t|
  cfiles = t.prerequisites.reject { |f| f !~ /\.c$/ }

  cfiles.each do |f|
    sh "gcc -c #{f} -o #{f.ext('.o')} -DRBX_WINDOWS -DRBX_BUILDING_VM"
  end

  ofiles = cfiles.map { |f| f.ext('.o') }

  sh "gcc -o #{t.name} #{ofiles.join(' ')}"
end

file 'libvm.a' => ['vm.exe'] do |t|
  exe = t.prerequisites.first
  cfiles = Rake::Task[exe].prerequisites.reject { |f| f !~ /\.c$/ }
  ofiles = cfiles.map { |f| f.ext('.o') }

  sh "dlltool --dllname #{exe} --output-lib #{t.name} #{ofiles.join(' ')}"
end

file 'extension.def' do |t|
  File.open(t.name, 'w') do |f|
    f.puts 'EXPORTS'
    f.puts "Init_#{t.name.ext('')}"
  end
end

file 'extension.so' => ['extension.c', 'interface.h', 'extension.def', 'libvm.a'] do |t|
  cfiles = t.prerequisites.reject { |f| f !~ /\.c$/ }
  libs = t.prerequisites.reject { |f| f !~ /\.a$/ }
  libs.map! { |l| l.gsub('lib', '-l').ext('') }

  sh "gcc -shared -o #{t.name} #{cfiles.join(' ')} -L. -Wl,--enable-auto-image-base,--enable-auto-import #{t.name.ext('.def')} #{libs.join(' ')}"
end

task :run => ['vm.exe', 'extension.so'] do
  sh "vm loadext"
end

sample-output.log

C:\Users\Luis\Projects\_sandbox\exe-lib>vm
Hello from main()

C:\Users\Luis\Projects\_sandbox\exe-lib>vm extension
Hello from main()
Loading extension.so
Getting address of Init_extension
Invoking Init_extension
Hello from extension
Hello from rb_hello()
here we define some stuff
Releasing extension.so