(Host) Lua utility to convert Lua source file into module fast include

lua-to-c-include.lua

#! /usr/bin/env lua
--[=[
  The purpose of this utility is to convert one or more Lua source files into an include block
  suitable for inclusion into the Nodule "fast" C module.  Files are treated in one of two
  categories: plain and chunked.  A plain file is treated as a single chunk.  A chunked file
  is identified by having a special SPLIT MODULE comment on the first line and is divided into
  separate chunks using the special SPLIT comments:

  --[[SPLIT MODULE name  Module name.  <the rest of the line is not parsed>

  --[[SPLIT IGNORE]]     <the whole line is ignored>

  --[[SPLIT HERE name]]  <a new chunk "name" is started; the whole line itself is ignored>

  These special comments allow a Lua source file to have a dual purpose: it can be compiled
  as a single source file into a LFS module; it can also be split into separate dynamically
  loadable chunks.  (See the Lua ftpserver module as an example of how this works)

  Each chunk is source-compressed using LuaSrcDiet and this stripped source is then converted
  into a valid compilable C string, and these strings are then assembled into a valid format
  to include into fast.c

]=]

getmetatable('').__mod = function(s, b)
  if not b then return a end
  if type(b) ~= 'table' then b = {b} end
  return s:format(unpack(b))
end

local cmdName, outFile = arg[0], 'output.h'
local LuaSrcDiet = os.getenv('LUASRCDIET') or 'LuaSrcDiet'

local function get_help()
  print ([[
Convert a list of Lua sources into an include file compatible with module fast

Usage: %s [options] parameterlist

Example:
  > %s -o /tmp/funcs.h *.lua

Options
  -h   prints this usage information and exit
  -o   output file.  Defaults to output.h

The parameters are a list of lua source files to be processed

The environment variable LUASRCDIET can but use to specify the LuaSrcDiet command
]] % {cmdName,cmdName})
  os.exit()
end

if not os.execute() then error('%s requires a shell to execute' % cmdName, 0) end

local function basemod(f) -- Extract module name out of full filename
  local name = f:sub(2+#f-(f:reverse():find'[%\\/]' or #f))
  local dot = name:find('.',1,true) or (#name+1)
  return name:sub(1,dot-1)
end

local function encodeChunk(name) -- Pack chunk in compressed Cstring source 
  local tmp = os.tmpname()
  if not os.execute('%s -o %s %s >/dev/null' % {LuaSrcDiet, tmp, name}) then
    os.remove(tmp)
    error('LuaSrcDiet on %s failed' % name, 0)
  end
  -- Convert the compressed but syntactically correct Lua source into a format that can
  -- be included into a C source file as a valid CString of the format "somebit" \<CR> ...
  -- that can be compiled by gcc.  Since the lua source is going to be double quoted,
  -- any " and \ character must be converted into their escaped \" and \\ equivalents.
  -- The line are concatenated using \n and the broken into 100-110 char chunks on a
  -- suitable boundary.   
  local f = io.open(tmp) -- temporary file that contains the compressed Lua source
  local p = {}           -- array used to assemble the input lines in one output line
  local len = 0          -- len of output line corresponding to p[1..#p] 
  local o = {}           -- array used to assemble output lines in the chunk
  local l = f:read()
  local doEscape = true
  while l do
    if doEscape then
      l = l:gsub('%\\','\\\\'):gsub('"','\\"') -- map \ to \\ and " to \"
    end
    doEscape = true  
    if len + #l < 100 then
      -- if the output line is going to <100 chars then just add l 
      p[#p+1], len, l = l, len + #l+2, f:read()
    elseif len + #l < 110 then
      -- if the output line is going to <110 chars then add l and start new output line
      p[#p+1] = l..'\\n'
      o[#o+1], p, len, l = table.concat(p, '\\n'),  {}, 0, f:read()
    else
      -- we need to break l on a suitable boundary
      local i0 = 101 - len  -- skip this bit before looking for a break
      local ltrunc = l:sub(1,i0+10)
     local i = ltrunc:find('[%.%[%]%(%)%+%-%*/%^%%#;= ][%w ]', i0) -- after sep/op
      if i == nil then
        i = ltrunc:find('[^%\\]%\\', i0)  -- before \
        if i == nil then
          i = ltrunc:find('[%w_ ][%w_ ]', i0) -- between two letters / spaces
          if i == nil then
            i = l:find('[^%\\][%.]', i) -- after any non \ on whole line
            if i == nil then
            i = #l
      end end end end
      p[#p+1] = l:sub(1,i)
      o[#o+1], p, len = table.concat(p, '\\n'),  {}, 0
      doEscape, l = false, l:sub(i+1) -- new l (might be empty) already escaped
    end
  end
  f:close(); os.remove(tmp)
  o[1], o[#o+1] = '  "'..o[1], table.concat(p, '\\n')..'\\n"'
  assert(#o>1)
  return table.concat(o, '" \\\n  "')
end

local cstring = {}
local function processFile(name)
  local f,e = io.open(name)
  if not f then error( e, 0) end
  local line = f:read()
  local modName = line:match('%-%-%[%[%s*SPLIT%s+MODULE%s+([%w_]+)%s*%]%]')
print(line)
print('module='..(modName or "???"))
  if not modName then
    modName=basemod(name)
    f:close()
    cstring[modName]=encodeChunk(name)
  else
    local chunkName, chunk = modName, {}
    line = f:read()
    while (line) do
      local split, param = line:match('%-%-%[%[%s*SPLIT%s+([A-Z]+)%s*([%w_%-]*)%s*%]%]')
      if split == 'HERE' then --close the previous chunk and start the next 'param'
        if #chunk > 0 then
          local tmp = os.tmpname()
          local of = io.open(tmp, 'w')
          for _,l in ipairs(chunk) do
            of:write(l,'\n')
          end
          of:close()
          cstring[chunkName]= encodeChunk(tmp)
          os.remove(tmp)
        end
        chunkName,chunk = param, {}
      elseif split ~= 'IGNORE' then
        chunk[#chunk+1] = line
      end
    line = f:read()
    end
  end
end

-- Process command line.  On completion, the table cstring is
-- of the form {chunk_name = Cstring_for_chunk, ...}
local i, argn = 1, #arg
if argn == 0 then get_help() end
while i <= #arg do
  if arg[i] == '-h' then
    get_help()
  elseif arg[i] == '-o' then
    i, outFile = i + 1, (arg[i+1] or get_help())
  else
    processFile(arg[i])
  end
  i = i + 1
end

--[[DEBUG print ('writing to '..outFile)]]
--[[DEBUG for k,s in pairs(cstring) do print('cstring',k,#s) end]]

-- To make the chunk lookup efficient, the algo groups the chunks by chunk
-- length and does a select(len) + case chain.  For each length a strcmp()
-- is done to match the chunk names for that length. 
local of,msg = io.open(outFile, 'w')
if not of then error(msg, 0) end

local selectcases = {}
for name in pairs(cstring) do
  local l = #name
  local case = selectcases[l] or {}
  case[#case+1] = name
  selectcases[l] = case
end
--[[DEBUG for k,l in pairs(selectcases) do print(k,table.concat(l,',')) end]]

local lens = {}; for k in pairs(selectcases) do lens[#lens+1] = k end
table.sort(lens)

for _,l in ipairs(lens) do
  local case = selectcases[l]
  table.sort(case)
--[[DEBUG print(l..":"..table.concat(case,','))]]
  of:write('    case %u:\n' % l);
  for j = 1, #case do
    local name = case[j]
    of:write('      if (!strcmp(name, "%s")) { src = "local FAST=1;" \\\n' % name)
    of:write(cstring[name], ';%s}\n' % (j==#case and '' or ' break;'))
  end
  of:write('      break;\n')
end

of:close()