Skip to content

Instantly share code, notes, and snippets.

@juntalis
Created February 25, 2013 05:20
Show Gist options
  • Select an option

  • Save juntalis/5027920 to your computer and use it in GitHub Desktop.

Select an option

Save juntalis/5027920 to your computer and use it in GitHub Desktop.
Limited expansion C/C++ preprocessor using the MSVC compiler. I wrote this, more or less, to avoid having to scour system headers for constants when writing bindings, etc. The script will expand all macros and preprocessor directives in the current file while leaving the included headers as their original #include statements.
import sys, os, re, string
import subprocess as _subp, tempfile as _tmp, time as _time, shutil as _shutil
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
# CPU Identification and ctypes general stuff
## Figure out platform architecture
from ctypes import c_void_p, c_ulonglong, c_long, c_uint, byref, c_ulong, WinDLL, cast, \
c_longlong, Structure, POINTER as _POINTER, c_int, c_wchar_p, sizeof as _szof
## Ctypes/Platform arch constants and globals
isx64 = _szof(c_void_p) == _szof(c_ulonglong)
NULL = 0
del _szof
### Override default POINTER
def POINTER(o):
""" Convert None to a real NULL pointer to work around bugs
in how ctypes handles None on 64-bit platforms """
ptr = _POINTER(o)
if not isinstance(ptr.from_param, classmethod):
def from_param(cls, x):
if x is None:
return cls()
else:
return x
ptr.from_param = classmethod(from_param)
return ptr
# General Helpers (Mostly dealing with types)
## Lambdas
instanceofany = lambda o, tl: any(map(lambda t: isinstance(o, t), tl))
listlike = lambda o: instanceofany(o, [ list, tuple, set, frozenset ])
## Functions
class Enum(object):
pass
def enum(*sequential, **named):
__name__ = named.get('__name__', 'Enum')
enums = dict(zip(sequential, range(len(sequential))), **named)
return type(__name__, (Enum,), enums)
# String Handling
## Helpful constants
ws = string.whitespace
## Regular Expressions
reblank = re.compile(r"^\s*$")
## Helpful lambdas
### Lambdas for validation
not_empty = lambda s: s is not None and len(s) > 0
no_empties = lambda l: filter(not_empty, l)
not_blank = lambda s: not reblank.match(s)
no_blanks = lambda l: filter(not_blank, l)
# The next one may seem tedious, since not_blank would also cover not_empty, but the way I see it, if a string fails
# not_empty, it won't even bother processing not_blank. Since it's faster to check an string's length than it is to
# test the string against a regular expression, this will allow an instant return for any string with the length of
# zero, and should give a (admittedly pointlessly miniscule) boost in script execution speed over testing all strings
# with not_blank.
not_empty_or_blank = lambda s: not_empty(s) and not_blank(s)
no_empties_or_blanks = lambda l: filter(not_empty_or_blank, l)
startswith_any = lambda s, l: any(map(lambda i: s.startswith(i), l))
### Lambdas for cleaning and normalizing (For the sake of map, etc.)
trim = lambda s: s.strip(ws)
lower = lambda s: s.lower()
upper = lambda s: s.upper()
nonl = lambda s: s.strip('\r\n')
quote = lambda s: '"%s"' % s.strip('"') # Quote a string. (Possibly for a filepath)
maybe_nl = lambda s: s + '\n' if not_blank(s) and s[-1] not in '\r\n' else s # Add a newline to a string if necessary.
### Lambdas for processing lists of strings
all_lower = lambda l: map(lower, l)
all_upper = lambda l: map(upper, l)
all_nonl = lambda l: map(nonl, l)
all_trim = lambda l: map(trim, l)
all_nonl_filter = lambda l: no_empties(all_nonl(l))
all_trim_filter = lambda l: no_empties(all_trim(l))
join_lines_nl = lambda l: '\n'.join(all_nonl(l)) # Remove all newlines, then join the list with '\n'
## Lambdas for splitting lines and processing above.
split_nonl = lambda ls: all_nonl(ls.splitlines())
split_trim = lambda ls: all_trim(ls.splitlines())
split_nonl_filter = lambda ls: all_nonl_filter(ls.splitlines())
split_trim_filter = lambda ls: all_trim_filter(ls.splitlines())
## Helpful functions
def norm_str(obj):
""" If obj is a string, immediately return it. If, on the other hand, obj is a collection of strings,
run it through join_lines_nl and return it. If obj is anything else, try to convert it to a string
and return it. """
if isinstance(obj, basestring):
return obj
elif listlike(obj):
return join_lines_nl(list(obj))
else:
return str(obj)
# IO/Filesystem Handling
## To shorten up calls to os.path functions..
dname = os.path.dirname
bname = os.path.basename
absp = os.path.abspath
isdir = os.path.isdir
isfile = os.path.isfile
pexists = os.path.exists
splitext = os.path.splitext
pjoin = os.path.join
## Lambdas
pdir = lambda fp: absp(dname(fp)) # Get the absolute path to the parent folder of filepath.
## Helpful functions for path manipulation
def abspj(*parts):
""" Absolute path of value returned by os.path.join """
return absp(pjoin(*parts))
## Helpful functions for reading/writing to and from files.
def readtext(p):
""" Read lines from a file """
fn = open(p, 'rt')
txt = fn.read()
fn.close()
return txt
def writetext(fpath, lines):
""" Write text/lines to a file """
fn = open(fpath, 'wt')
fn.write(norm_str(lines))
fn.close()
## Safe removal functions.
### Constants
FO_DELETE = 3
FOF_NOCONFIRMATION = 16 # Don't prompt the user.
FOF_ALLOWUNDO = 64
# Types needed for recycle
class SHFILEOPSTRUCT(Structure):
_fields_ = [("hwnd", c_void_p),
("wFunc", c_uint),
("pFrom", c_wchar_p),
("pTo", c_wchar_p),
("fFlags", c_ulong),
("fAnyOperationsAborted", c_long),
("hNameMappings", c_void_p),
("lpszProgressTitle", c_wchar_p)]
@property
def From(self): return self.pFrom if not_empty(self.pFrom) else None
@From.setter
def From(self, value): self.pFrom = cast(NULL, c_wchar_p) if value is None else u"%s\x00" % value
@property
def To(self): return self.pTo if not_empty(self.pTo) else None
@To.setter
def To(self, value): self.pTo = cast(NULL, c_wchar_p) if value is None else u"%s\x00" % value
@property
def AnyOperationsAborted(self): return False if self.fAnyOperationsAborted == 0L else True
@AnyOperationsAborted.setter
def AnyOperationsAborted(self, value): self.fAnyOperationsAborted = 1L if value else 0L
LPSHFILEOPSTRUCT = POINTER(SHFILEOPSTRUCT)
# Prototypes
_shell32 = WinDLL('shell32.dll')
SHFileOperation = _shell32.SHFileOperationW
SHFileOperation.restype = c_int
SHFileOperation.argtypes = [ LPSHFILEOPSTRUCT ]
def recycle(fpath):
""" Delete to recycling bin. """
sho = SHFILEOPSTRUCT()
sho.hwnd = 0
sho.wFunc = FO_DELETE
sho.fFlags = FOF_ALLOWUNDO|FOF_NOCONFIRMATION
sho.From = fpath
sho.To = None
sho.AnyOperationsAborted = False
sho.hNameMappings = None
sho.lpszProgressTitle = None
res = SHFileOperation(byref(sho))
return not res and not sho.fAnyOperationsAborted
def unlink(fpath, use_recycling_bin=True):
""" Stand-in for os.unlink and os.rmdir. This function checks if fpath exists. If not, it will
immediately return successfully. If it does exist and use_recycling_bin is true,
it will delete the path to the recycling bin. If use_recycling_bin is false, it will check if the path pointers
to a folder or a file, and then use the appropriate function to remove it. """
if not pexists(fpath):
return True
elif use_recycling_bin:
return recycle(fpath)
elif isfile(fpath):
return os.unlink(fpath)
elif isdir(fpath):
return _shutil.rmtree(fpath, True)
# Process/Environment Handling
## Constants forwarded from subprocess.
PIPE = _subp.PIPE
STDOUT = _subp.STDOUT
VOID = object() # os.devnull
# Forwarded from os.path
expvars = os.path.expandvars
## Lambdas
default_to_pipe = lambda kw, v: PIPE if not v in kw else kw.get(v) # Get the value of dict[key], or default to PIPE.
is_pipe = lambda v: v == PIPE
contains_pipe = lambda v: any(map(is_pipe, v)) # Check if an arglist contains PIPE or not
## Helper functions
def which(cmd, paths = None):
"""Return full path of command (using the pathsep-seperated string/list of folders in the paths arg,
or the PATH environment variable if None is specified. When searching, we first look for cmd as specified,
and if that fails, we use the PATHEXT env variable to check. Returns None if
the command could not be found. """
# If no path specified, use the cwd and our PATH env var.
if paths is None:
paths = os.getcwd()
if 'PATH' in os.environ:
paths += os.pathsep + os.environ['PATH']
elif isinstance(paths, dict):
paths = paths.get('PATH')
# Check path for string and split if necessary.
if isinstance(paths, basestring):
paths = paths.split(os.pathsep)
# Finally, expand any env vars in the list, and remove any blanks.
paths = map(expvars, paths)
paths = all_trim_filter(paths)
# Pre-determine PATHEXTs value.
if 'PATHEXT' in os.environ:
pathexts = all_lower(os.environ['PATHEXT'].split(os.pathsep))
else:
pathexts = [ '.exe', '.com', '.cmd', '.bat', '.vbs', '.js', '.py' ]
pathexts = all_trim_filter(pathexts)
for folder in paths:
fpath = abspj(folder, cmd)
if isfile(fpath):
return fpath
# Next, try with the file extensions found earlier
for pathext in pathexts:
fpathext = fpath + pathext
if isfile(fpathext):
return fpathext
return None
def outputof(cmd, args=None, check_return=True, **kwargs):
""" Execute a process, then return the output as a string. (Or tuple, in the case where both stdout and stderr
specify PIPE. If neither stdout or stderr are specified, both default to PIPE. check_return specifies whether or
not to throw an error when a non-zero return code occurs.
For stdout/stderr values:
* For output to go to the foreground (to be visible in the script's console), use None.
* For output to be hidden (go to os.devnull), specify VOID.
* At least one of the two args must be PIPE. If both contain a value of VOID or None, an error will be thrown.
"""
# First, we resolve what the user wants to use for stdout and stderr
stdout = default_to_pipe(kwargs, 'stdout')
stderr = default_to_pipe(kwargs, 'stderr')
# In the case of None/VOID being specified for both stdout and stderr.
if not contains_pipe([stdout, stderr]):
raise Exception('Must specify at least one pipe to listen on.')
# Next, set kwarg's values to PIPE for our own purposes.
kwargs['stdout'] = PIPE
kwargs['stderr'] = PIPE
# Args defaults to an empty list
if args is None:
args = [ ]
elif isinstance(args, basestring):
args = [ args ]
# Finally, execute the process.
subp = _subp.Popen([ cmd ] + args, **kwargs)
pout, perr = subp.communicate()
# Check for non-zero return code. (If necessary)
if check_return and subp.returncode != 0:
errmsg = 'STDOUT: %s\n\nSTDERR: %s' % (pout, perr)
sys.stderr.write(errmsg)
raise _subp.CalledProcessError(subp.returncode, cmd, errmsg)
# Figure out what to return and return it.
if not is_pipe(stderr): return pout
elif not is_pipe(stdout): return perr
return pout, perr
def envcmd(batfile, args=None, **kwargs):
""" Execute a .bat/.cmd file, and return a dict containing the resulting environment variables post-run. Kwargs
can contain some of the subprocess.Popen args, such as cwd, env, etc. IO redirection, however,
is specified in the function itself, and therefore any value written to stdin, stdout will be overwritten. The
function also mandates that the return code be respected. """
kwargs.update({
'stdin': None,
'stdout': PIPE,
'stderr': None
})
cmd = os.getenv('COMSPEC', default='cmd.exe')
tmp = os.path.join(_tmp.gettempdir(), 'cmdenv%x' % _time.time() + '.cmd')
cmdline = [
'@if not exist "%s" exit 1' % batfile,
'@(call %s)>nul 2>nul' % ' '.join([ quote(batfile) ] + list(args)),
'@set'
]
writetext(tmp, cmdline)
args = [ '/C', tmp ]
lines = split_trim(outputof(cmd, args, True, **kwargs))
unlink(tmp)
results = {}
for line in lines:
sep = line.find('=')
key, val = line[:sep].upper(), line[sep+1:]
if key in os.environ and val.lower() == os.environ[key].lower(): continue
results[key] = val
return results
class Executable(object):
""" Simple helper class that implements a function-like object for launching
subprocesses and getting the output. """
__slots__ = ('__doc__', 'cmd', '_args', '_kwargs')
def __init__(self, cmd, args=None, **kwargs):
self.cmd = cmd
self._kwargs = kwargs
self._args = self._norm_args(args)
def _norm_args(self, args):
""" Normalize args. """
if listlike(args):
return list(args)
elif isinstance(args, basestring):
return args.split(' ')
elif args is None:
return []
else:
raise TypeError(type(args))
def __call__(self, args, **kwargs):
args = self._norm_args(args)
args.extend(self._args)
kwargs.update(self._kwargs.copy())
if 'environ' in kwargs:
kwargs['env'] = kwargs['environ']
del kwargs['environ']
return outputof(self.cmd, args, **kwargs)
@staticmethod
def which(cmd, paths=None):
return Executable(which(cmd, paths))
# Registry handling
## Try to use the built-in _winreg module. If, for some reason,
## the user doesn't have the builtin module, we implement our
## own functions using ctypes. (Though the only condition I can
## think of for this to occur is using a cygwin version of
## Python)
# Types
try:
from _winreg import *
except ImportError:
from ctypes import c_char_p, create_unicode_buffer, create_string_buffer, GetLastError
# Our DLL
_advapi32 = WinDLL('advapi32.dll')
# Types
ULONG_PTR = c_ulonglong if isx64 else c_ulong
LONG_PTR = c_longlong if isx64 else c_long
PLONG = POINTER(c_long)
LPDWORD = POINTER(c_ulong)
LPBYTE = c_char_p # It's actually unsigned char*, but for our purposes, this should work.
HKEY = c_void_p
ACCESS_MASK = c_ulong
REGSAM = c_ulong
# Constants
HKEY_LOCAL_MACHINE = HKEY(0x80000002)
KEY_WOW64_32KEY = HKEY_LOCAL_MACHINE
KEY_ALL_ACCESS = 0xF003F
KEY_QUERY_VALUE = 0x0001
KEY_READ = 0x20019
MAX_ENV = 16383
MAX_PATH = 260
# Function prototypes
RegOpenKey = _advapi32.RegOpenKeyW
RegOpenKey.restype = c_long
RegOpenKey.argtypes = [ HKEY, c_wchar_p, POINTER(HKEY) ]
RegOpenKeyEx = _advapi32.RegOpenKeyExW
RegOpenKeyEx.restype = c_long
RegOpenKeyEx.argtypes = [ HKEY, c_wchar_p, c_ulong, REGSAM, POINTER(HKEY) ]
RegQueryValue = _advapi32.RegQueryValueW
RegQueryValue.restype = c_long
RegQueryValue.argtypes = [ HKEY, c_wchar_p, c_wchar_p, PLONG ]
RegQueryValueEx = _advapi32.RegQueryValueExA
RegQueryValueEx.restype = c_long
RegQueryValueEx.argtypes = [ HKEY, c_char_p, LPDWORD, LPDWORD, LPBYTE, LPDWORD ]
RegCloseKey = _advapi32.RegCloseKey
RegCloseKey.restype = c_long
RegCloseKey.argtypes = [ HKEY ]
# Function wrappers
def OpenKey(key, subkey):
""" Open a handle to a registry key. """
result = HKEY(0)
if int(RegOpenKey(key, subkey, byref(result))) != 0:
return None
return result
def OpenKeyEx(key, subkey, opts, flags):
""" OpenKey with more options and flags. """
result = HKEY(0)
if int(RegOpenKeyEx(key, subkey, opts, flags, byref(result))) != 0:
return None
return result
def QueryValue(key, value):
""" Read the value of a registry key. """
dwValue = c_long(0)
RegQueryValue(key, value, cast(NULL, c_wchar_p), byref(dwValue))
buf = create_unicode_buffer(dwValue.value+1)
if int(RegQueryValue(key, value, buf, byref(dwValue))) != 0:
return None
return buf.value
def QueryValueEx(key, value):
""" QueryValue with type information. """
dwValue = c_ulong(0)
dwType = c_ulong(0)
RegQueryValueEx(key, value, cast(NULL, LPDWORD), cast(NULL, LPDWORD), cast(NULL, LPBYTE), byref(dwValue))
buf = create_string_buffer(dwValue.value+1)
if RegQueryValueEx(key, value, cast(NULL, LPDWORD), byref(dwType), buf, byref(dwValue)) and GetLastError() != 0:
return None, -1
return buf.value, int(dwType.value)
def CloseKey(key):
""" Close a previously opened registry key. """
return int(RegCloseKey(key)) == 0
## Helper functions
def read_reg_value(key, subkey=None, root=HKEY_LOCAL_MACHINE):
""" Registry query helper. """
try:
if subkey is None:
result = QueryValue(root, key)
else:
hk = None
try:
hk = OpenKeyEx(root, key, 0, KEY_QUERY_VALUE | KEY_READ)
if hk is None: return None
result = QueryValueEx(hk, subkey)[0]
except:
if hk:
CloseKey(hk)
raise Exception()
except:
result = None
return result
# MSVC Resolution/Execution, etc
## Visual C/C++
if isx64:
MSVCROOTKEY = 'SOFTWARE\\Wow6432Node\\Microsoft\\VisualStudio'
MSVCEXPROOTKEY = 'SOFTWARE\\Wow6432Node\\Microsoft\\VCExpress'
else:
MSVCROOTKEY = 'SOFTWARE\\Microsoft\\VisualStudio'
MSVCEXPROOTKEY = 'SOFTWARE\\Microsoft\\VCExpress'
MSVC_BASE_LABEL = 'Microsoft Visual C++'
MSVC_VALUEKEY = 'ProductDir'
MSVC_ENVCMD = 'vcvarsall.bat'
MSVC_KEYS = [
(10, '10', u'%s\\10.0\\Setup\\VC' % MSVCROOTKEY),
(10, '10 Express', u'%s\\10.0\\Setup\\VC' % MSVCEXPROOTKEY),
(11, '11', u'%s\\11.0\\Setup\\VC' % MSVCROOTKEY),
(9, '9', u'%s\\9.0\\Setup\\VC' % MSVCROOTKEY),
(9, '9 Express', u'%s\\9.0\\Setup\\VC' % MSVCEXPROOTKEY),
(8, '8', u'%s\\8.0\\Setup\\VC' % MSVCROOTKEY),
(7, '8 Express', u'%s\\8.0\\Setup\\VC' % MSVCEXPROOTKEY),
(7, '7', u'%s\\7.1\\Setup\\VC' % MSVCROOTKEY),
(6, '6', u'%s\\6.0\\Setup\\Microsoft Visual C++' % MSVCROOTKEY),
]
def find_msvc_root(vers=None):
""" Find the most recently released (or whatever is specified version of Visual Studio,
and return the root VC folder. (Though it favors VS2010 over VS2012 for the time being. """
result = None
keys = MSVC_KEYS
if vers is not None:
keys = filter(lambda k: k[0] == vers, keys)
for vcvers, label, key in keys:
#print 'Checking for installation of %s %s..' % (MSVC_BASE_LABEL, label)
check = read_reg_value(key, MSVC_VALUEKEY)
if check is None or len(check) == 0 or not isdir(check):
continue
# print 'Found installation. Using VC%d configuration.' % vcvers
result = vcvers, check
break
return result
### Platform SDK
SDK_ROOT_KEY = 'SOFTWARE\\Microsoft\\MicrosoftSDK\\InstalledSDKs'
MSSDK_BASE_LABEL = 'Windows SDK'
MSSDK_VALUEKEY = 'InstallationFolder'
MSSDK_ENVCMD = 'SetEnv.cmd'
MSSDK_KEYS = [
([10], 71, '7.1', u'SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v7.1'),
([10], 701, '7.0 (VS2010)', u'SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v7.0A'),
([10], 70, '7.0', u'SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v7.0'),
([11], 801, '8.0 (VS2012)', u'SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v8.0A'),
([9], 61, '6.1', u'SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v6.1'),
([9], 601, '6.0 (VS2008)', u'SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v6.0A'),
([9], 60, '6.0', u'SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v6.0'),
# Don't actually know what these support, but whatever.
([10, 9, 8, 7, 6], 502, '2003 SP2', u'%s\\D2FF9F89-8AA2-4373-8A31-C838BF4DBBE1' % SDK_ROOT_KEY),
([10, 9, 8, 7, 6], 501, '2003 SP1', u'%s\\8F9E5EF3-A9A5-491B-A889-C58EFFECE8B3' % SDK_ROOT_KEY),
]
## Path/Environment resolution functions
def find_sdk_root(vers=None, vcvers=None):
""" Find the most recently released Windows SDK (or whatever is specified), and return the root folder.
TODO: Add the Windows 8.0 SDK """
result = None, 0
keys = MSSDK_KEYS
if vers is not None:
keys = filter(lambda k: k[1] == vers, keys)
if vcvers is not None:
keys = filter(lambda k: vcvers in k[0], keys)
for supports, sdkvers, label, key in keys:
#print 'Checking for installation of %s %s..' % (MSSDK_BASE_LABEL, label)
check = read_reg_value(key, MSSDK_VALUEKEY)
if check is None or len(check) == 0 or not isdir(check):
continue
#print 'Found installation. Using VC%d configuration.' % sdkvers
result = sdkvers, check
break
return result
def find_msvc_toolset(vcvers=None, sdkvers=None):
""" Find the platform SDK and VC++ version specified. (Or most recent, if none is specified) It sort of tries to
match up the proper Platform SDK with the proper VC++ version. I can't remember how well it does that, however,
since I originally wrote these functions for another utility script. """
vcroot, sdkroot = None, None
if vcvers is None:
vcvers, vcroot = find_msvc_root()
else:
vcvers, vcroot = find_msvc_root(vcvers)
if sdkvers is None and vcroot is None:
sdkvers, sdkroot = find_sdk_root()
elif sdkvers is not None:
sdkvers, sdkroot = find_sdk_root(sdkvers)
if sdkroot is None and vcroot is None:
raise Exception('Could not locate a toolset to use!')
buildenv = {}
if vcroot is not None:
buildenv = envcmd(pjoin(vcroot, MSVC_ENVCMD), [ 'x86' ])
if sdkroot is not None:
assert(isinstance(sdkroot, basestring))
sdkenv = envcmd(pjoin(sdkroot, 'Bin', MSSDK_ENVCMD), [ '/Release', '/x86' ])
for k in sdkenv.keys():
ku = k.upper()
if ku in [ bk.upper() for bk in buildenv.keys() ]:
if ku in [ 'PATH', 'INCLUDE', 'LIB', 'LIBPATH' ]:
newv = all_trim_filter(sdkenv[k].split(';'))
oldv = all_trim_filter([ v.lower() for v in buildenv[k].split(';') ])
for v in newv:
if v.lower() not in oldv:
buildenv[k] = (v + ';' + buildenv[k])
else:
buildenv[k] = sdkenv[k]
else:
buildenv[k] = sdkenv[k]
else:
for k in buildenv.keys():
if k.upper() in [ 'PATH', 'INCLUDE', 'LIB', 'LIBPATH' ]:
vals = all_trim_filter(buildenv[k].split(';'))
if k in os.environ:
result_lower = all_trim_filter([v.lower() for v in os.environ[k].split(';')])
else:
result_lower = []
result = []
for val in vals:
lval = val.lower()
if lval not in result_lower:
result.append(val)
result_lower.append(lval)
buildenv[k] = ';'.join(result)
return buildenv
# Script-specific Stuff
## Constants
landmarkstr = "#pragma start_landmark(\\1\\2)\n#include \\1\\2\n#pragma end_landmark(\\1\\2)"
## Regular expressions
reinc = re.compile(r'^#\s*include\s+(["<])([^>"]+[>"])\s*(?://.+|/\*.+\*/)?$', re.MULTILINE)
refix = re.compile(r"^#pragma start_landmark\(([^)]+)\)$.*?^#pragma end_landmark\(\1\)$", re.DOTALL | re.MULTILINE)
## Enums
ArgType = enum('SRC', 'FLAG', __name__='Argument')
CompilerType = enum('MSVC', 'GCC', __name__='Compiler')
### Ignored command line args.
ignored_flags = {
'analyze', 'nologo', 'c', 'E', 'P',
'Zi', 'ZI', 'Z7',
'bigobj', 'Zs', 'Zl', 'hotpatch',
'Fa', 'FA', 'Fd', 'Fm', 'Fe', 'Fo', 'Fp', 'FR', 'doc', 'Fi', 'Fr', 'Fo', 'Fe'
}
## Helper classes
class PreviousLine(object):
""" I realize I could've done this with type('PreviousLine', (,), { ... }), but my IDE doesn't seem to like giving
intellisense for types created that way. """
_recomment = re.compile(r"^\s*(?://|/\*)")
def __init__(self, blank=False, comment=False):
self.blank = blank
self.comment = comment
@staticmethod
def check(line): return PreviousLine._recomment.match(line) is not None
class CleanOutput(object):
""" A wrapper around a file-like object. (file or StringIO) The aim is to provide the same interface, but to
add in functionality for filtering and cleaning the output during the write process. (By limiting the number
of consecutive blank lines, in addition to a few other trivial things) """
_output = None
_name = None
_previous = None
def __init__(self, output=None):
""" Output can be a string containing a filepath, a file-like object, or left as None. If left as None,
the class with create a new StringIO object to use as its buffer. """
if isinstance(output, basestring):
self._name = output
output = open(output, 'wt+')
elif output is None:
output = StringIO()
elif not hasattr(output, 'write') or not hasattr(output, 'close'):
raise TypeError(type(output))
if not hasattr(output, 'name'):
self._name = None
self._output = output
self._previous = PreviousLine()
def getvalue(self):
""" Get the value of the current buffer. (Implemented for file objects, as well) """
if not hasattr(self._output, 'getvalue'):
pos = self.tell()
self._output.seek(0)
val = self.read()
self._output.seek(pos)
else:
val = self._output.getvalue()
return val
def writelines(self, iterable):
for line in iterable:
self.writeline(line)
def writeline(self, line):
line = nonl(line)
if not_empty_or_blank(line):
self._previous.blank = False
self._previous.comment = PreviousLine.check(line)
self._output.write(line + '\n')
else:
if self._previous.blank or self._previous.comment: return
self._output.write('\n')
self._previous.blank = True
def seek(self, pos):
current = pos - 1
c = None
# seek to the last newline
while current > 0 and c != '\n' and c != '':
self._output.seek(current)
c = self.read(1)
current -= 1
# And then do it again, to check to see if the previous line was blank.
if c == '\n':
c = ' '
while current > 0 and c != '\n' and c != '':
self._output.seek(current)
c = self.read(1)
if c not in ' \t\n\r':
self._previous_empty = True
break
current -= 1
self._output.seek(pos)
def write(self, s):
self.writelines(s.splitlines())
def isatty(self): return self._output.isatty()
def tell(self): return self._output.tell()
def read(self, n = -1): return self._output.read(n)
def truncate(self): return self._output.truncate()
def flush(self): self._output.flush()
def close(self): self._output.close()
def readlines(self): return self._output.readlines()
def next(self): return self._output.next()
def readline(self): return self._output.readline()
def __iter__(self): return self._output.__iter__()
def __enter__(self): return self
def __exit__(self, exc_type, exc_val, exc_tb): self.close()
@property
def closed(self): return self._output.closed
@property
def name(self): return self._name
class CPP(object):
""" Our C/C++ limited-expansion preprocessor
Note: Might be some bugs if you try to use this with GCC at the moment. It was an after-thought that I hacked in
kind of poorly. I need to go back through to fix some of the methods dealing with sorting flags,
as well as add a member to the class for tracking the compiler family. """
_compiler = None
_env = None
_flags = [ ]
def __init__(self, compiler_path=None, env=None, compiler_family=CompilerType.MSVC):
if compiler_path is None:
if compiler_family == CompilerType.MSVC:
if env is None:
env = find_msvc_toolset()
compiler_path = which('cl.exe', env)
elif compiler_family == CompilerType.GCC:
compiler_path = which('gcc.exe')
else:
raise ValueError('Unknown compiler_family. Must be CompilerType.MSVC or CompilerType.GCC')
if compiler_path is None:
raise Exception('Could not locate compiler.')
if env is None:
env = os.environ.copy()
self._env = env
self._compiler = Executable(compiler_path)
def _process_rsp(self, rsp):
""" Process a response file, passing the args onto self._process_args. """
text = readtext(rsp)
if '\n' in text:
return self._process_args(split_trim_filter(text))
else:
return self._process_args(all_trim_filter(text.split(' ')))
def _process_flag(self, args, idx):
""" Process args looking for stuff like -D NDEBUG=1 or -I include """
arg = args[idx]
arglen = len(arg)
assert(arglen > 1)
if arg[1] in 'IDU':
if arglen == 2:
assert(len(args) > idx + 1)
arg += args[idx+1]
return True, arg
elif arg[1:3] == 'AI':
if arglen == 3:
assert(len(args) > idx + 1)
arg += args[idx+1]
return True, arg
elif startswith_any(arg[1:], ignored_flags):
return False, None
return False, arg
def _process_arg(self, arg):
""" Report the type of arg. (ArgType.SRC or ArgType.FLAG) """
assert(not_empty(arg))
if arg[0] in '-/':
return ArgType.FLAG
else:
return ArgType.SRC
def _process_args(self, args):
""" Simple method that splits a list of command-line arguments into flags,
script args, and input files. Any response files found will also be processed. Returns a tuple of:
(flags, script_flags, sources) """
flags, srcs = [], []
skipnext = False
args = all_trim_filter(args)
for i, arg in enumerate(args):
if skipnext:
skipnext = False
continue
if arg[0] == '@':
rsp_flags, rsp_srcs = self._process_rsp(arg[1:])
flags.extend(rsp_flags)
srcs.extend(rsp_srcs)
else:
argtype = self._process_arg(arg)
# noinspection PyUnresolvedReferences
if argtype == ArgType.FLAG:
if arg[1:] == 'link': break
skipnext, arg = self._process_flag(args, i)
if arg is not None:
flags.append(arg[1:])
else:
srcs.append(arg)
return flags, srcs
def _remove_flag(self, arg):
""" Search the currently stored flags for a particular flag and remove it if found. """
assert(not_empty(arg))
if arg[0] in '-/': arg = arg[1:]
if arg in self._flags:
self._flags.remove(arg)
def _remove_flags(self, args):
for arg in args:
self._remove_flag(arg)
def _finalize_flags(self, srcdir, use_stored_flags, flags):
""" Add compiler-specific preprocessor flags, prefix flags with -, and add the source file's parent folder
"""
flags = list(flags)
if use_stored_flags:
flags.extend(self._flags)
# Add the preprocessor flags.
if bname(self._compiler.cmd).lower() == 'cl.exe':
flags.insert(0, 'nologo')
flags.extend([ 'EP', 'showIncludes', 'C' ])
# TODO: Add processing of include files.
else:
flags.extend(['E'])
# Add - prefix to flags
flags = map(lambda s: '-' + s, flags)
# Lastly, add a -I <srcdir> to the beginning of flags. We need to do this,
# since the file we'll actually be preprocessing will be in the temp folder.
flags.insert(0, srcdir)
flags.insert(0, '-I')
return flags
def _preprocess(self, src, output=None, use_stored_flags=True, *flags):
# Resolve flag-type args we'll be using.
srcdir = pdir(src)
flags = self._finalize_flags(srcdir, use_stored_flags, flags)
# Read in the source file and add pragmas to "landmark" all include statements.
code = readtext(src)
landmarked = reinc.sub(landmarkstr, code)
# Generate a temporary file to preprocess our landmarked code.
base, ext = tuple(splitext(bname(src)))
# Unfortunately, can't use temporary folder for this.
tempsrc = pjoin(srcdir, 'cpp_%s_%x' % (base, _time.time()) + ext)
writetext(tempsrc, landmarked)
# Finally, use our compiler to preprocess the landmarked code./
assert(isinstance(self._compiler, Executable))
cppenv = os.environ.copy()
cppenv.update(self._env)
preprocessed = self._compiler(flags + [ tempsrc ], env=cppenv, stdout=PIPE, stderr=VOID)
unlink(tempsrc)
# And then clean out the included headers while re-inserting the #include statements.
preprocessed = '\n'.join(split_nonl(preprocessed)) # Unfortunately necessary.
fixed = refix.sub(r"#include \1", preprocessed)
output = CleanOutput(output)
output.write(fixed)
return output
def extract_sources(self, args):
""" Given a list of command line arguments,
update the class's stored flags and return a list
of all source files found. """
flags, srcs = self._process_args(args)
self._flags.extend(flags)
return srcs
def preprocess(self, src, output=None, use_stored_flags=True, *flags):
""" Preprocess file specified by src. If you want to use any extra
command-line args just for this file, you can do so with the flags args. If you
don't want to use any of the flags stored up until now, set use_stored_flags to
False. Lastly, output can be a string or file-like object and will have the resulting
code written to it. Returns a CleanOutput instance wrapping your output. """
return self._preprocess(src, output, use_stored_flags, *flags)
def __call__(self, src, output=None, use_stored_flags=True, *flags):
""" Pass through to preprocess. """
return self._preprocess(src, output, use_stored_flags, *flags)
@property
def flags(self):
return self._flags
@flags.setter
def flags(self, value):
if listlike(value):
self._flags = self._process_args(list(value))[0]
elif isinstance(value, basestring):
self._flags = [ value ]
else:
raise TypeError(type(value))
@flags.deleter
def flags(self):
self._flags = [ ]
@property
def compiler(self):
return self._compiler.cmd
@compiler.setter
def compiler(self, value):
if isinstance(value, basestring):
assert(not_empty(value) and isfile(value))
self._compiler = Executable(value)
elif isinstance(value, Executable):
self._compiler = value
else:
raise TypeError(type(value))
@property
def environ(self):
return self._env
@environ.setter
def environ(self, value):
if isinstance(value, dict):
self._env = value
elif isinstance(value, tuple):
if len(value) == 2:
key, val = value
self._env[key] = val
else:
raise TypeError('Setting environ to a tuple-type value is only valid when the length of the tuple is '
'2. For example: ("PATH", "C:\\Windows;C:\\python")')
else:
raise TypeError('Environ only accepts dicts and tuples with two items.')
def __iadd__(self, args):
""" Updates flags. Expects only flag-type args. (ex: -DNDEBUG, -I include, etc.) Any source files
specified will be ignored. """
self.extract_sources(args)
return self
def __isub__(self, obj):
""" Remove a stored flag. (The - or / prefix is optional. """
if isinstance(obj, basestring):
self._remove_flag(obj)
elif listlike(obj):
self._remove_flags(list(obj))
else:
raise TypeError('Dont know what to do with a value that isnt a collection or string.')
return self
def main(args):
cpp = CPP(compiler_path=which('cl.exe'))
srcs = cpp.extract_sources(args)
for src in map(absp, srcs):
base, ext = tuple(splitext(src))
outpath = base + '.out' + ext
print 'Writing to %s' % outpath
cpp.preprocess(src, outpath).close()
if __name__=='__main__':
main(sys.argv[1:])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment