Skip to content

Instantly share code, notes, and snippets.

@mikewest

mikewest/build.py

Created April 22, 2010 08:55
Show Gist options
  • Save mikewest/374996 to your computer and use it in GitHub Desktop.
Save mikewest/374996 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
build.py
#!/usr/bin/env python
# encoding: utf-8;
from __future__ import with_statement
import os
import errno
import sys
import re
import subprocess
import hashlib
import shutil
from optparse import OptionParser
try:
from yaml import load, CLoader as Loader
except ImportError:
from yaml import Loader, Dumper
def mkdir_p(path):
try:
os.makedirs(path)
except OSError, exc:
if exc.errno == errno.EEXIST:
pass
else: raise
def versionhash( buffer ):
return hashlib.sha1( buffer ).hexdigest()[0:8]
def hashfile( file ):
with open( file, 'rb' ) as f:
return versionhash( f.read() )
class StaticAssetBuilder(object):
def __init__( self, settings ):
self.urls_seen = {}
self.settings = settings
def log( self, message ):
if self.settings[ 'VERBOSE' ]:
print message
def process_urls( self, line, source ):
"""
Given a line of text, find all strings of the form
`/static_assets/dev/...`, and generate appropriate
live URLs.
For our purposes, that means:
1. Assign the file a CDN server (reusing previously assigned
servers if we've already seen the file).
2. Strip the dev root pattern (`/static_assets/dev`), and replace
with the live pattern (`/static_assets/build`)
3. Tag each file with it's content hash.
"""
urls_seen = self.urls_seen
settings = self.settings
def replacer( matchobj ):
url = matchobj.group(0)
if not url in urls_seen:
static_file = url.replace( matchobj.group( 1 ), settings[ 'SOURCE_ROOT' ] )
if self.settings[ 'CDN' ]:
if not os.path.isfile( static_file ):
print "WARNING: You've included `%s` in `%s`. It doesn't seem to exist." % ( static_file, source )
hash = '00000000'
else:
hash = hashfile( static_file )
cdn = settings[ 'CDN_SERVERS' ][ len( urls_seen ) % len( settings[ 'CDN_SERVERS' ] ) ]
hashed_file = re.sub( r'(.\w+)$', r'.%s\1' % hash, matchobj.group( 2 ) )
urls_seen[ url ] = "http://%s%s/%s" % ( cdn, settings[ 'LIVE_ROOT_PATTERN' ], hashed_file )
static_file_dest = os.path.join( settings[ 'OUTPUT_ROOT' ], hashed_file )
else:
urls_seen[ url ] = url
static_file_dest = os.path.join( settings[ 'OUTPUT_ROOT' ], matchobj.group( 2 ) )
mkdir_p( os.path.dirname( static_file_dest ) )
shutil.copy( static_file, static_file_dest )
return urls_seen[ url ]
return re.sub(
self.settings[ 'DEV_ROOT_PATTERN' ],
replacer,
line)
def static_combine( self, end_file, to_combine, delimiter="\n/* Begin: %s */\n" ):
self.log( "* Building `%s`" % os.path.basename( end_file ) )
if end_file.find( '.css' ) != -1:
mode = 'CSS'
else:
mode = 'JS'
processed_output = ''
if mode == 'CSS':
processed_output = """@charset "UTF-8";\n"""
for static_file in to_combine:
if os.path.isfile( static_file ):
if delimiter:
processed_output += delimiter % os.path.split( static_file )[ 1 ]
with open( static_file, 'r' ) as f:
for line in f:
processed_output += self.process_urls( line, static_file )
if processed_output:
with open( end_file, 'w' ) as combo:
combo.write( processed_output )
if self.settings[ 'COMPRESS' ]:
try:
command = self.settings[ 'COMPRESSION_COMMAND' ] % end_file
proc = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE )
output = proc.communicate()[ 0 ]
hash = versionhash( output )
with open( re.sub( r'(\.\w+)$', r'.%s\1' % hash, end_file ), 'w' ) as minified:
minified.write( output )
except AttributeError, error:
raise CommandError("COMPRESSION_COMMAND not set")
except TypeError, error:
raise CommandError("No string substitution provided for the input file to be passed to the argument ('cmd %s')")
def build( self ):
for filetype in [ 'js', 'css' ]:
if filetype in self.settings[ 'FILES' ]:
to_build = self.settings[ 'FILES' ][ filetype ]
build_root = os.path.join( self.settings[ 'OUTPUT_ROOT' ], filetype )
mkdir_p( build_root )
for filename, filelist in to_build.items():
self.static_combine(
end_file=os.path.join( build_root, filename ),
to_combine=[ os.path.join( self.settings[ 'SOURCE_ROOT' ], filetype, f) for f in filelist ]
)
def main(argv=None):
if argv is None:
argv = sys.argv
default_root = os.path.dirname(os.path.abspath(__file__))
parser = OptionParser(usage="Usage: %prog [options]", version="%prog 0.1")
parser.add_option( "--verbose",
action="store_true", dest="verbose_mode", default=False,
help="Verbose mode")
parser.add_option( "-o", "--output",
action="store",
dest="output_root",
default=os.path.join(default_root, '../build'),
help="Directory to which output will be written")
parser.add_option( "-s", "--source",
action="store",
dest="source_root",
default=os.path.join(default_root, "../src"),
help="Directory from which source files will be read")
parser.add_option( "-c", "--compress",
action="store_true",
dest="compress",
default=False,
help="Compress the generated files using the specified compression command")
parser.add_option( "--cdn",
action="store_true",
dest="use_cdn",
default=False,
help="Generate CDN URLs for static assets in generated files.")
(options, args) = parser.parse_args()
with open( "%s/_config/build.yaml" % options.source_root, 'r' ) as f:
settings = load( f )
settings[ 'COMPRESSION_COMMAND' ] = settings[ 'COMPRESSION_COMMAND' ] % default_root
settings[ 'VERBOSE' ] = options.verbose_mode
settings[ 'CDN' ] = options.use_cdn
settings[ 'COMPRESS'] = options.compress
settings[ 'OUTPUT_ROOT' ] = options.output_root
settings[ 'SOURCE_ROOT' ] = options.source_root
builder = StaticAssetBuilder( settings )
builder.build()
if __name__ == "__main__":
sys.exit(main())
Raw
config.py
COMPRESSION_COMMAND: 'java -jar "%s/lib/yuicompressor-2.4.2.jar" "%%s"'
CDN_SERVERS:
- "pix1.sueddeutsche.de"
- "pix2.sueddeutsche.de"
DEV_ROOT_PATTERN: '(/static_assets/dev)/([\-_a-zA-Z0-9\.\/]+)'
LIVE_ROOT_PATTERN: '/static_assets/build'
FILES:
js:
"grid.js":
- "copyright.js"
- "third-party/jquery-1.4.2.js"
- "third-party/jquery.onAvailable-1.0.js"
- "third-party/linktracker.js"
- "third-party/swfobject.js"
- "sde/sde.js"
- "advertisement/advertisement.js"
- "advertisement/google.js"
- "advertisement/sitestat.js"
- "advertisement/yahoo.js"
#
# JS Utils
#
- "utils/utils.js"
- "utils/config.js"
- "utils/embedflash.js"
- "utils/interval.js"
- "utils/listtoselect.js"
- "utils/popup.js"
#
# Widget Base Classes
#
- "widget/widget.js"
- "widget/dropdowncontrol.js"
- "widget/lightboxcontrol.js"
- "widget/overlay.js"
- "widget/tabcontrol.js"
#
# Widget Implementations
#
- "widget/bookmarking.js"
- "widget/dynamiclist.js"
- "widget/flashelement.js"
- "widget/login.js"
- "widget/mailtofriend.js"
- "widget/poll.js"
- "widget/projector.js"
- "widget/tabs.js"
- "widget/video.js"
- "widget/weitereangebote.js"
- "widget/zoomable.js"
#
# Initalization
#
- "init/siteheader.js"
- "init/article3.js"
- "init/homepage.js"
css:
"grid.css":
- "copyright.css"
- "yui/reset.css"
- "yui/fonts.css"
- "decoration.css"
- "grid.css"
- "sidebar.css"
- "site/siteheader.css"
- "site/sitefooter.css"
- "colors/colors-ressort.css"
- "colors/colors-special.css"
- "article/article.css"
- "article/header.css"
- "article/footer.css"
- "gallery/gallery.css"
- "gallery/header.css"
- "gallery/footer.css"
- "modules/paging.css"
- "modules/modules.css"
- "modules/basebox.css"
- "modules/basebox_columns.css"
- "modules/basebox_columns_infothek.css"
- "modules/basebox_columns_sparmeister.css"
- "modules/basebox_columns_themelist.css"
- "modules/basebox_comments.css"
- "modules/basebox_impression.css"
- "modules/basebox_singlelink.css"
- "modules/basebox_inlinevideo.css"
- "modules/basebox_dynamiclist.css"
- "modules/basebox_focusedteaser.css"
- "modules/basebox_gallerylist.css"
- "modules/basebox_imageblock.css"
- "modules/basebox_imageblock_related.css"
- "modules/basebox_infothek.css"
- "modules/basebox_liveticker.css"
- "modules/basebox_opinion.css"
- "modules/basebox_szpromo.css"
- "modules/basebox_tagcloud.css"
- "modules/basebox_mailtofriend.css"
- "modules/mehrzumthema.css"
- "modules/themenbox.css"
- "modules/inlinemap.css"
- "modules/bildbanderolle.css"
- "modules/breakingnewsletter.css"
- "modules/headslot.css"
- "modules/headslot_image.css"
- "modules/bildergaleriebox-stoerer.css"
- "modules/projector.css"
- "modules/tabbox.css"
- "modules/socialbookmarking.css"
- "modules/servicebox.css"
- "modules/xlinks.css"
- "modules/toplisten.css"
- "modules/poll.css"
- "modules/tabbox_finance.css"
- "modules/teaserlist.css"
- "modules/teaserlist_breakingnews.css"
- "modules/teaserlist_metadata.css"
- "modules/teaserlist_paging.css"
- "modules/teaserlist_topteaser.css"
- "modules/themehighlights.css"
# Ressortblock, _then_ stockchart, _then_ basebox_stockchart.
- "modules/ressortblock.css"
- "modules/stockchart.css"
- "modules/basebox_stockchart.css"
- "modules/hp_sonderthemen.css"
# Whisper, _then_ basebox_whisper.
- "modules/whisper.css"
- "modules/basebox_whisper.css"
#
# Advertisement Styles. These are my favourites.
#
- "advertisement/yahoo.css"
- "advertisement/google.css"
- "advertisement/ivw.css"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment