Python - MinGW locally - Helper to downloaded toolchain into local subdirectory

mingwsdk.py

#!/usr/bin/env python3
#
# Use `locally` bootstrap code to get MinGW SDK for Windows
# operational from the local directory.

# This stuff is placed into public domain by
#    anatoly techtonik <[email protected]>

# --- bootstrap .locally ---
#
# this creates .locally/ subdirectory in the script's dir
# and sets a few global variables for convenience:
#
#   LOOT  - absolute path to .locally/ subdir
#   PY3K  - flag detecting Python 3 presence
#
# this provides some helpers:
#
#   localdir(name)   - returns absolute path to the new `name`
#                      dir created .locally
#
# this also downloads and imports to make available in global
# scope the following helper libraries:
#
#   wget
#

import os
import sys

PY3K = sys.version_info >= (3, 0)
if PY3K:
  import urllib.request as urllib
else:
  import urllib

# 1. create LIB dir for locally modules (.locally/.bootstrap)
#    and add it to Python sys.path to make them importable

LOOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '.locally'))

def localdir(name):
  '''create subdir in LOOT, return its path with ending '/' '''
  nupath = LOOT + '/' + name + '/'
  if not os.path.exists(nupath):
    os.makedirs(nupath)
  return nupath

LIB = LOOT + '/.bootstrap/'
if not os.path.exists(LIB):
  os.makedirs(LIB)
sys.path.insert(0, LIB)

# 2. download required Python files into LIB dir

required_files = [
  ('wget.py', 'https://bitbucket.org/techtonik/python-wget/raw/tip/wget.py'),
]

for f, url in required_files:
  if os.path.exists(LIB + f):
    print("Skipping " + f + " download")
  else:
    print("Downloading %s into %s" % (f, LIB))
    urllib.urlretrieve(url, LIB + f)

# 3. import downloaded stuff

from wget import download, filename_from_url, progress_callback

# --- /bootstrap


# --- custom code for downloading/installing MinGW

# specify additional languages to install
LANGUAGES = ['gcc-g++']

import os
import re

if not PY3K:
  print('\nNOTE: MinGW unpack requires `lzma` module available from '
        'Python 3.3\n')
  raw_input('Press Enter to continue..')
else:
  import tarfile

# a1. create local directory for MinGW SDK
MINGW = localdir('MinGW')

# a2. scrape http://www.mingw.org/wiki/InstallationHOWTOforMinGW
start = "Download at least the following"
end   = "These are the minimum requirements for a working MinGW GCC toolchain for C."
endlang  = "You may also want the following additional features"

print('Reading links from http://www.mingw.org/wiki/InstallationHOWTOforMinGW')
html = urllib.urlopen("http://www.mingw.org/wiki/InstallationHOWTOforMinGW").read()
if PY3K:
  html = html.decode('utf-8')
text = html.split(start)[1].split(end)[0]
links = re.findall('http://[^"]+', text)

# additional languages
text = html.split(end)[1].split(endlang)[0]
for lang in LANGUAGES:
  match = re.search(re.escape(lang) + '\s+\(([^\)]+)', text)
  if match:
    langlinks = re.findall('http://[^"]+', match.group(1))
    links.extend(langlinks)
  else:
    print('ERROR: requested language %s can not be found' % lang)
    input('Press Enter to continue..')

# a3. download files into .cache
CACHE = localdir('.cache')
os.chdir(CACHE)
print('Downloading %d links into %s' % (len(links), CACHE))
files = []
for l in links:
  # [ ] wget.py can not redetect filename after every redirect,
  #     so, preprocessing links to get proper filenames
  filename = filename_from_url(l.strip('/'))
  if os.path.exists(filename):
    print("..skipping " + filename + " download")
  else:
    print("..downloading %s" % (filename))
    filename = download(l.strip('/'), callback=progress_callback)
    print("")
    # print('\n....downloaded %s' % filename)
  files += [filename]

# a4. unpack archives
print('Changing into %s' % MINGW)
os.chdir(MINGW)
print('Unpacking %d files into %s' % (len(links), MINGW))
for f in files:
  tar = tarfile.open(CACHE + f)
  # if first file from archive exists, skip unpacking
  for check in tar:
    if check.isfile():
      break
  if os.path.exists(MINGW + check.name):
    print('..skipping unpack of %s' % (f))
  else:
    print('..unpacking %s' % (f))
    # reopening tar file (Python issue #16631)
    tar.close()
    tar = tarfile.open(CACHE + f)
    tar.extractall(MINGW)
  tar.close()

print('Done.')


# --- custom code for downloading/installing additional dependencies

ZLIB = True
# ^ if you have
# fatal error: zlib.h: No such file or directory

# the stuff is built in MinGW-MoarLibs
MOARLIBS = localdir('MinGW-MoarLibs')

links = []
files = []

# b1. finding dependencies
if ZLIB:
  # file http://zlib.net/zlib-1.2.7.tar.gz is linked from http://zlib.net/
  name = 'zlib'
  watch = ('http://zlib.net/', 'zlib-(?P<version>[.\d]+).tar.gz')
  regex = re.escape(watch[0]) + watch[1]
  print('Checking %s from %s' % (name, watch[0]))
  html = urllib.urlopen(watch[0]).read()
  if PY3K:
    html = html.decode('utf-8')
  versions = re.findall(regex, html)
  lastver = sorted([list(map(int, x.split('.'))) for x in versions])[-1]
  url = re.search(regex, html).group(0)
  links.append(url)

# b2. downloading dependencies
os.chdir(CACHE)
for l in links:
  filename = filename_from_url(l)
  if os.path.exists(filename):
    print("..skipping " + filename + " download")
  else:
    print("..downloading %s" % (filename))
    filename = download(l, callback=progress_callback)
    print("")
  files += [filename]

os.chdir(LOOT)