Cache all packages from OpenWrt trunk repository

OpenWrt-Package-Downloader.py

#!/usr/bin/env python2  
#coding=utf-8
#  
# Openwrt Package Grabber  
#  
# Copyright (C) 2014 http://www.shuyz.com
# modified 2019 craigphicks
#     

import urllib2 
import re
import os
import sys

if len(sys.argv[1]) != 2:
    print 'USAGE <url of openwrt packages dir> <destination dir>'

# the url of package list page, end with "/"
#baseurl = 'http://downloads.openwrt.org/snapshots/trunk/ar71xx/packages/'
baseurl = sys.argv[1]

# which directory to save all the packages, end with "/"
savedir = './download/'
if not os.path.exists(savedir):
    os.makedirs(savedir)

print 'fetching package list from ' + baseurl
content = urllib2.urlopen(baseurl, timeout=15).read()

print 'packages list ok, analysing...'
pattern = r'<a href="(.*?)">'
items = re.findall(pattern, content)

print 'printing each matching string in page'
for item in items:
    print '    ' + item

print ''
print 'begin downloading'

cnt = 0
for item in items:
    if item == '../':
        continue
    if item[0]=='/':
        continue
    else:
        cnt += 1
        print 'downloading item %d: '%(cnt) + item
        if os.path.isfile(savedir + item):
            print 'file exists, ignored.'
        else:
            rfile = urllib2.urlopen(baseurl + item) 
            with open(savedir + item, "wb") as code:
                code.write(rfile.read())

print 'done!'