Download and create file and folders for web filter

create_files.py

#!/usr/local/bin/python
# -*- coding: utf-8 -*-
#
#  Get files for squid33 on packages server of the pfSense
#  Author: Silvio Giunge a.k.a Kanazuchi
#  1410180880
#  a2FuYXp1Y2hpQGFsdm9saXZyZS5jb20=
#

import os
import re
from urllib2 import urlopen
import xml.etree.ElementTree as etree

S_PKG_SITE = "https://packages.pfsense.org/packages/config/squid3/33/squid.xml"
WF_PKG_SITE = "http://192.168.8.14/packages/BluePexUTM-beta-packages/config/webfilter/webfilter.xml"

if os.path.exists('/usr/local/pkg/squid.xml'):
    os.unlink('/usr/local/pkg/squid.xml')
for line in urlopen(S_PKG_SITE).read().split('\n'):
    if line != "":
        print >> open('/usr/local/pkg/squid.xml', 'a'), \
            re.sub(r'pbi\/squid-amd64', 'local', line)

os.chmod('/usr/local/pkg/squid.xml', 0755)

def sync_files_squid(xml_file):

    for child in xml_file.findall('additional_files_needed'):
        if(re.match(r"http", child[2].text)):
            print "Reading file from \033[93m%s\033[97m" % child[2].text
            file_name = os.path.split(child[2].text)[1]
            print "Creating file \033[97m%s in \033[93m%s\033[97m" \
                % (file_name, child[0].text)
            if not os.path.isdir(child[0].text):
                os.mkdir(child[0].text, 0755)
            path_file = os.path.join(child[0].text, file_name)
            new_file = urlopen(child[2].text).read().split('\n')
            if os.path.exists(path_file):
                os.unlink(path_file)
            for line in new_file:
                print >> open(path_file, 'a'), re.sub(r'pbi\/squid-amd64', 'local', 
                    line)
            print "Give permisson \033[93m%s\033[97m for \033[93m%s\033[97m in \033[93m%s\033[97m" \
                % (child[1].text, file_name, child[0].text)
            os.chmod(path_file, int(child[1].text))
            print ''

sync_files_squid(etree.fromstring(open('/usr/local/pkg/squid.xml', 'r').read()))

if os.path.exists('/usr/local/pkg/webfilter.xml'):
    os.unlink('/usr/local/pkg/webfilter.xml')
for line in urlopen(WF_PKG_SITE).read().split('\n'):
    if line != "":
        print >> open('/usr/local/pkg/webfilter.xml', 'a'), \
            re.sub(r'pbi\/squid-amd64', 'local', line)

os.chmod('/usr/local/pkg/webfilter.xml', 0755)

def sync_files_wf(xml_file):

    for child in xml_file.findall('additional_files_needed'):
        if(re.match(r"http", child[2].text)):
            print "Reading file from \033[93m%s\033[97m" % re.sub(\
                r'^(.*packages)', \
                'http://192.168.8.14/packages/BluePexUTM-beta-packages', \
                child[2].text)
            file_name = os.path.split(child[2].text)[1]
            print "Creating file \033[97m%s in \033[93m%s\033[97m" \
                % (file_name, child[0].text)
            if not os.path.isdir(child[0].text):
                os.mkdir(child[0].text, 0755)
            path_file = os.path.join(child[0].text, file_name)
            loc_file = re.sub(
                "http://www.pfsense.org/packages", 
                "http://192.168.8.14/packages/BluePexUTM-beta-packages", 
                child[2].text)
            new_file = urlopen(loc_file).read().split('\n')
            if os.path.exists(path_file):
                os.unlink(path_file)
            for line in new_file:
                print >> open(path_file, 'a'), \
                    re.sub(r'pbi\/squid-amd64', 'local', line)
            print "Give permisson \033[93m%s\033[97m for \033[93m%s\033[97m in \033[93m%s\033[97m" \
                % (child[1].text, file_name, child[0].text)
            os.chmod(path_file, int(child[1].text))
            print ''

sync_files_wf(etree.fromstring(open('/usr/local/pkg/webfilter.xml', 'r').read()))

files = open('files_webfilter_needed', 'r').read().split('\n')

for f in files:
    if f != '':
        path, file_name = os.path.split(f)
        if os.path.exists(f):
            os.system('cp -v {} {}'.format(f, os.path.join(path, "{}_old".format(
                file_name))))
            os.system('cp -v files_webfilter/{} {}'.format(file_name, f))
        else:
            os.system('cp -v files_webfilter/{} {}'.format(file_name, f))