gistfile1.py

#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# author: Christian Berendt <[email protected]>

from git import Repo # GitPython == 0.3.2.RC1
import glob2
from lxml import etree
import os
import shutil
import sys
import urllib2


def get_repository(repository):
    if os.path.exists(repository):
        repo = Repo(repository)
        repo.head.reference = repo.heads.master
        repo.remotes.origin.pull()
    else:
        Repo.clone_from("http://github.com/openstack/%s" % repository, repository)
        repo = Repo(repository)

    return repo


def main():
    #repository = 'openstack-manuals'
    #repository = 'operations-guide'
    repository = 'security-doc'

    if os.path.exists(repository):
        shutil.rmtree(repository)

    get_repository(repository)

    for xmlFile in glob2.glob("%s/**/*.xml" % repository):
        tree = etree.parse(xmlFile)

        for entry in tree.findall('//docbook:link',
                                      namespaces={'docbook': 'http://docbook.org/ns/docbook',
                                                  'xlink': 'http://www.w3.org/1999/xlink'}):

            try:
                url = entry.attrib['{http://www.w3.org/1999/xlink}href']
            except:
                continue

            try:
                urllib2.urlopen(url)
            except (urllib2.HTTPError, urllib2.URLError, ValueError) as e:
                print("%s - %s - %s" % (xmlFile, url, e))

if __name__ == '__main__':
    sys.exit(main())