A basic attempt at using the newspaper module

quick-example.py

#!/usr/bin/env python

import sys
import logging
import newspaper


def main(argv=None):
    if argv is None:
        argv = sys.argv

    if len(argv) != 2:
        print '{} <url>'.format(argv[0])
        return -1

    logging.basicConfig(filename='newspaper.log', format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', level=logging.INFO)
    paper = newspaper.build(argv[1])

    print '==> articles <=='
    for article in paper.articles:
        logging.info('starting article download')
        article.download()
        logging.info('finished article download')

        print '** {} **'.format(article.title.encode('ascii', 'ignore'))
        print 'URL: {}'.format(article.url)

        logging.info('starting article parsing')
        article.parse()
        logging.info('finished article parsing')
        print 'Authors: {}'.format(', '.join(article.authors))

        logging.info('starting article neuro-linguistic processing')
        article.nlp()
        logging.info('finishing article neuro-linguistic processing')
        print 'Summary: {}'.format(article.summary.encode('ascii', 'ignore'))
        print 'Keywords: {}'.format(', '.join(article.keywords))

        print

    print '==> categories <=='
    for category in paper.category_urls():
        print category

    return 0


if __name__ == '__main__':
    sys.exit(main())