Skip to content

Instantly share code, notes, and snippets.

@pstaender

pstaender/list.py

Created November 23, 2014 21:24
Show Gist options
  • Save pstaender/1d9db8fddc91826f9578 to your computer and use it in GitHub Desktop.
Save pstaender/1d9db8fddc91826f9578 to your computer and use it in GitHub Desktop.
Download ZIP
List google scholar search results for csv usage: ./list.py > list.csv
Raw
list.py
#! /usr/bin/env python
"""
scholar.py -c 1 --txt --phrase "Upper Echelons - The Organization as a Reflection of Its Top Managers"
"""
import os
import re
import subprocess
directory = os.path.dirname(os.path.abspath(__file__))
from os import listdir
from os.path import isfile, join
pdfs = [ f for f in listdir(directory) if isfile(join(directory,f)) and re.search(r"\.pdf$", f, re.IGNORECASE) ]
print 'title;url;year;citations;versions;cluster_id;citations_list;versions_list;citations;filename'
for pdf in pdfs:
title = re.sub(r'(?i)\.pdf$','', pdf)
p = subprocess.Popen(['python', directory+'/scholar.py', '-c', '1', '--csv','--txt', '--phrase', title], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
print out.strip()+';'+pdf
Raw
scholar.py
#! /usr/bin/env python
"""
This module provides classes for querying Google Scholar and parsing
returned results. It currently *only* processes the first results
page. It is not a recursive crawler.
"""
# ChangeLog
# ---------
#
# 2.4: Bugfixes:
#
# - Correctly handle Unicode characters when reporting results
# in text format.
#
# - Correctly parse citation-only (i.e. linkless) results in
# Google Scholar results.
#
# 2.3: Additional features:
#
# - Direct extraction of first PDF version of an article
#
# - Ability to pull up an article cluster's results directly.
#
# This is based on work from @aliparsai on GitHub -- thanks!
#
# - Suppress missing search results (so far shown as "None" in
# the textual output form.
#
# 2.2: Added a logging option that reports full HTML contents, for
# debugging, as well as incrementally more detailed logging via
# -d up to -dddd.
#
# 2.1: Additional features:
#
# - Improved cookie support: the new --cookie-file options
# allows the reuse of a cookie across invocations of the tool;
# this allows higher query rates than would otherwise result
# when invoking scholar.py repeatedly.
#
# - Workaround: remove the num= URL-encoded argument from parsed
# URLs. For some reason, Google Scholar decides to propagate
# the value from the original query into the URLs embedded in
# the results.
#
# 2.0: Thorough overhaul of design, with substantial improvements:
#
# - Full support for advanced search arguments provided by
# Google Scholar
#
# - Support for retrieval of external citation formats, such as
# BibTeX or EndNote
#
# - Simple logging framework to track activity during execution
#
# 1.7: Python 3 and BeautifulSoup 4 compatibility, as well as printing
# of usage info when no options are given. Thanks to Pablo
# Oliveira (https://github.com/pablooliveira)!
#
# Also a bunch of pylinting and code cleanups.
#
# 1.6: Cookie support, from Matej Smid (https://github.com/palmstrom).
#
# 1.5: A few changes:
#
# - Tweak suggested by Tobias Isenberg: use unicode during CSV
# formatting.
#
# - The option -c|--count now understands numbers up to 100 as
# well. Likewise suggested by Tobias.
#
# - By default, text rendering mode is now active. This avoids
# confusion when playing with the script, as it used to report
# nothing when the user didn't select an explicit output mode.
#
# 1.4: Updates to reflect changes in Scholar's page rendering,
# contributed by Amanda Hay at Tufts -- thanks!
#
# 1.3: Updates to reflect changes in Scholar's page rendering.
#
# 1.2: Minor tweaks, mostly thanks to helpful feedback from Dan Bolser.
# Thanks Dan!
#
# 1.1: Made author field explicit, added --author option.
#
# Don't complain about missing docstrings: pylint: disable-msg=C0111
#
# Copyright 2010--2014 Christian Kreibich. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.
#
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import optparse
import os
import sys
import re
try:
# Try importing for Python 3
# pylint: disable-msg=F0401
# pylint: disable-msg=E0611
from urllib.request import HTTPCookieProcessor, Request, build_opener
from urllib.parse import quote
from http.cookiejar import MozillaCookieJar
except ImportError:
# Fallback for Python 2
from urllib2 import Request, build_opener, HTTPCookieProcessor
from urllib import quote
from cookielib import MozillaCookieJar
# Import BeautifulSoup -- try 4 first, fall back to older
try:
from bs4 import BeautifulSoup
except ImportError:
try:
from BeautifulSoup import BeautifulSoup
except ImportError:
print('We need BeautifulSoup, sorry...')
sys.exit(1)
# Support unicode in both Python 2 and 3. In Python 3, unicode is str.
if sys.version_info[0] == 3:
unicode = str # pylint: disable-msg=W0622
encode = lambda s: s # pylint: disable-msg=C0103
else:
encode = lambda s: s.encode('utf-8') # pylint: disable-msg=C0103
class Error(Exception):
"""Base class for any Scholar error."""
class FormatError(Error):
"""A query argument or setting was formatted incorrectly."""
class QueryArgumentError(Error):
"""A query did not have a suitable set of arguments."""
class ScholarConf(object):
"""Helper class for global settings."""
VERSION = '2.4'
LOG_LEVEL = 1
MAX_PAGE_RESULTS = 20 # Current maximum for per-page results
SCHOLAR_SITE = 'http://scholar.google.com'
# USER_AGENT = 'Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.9.2.9) Gecko/20100913 Firefox/3.6.9'
# Let's update at this point (3/14):
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0'
# If set, we will use this file to read/save cookies to enable
# cookie use across sessions.
COOKIE_JAR_FILE = None
class ScholarUtils(object):
"""A wrapper for various utensils that come in handy."""
LOG_LEVELS = {'error': 1,
'warn': 2,
'info': 3,
'debug': 4}
@staticmethod
def ensure_int(arg, msg=None):
try:
return int(arg)
except ValueError:
raise FormatError(msg)
@staticmethod
def log(level, msg):
if level not in ScholarUtils.LOG_LEVELS.keys():
return
if ScholarUtils.LOG_LEVELS[level] > ScholarConf.LOG_LEVEL:
return
sys.stderr.write('[%5s] %s' % (level.upper(), msg + '\n'))
sys.stderr.flush()
class ScholarArticle(object):
"""
A class representing articles listed on Google Scholar. The class
provides basic dictionary-like behavior.
"""
def __init__(self):
self.attrs = {
'title': [None, 'Title', 0],
'url': [None, 'URL', 1],
'year': [None, 'Year', 2],
'num_citations': [0, 'Citations', 3],
'num_versions': [0, 'Versions', 4],
'cluster_id': [None, 'Cluster ID', 5],
'url_pdf': [None, 'PDF link', 6],
'url_citations': [None, 'Citations list', 7],
'url_versions': [None, 'Versions list', 8],
'url_citation': [None, 'Citation link', 9],
}
# The citation data in one of the standard export formats,
# e.g. BibTeX.
self.citation_data = None
def __getitem__(self, key):
if key in self.attrs:
return self.attrs[key][0]
return None
def __len__(self):
return len(self.attrs)
def __setitem__(self, key, item):
if key in self.attrs:
self.attrs[key][0] = item
else:
self.attrs[key] = [item, key, len(self.attrs)]
def __delitem__(self, key):
if key in self.attrs:
del self.attrs[key]
def set_citation_data(self, citation_data):
self.citation_data = citation_data
def as_txt(self):
# Get items sorted in specified order:
items = sorted(list(self.attrs.values()), key=lambda item: item[2])
# Find largest label length:
max_label_len = max([len(str(item[1])) for item in items])
fmt = '%%%ds %%s' % max_label_len
res = []
for item in items:
if item[0] is not None:
res.append(fmt % (item[1], item[0]))
return '\n'.join(res)
def as_csv(self, header=False, sep=';'):
# Get keys sorted in specified order:
keys = [pair[0] for pair in \
sorted([(key, val[2]) for key, val in list(self.attrs.items())],
key=lambda pair: pair[1])]
res = []
if header:
res.append(sep.join(keys))
res.append(sep.join([unicode(self.attrs[key][0]) for key in keys]))
return '\n'.join(res)
def as_citation(self):
"""
Reports the article in a standard citation format. This works only
if you have configured the querier to retrieve a particular
citation export format. (See ScholarSettings.)
"""
return self.citation_data or ''
class ScholarArticleParser(object):
"""
ScholarArticleParser can parse HTML document strings obtained from
Google Scholar. This is a base class; concrete implementations
adapting to tweaks made by Google over time follow below.
"""
def __init__(self, site=None):
self.soup = None
self.article = None
self.site = site or ScholarConf.SCHOLAR_SITE
self.year_re = re.compile(r'\b(?:20|19)\d{2}\b')
def handle_article(self, art):
"""
The parser invokes this callback on each article parsed
successfully. In this base class, the callback does nothing.
"""
def parse(self, html):
"""
This method initiates parsing of HTML content, cleans resulting
content as needed, and notifies the parser instance of
resulting instances via the handle_article callback.
"""
self.soup = BeautifulSoup(html)
for div in self.soup.findAll(ScholarArticleParser._tag_checker):
self._parse_article(div)
self._clean_article()
if self.article['title']:
self.handle_article(self.article)
def _clean_article(self):
"""
This gets invoked after we have parsed an article, to do any
needed cleanup/polishing before we hand off the resulting
article.
"""
if self.article['title']:
self.article['title'] = self.article['title'].strip()
def _parse_article(self, div):
self.article = ScholarArticle()
for tag in div:
if not hasattr(tag, 'name'):
continue
if tag.name == 'div' and self._tag_has_class(tag, 'gs_rt') and \
tag.h3 and tag.h3.a:
self.article['title'] = ''.join(tag.h3.a.findAll(text=True))
self.article['url'] = self._path2url(tag.h3.a['href'])
if self.article['url'].endswith('.pdf'):
self.article['url_pdf'] = self.article['url']
if tag.name == 'font':
for tag2 in tag:
if not hasattr(tag2, 'name'):
continue
if tag2.name == 'span' and \
self._tag_has_class(tag2, 'gs_fl'):
self._parse_links(tag2)
def _parse_links(self, span):
for tag in span:
if not hasattr(tag, 'name'):
continue
if tag.name != 'a' or tag.get('href') is None:
continue
if tag.get('href').startswith('/scholar?cites'):
if hasattr(tag, 'string') and tag.string.startswith('Cited by'):
self.article['num_citations'] = \
self._as_int(tag.string.split()[-1])
# Weird Google Scholar behavior here: if the original
# search query came with a number-of-results limit,
# then this limit gets propagated to the URLs embedded
# in the results page as well. Same applies to
# versions URL in next if-block.
self.article['url_citations'] = \
self._strip_url_arg('num', self._path2url(tag.get('href')))
# We can also extract the cluster ID from the versions
# URL. Note that we know that the string contains "?",
# from the above if-statement.
args = self.article['url_citations'].split('?', 1)[1]
for arg in args.split('&'):
if arg.startswith('cites='):
self.article['cluster_id'] = arg[6:]
if tag.get('href').startswith('/scholar?cluster'):
if hasattr(tag, 'string') and tag.string.startswith('All '):
self.article['num_versions'] = \
self._as_int(tag.string.split()[1])
self.article['url_versions'] = \
self._strip_url_arg('num', self._path2url(tag.get('href')))
if tag.getText().startswith('Import'):
self.article['url_citation'] = self._path2url(tag.get('href'))
@staticmethod
def _tag_has_class(tag, klass):
"""
This predicate function checks whether a BeatifulSoup Tag instance
has a class attribute.
"""
res = tag.get('class') or []
if type(res) != list:
# BeautifulSoup 3 can return e.g. 'gs_md_wp gs_ttss',
# so split -- conveniently produces a list in any case
res = res.split()
return klass in res
@staticmethod
def _tag_checker(tag):
return tag.name == 'div' \
and ScholarArticleParser._tag_has_class(tag, 'gs_r')
@staticmethod
def _as_int(obj):
try:
return int(obj)
except ValueError:
return None
def _path2url(self, path):
"""Helper, returns full URL in case path isn't one."""
if path.startswith('http://'):
return path
if not path.startswith('/'):
path = '/' + path
return self.site + path
def _strip_url_arg(self, arg, url):
"""Helper, removes a URL-encoded argument, if present."""
parts = url.split('?', 1)
if len(parts) != 2:
return url
res = []
for part in parts[1].split('&'):
if not part.startswith(arg + '='):
res.append(part)
return parts[0] + '?' + '&'.join(res)
class ScholarArticleParser120201(ScholarArticleParser):
"""
This class reflects update to the Scholar results page layout that
Google recently.
"""
def _parse_article(self, div):
self.article = ScholarArticle()
for tag in div:
if not hasattr(tag, 'name'):
continue
if tag.name == 'h3' and self._tag_has_class(tag, 'gs_rt') and tag.a:
self.article['title'] = ''.join(tag.a.findAll(text=True))
self.article['url'] = self._path2url(tag.a['href'])
if self.article['url'].endswith('.pdf'):
self.article['url_pdf'] = self.article['url']
if tag.name == 'div' and self._tag_has_class(tag, 'gs_a'):
year = self.year_re.findall(tag.text)
self.article['year'] = year[0] if len(year) > 0 else None
if tag.name == 'div' and self._tag_has_class(tag, 'gs_fl'):
self._parse_links(tag)
class ScholarArticleParser120726(ScholarArticleParser):
"""
This class reflects update to the Scholar results page layout that
Google made 07/26/12.
"""
def _parse_article(self, div):
self.article = ScholarArticle()
for tag in div:
if not hasattr(tag, 'name'):
continue
if str(tag).lower().find('.pdf'):
if tag.find('div', {'class': 'gs_ttss'}):
self._parse_links(tag.find('div', {'class': 'gs_ttss'}))
if tag.name == 'div' and self._tag_has_class(tag, 'gs_ri'):
# There are (at least) two formats here. In the first
# one, we have a link, e.g.:
#
# <h3 class="gs_rt">
# <a href="http://dl.acm.org/citation.cfm?id=972384" class="yC0">
# <b>Honeycomb</b>: creating intrusion detection signatures using
# honeypots
# </a>
# </h3>
#
# In the other, there's no actual link -- it's what
# Scholar renders as "CITATION" in the HTML:
#
# <h3 class="gs_rt">
# <span class="gs_ctu">
# <span class="gs_ct1">[CITATION]</span>
# <span class="gs_ct2">[C]</span>
# </span>
# <b>Honeycomb</b> automated ids signature creation using honeypots
# </h3>
#
# We now distinguish the two.
try:
atag = tag.h3.a
self.article['title'] = ''.join(atag.findAll(text=True))
self.article['url'] = self._path2url(atag['href'])
if self.article['url'].endswith('.pdf'):
self.article['url_pdf'] = self.article['url']
except:
# Remove a few spans that have unneeded content (e.g. [CITATION])
for span in tag.h3.findAll(name='span'):
span.clear()
self.article['title'] = ''.join(tag.h3.findAll(text=True))
if tag.find('div', {'class': 'gs_a'}):
year = self.year_re.findall(tag.find('div', {'class': 'gs_a'}).text)
self.article['year'] = year[0] if len(year) > 0 else None
if tag.find('div', {'class': 'gs_fl'}):
self._parse_links(tag.find('div', {'class': 'gs_fl'}))
class ScholarQuery(object):
"""
The base class for any kind of results query we send to Scholar.
"""
def __init__(self):
self.url = None
self.num_results = ScholarConf.MAX_PAGE_RESULTS
def set_num_page_results(self, num_page_results):
msg = 'maximum number of results on page must be numeric'
self.num_results = ScholarUtils.ensure_int(num_page_results, msg)
def get_url(self):
"""
Returns a complete, submittable URL string for this particular
query instance. The URL and its arguments will vary depending
on the query.
"""
return None
class ClusterScholarQuery(ScholarQuery):
"""
This version just pulls up an article cluster whose ID we already
know about.
"""
SCHOLAR_CLUSTER_URL = ScholarConf.SCHOLAR_SITE + '/scholar?' \
+ 'cluster=%(cluster)s' \
+ '&num=%(num)s'
def __init__(self, cluster=None):
ScholarQuery.__init__(self)
self.cluster = None
self.set_cluster(cluster)
def set_cluster(self, cluster):
"""
Sets search to a Google Scholar results cluster ID.
"""
msg = 'cluster ID must be numeric'
self.cluster = ScholarUtils.ensure_int(cluster, msg)
def get_url(self):
if self.cluster is None:
raise QueryArgumentError('cluster query needs cluster ID')
urlargs = {'cluster': self.cluster,
'num': self.num_results or ScholarConf.MAX_PAGE_RESULTS}
for key, val in urlargs.items():
urlargs[key] = quote(str(val))
return self.SCHOLAR_CLUSTER_URL % urlargs
class SearchScholarQuery(ScholarQuery):
"""
This version represents the search query parameters the user can
configure on the Scholar website, in the advanced search options.
"""
SCHOLAR_QUERY_URL = ScholarConf.SCHOLAR_SITE + '/scholar?' \
+ 'as_q=%(words)s' \
+ '&as_epq=%(phrase)s' \
+ '&as_oq=%(words_some)s' \
+ '&as_eq=%(words_none)s' \
+ '&as_occt=%(scope)s' \
+ '&as_sauthors=%(authors)s' \
+ '&as_publication=%(pub)s' \
+ '&as_ylo=%(ylo)s' \
+ '&as_yhi=%(yhi)s' \
+ '&btnG=&hl=en&as_sdt=0,5&num=%(num)s'
def __init__(self):
ScholarQuery.__init__(self)
self.words = None # The default search behavior
self.words_some = None # At least one of those words
self.words_none = None # None of these words
self.phrase = None
self.scope_title = False # If True, search in title only
self.author = None
self.pub = None
self.timeframe = [None, None]
def set_words(self, words):
"""Sets words that *all* must be found in the result."""
self.words = words
def set_words_some(self, words):
"""Sets words of which *at least one* must be found in result."""
self.words_some = words
def set_words_none(self, words):
"""Sets words of which *none* must be found in the result."""
self.words_none = words
def set_phrase(self, phrase):
"""Sets phrase that must be found in the result exactly."""
self.phrase = phrase
def set_scope(self, title_only):
"""
Sets Boolean indicating whether to search entire article or title
only.
"""
self.scope_title = title_only
def set_author(self, author):
"""Sets names that must be on the result's author list."""
self.author = author
def set_pub(self, pub):
"""Sets the publication in which the result must be found."""
self.pub = pub
def set_timeframe(self, start=None, end=None):
"""
Sets timeframe (in years as integer) in which result must have
appeared. It's fine to specify just start or end, or both.
"""
if start:
start = ScholarUtils.ensure_int(start)
if end:
end = ScholarUtils.ensure_int(end)
self.timeframe = [start, end]
def get_url(self):
if self.words is None and self.words_some is None \
and self.words_none is None and self.phrase is None \
and self.author is None and self.pub is None \
and self.timeframe[0] is None and self.timeframe[1] is None:
raise QueryArgumentError('search query needs more parameters')
urlargs = {'words': self.words or '',
'words_some': self.words_some or '',
'words_none': self.words_none or '',
'phrase': self.phrase or '',
'scope': 'title' if self.scope_title else 'any',
'authors': self.author or '',
'pub': self.pub or '',
'ylo': self.timeframe[0] or '',
'yhi': self.timeframe[1] or '',
'num': self.num_results or ScholarConf.MAX_PAGE_RESULTS}
for key, val in urlargs.items():
urlargs[key] = quote(str(val))
return self.SCHOLAR_QUERY_URL % urlargs
class ScholarSettings(object):
"""
This class lets you adjust the Scholar settings for your
session. It's intended to mirror the features tunable in the
Scholar Settings pane, but right now it's a bit basic.
"""
CITFORM_NONE = 0
CITFORM_REFWORKS = 1
CITFORM_REFMAN = 2
CITFORM_ENDNOTE = 3
CITFORM_BIBTEX = 4
def __init__(self):
self.citform = 0 # Citation format, default none
self.per_page_results = ScholarConf.MAX_PAGE_RESULTS
self._is_configured = False
def set_citation_format(self, citform):
citform = ScholarUtils.ensure_int(citform)
if citform < 0 or citform > self.CITFORM_BIBTEX:
raise FormatError('citation format invalid, is "%s"' \
% citform)
self.citform = citform
self._is_configured = True
def set_per_page_results(self, per_page_results):
msg = 'page results must be integer'
self.per_page_results = ScholarUtils.ensure_int(per_page_results, msg)
self.per_page_results = min(self.per_page_results,
ScholarConf.MAX_PAGE_RESULTS)
self._is_configured = True
def is_configured(self):
return self._is_configured
class ScholarQuerier(object):
"""
ScholarQuerier instances can conduct a search on Google Scholar
with subsequent parsing of the resulting HTML content. The
articles found are collected in the articles member, a list of
ScholarArticle instances.
"""
# Default URLs for visiting and submitting Settings pane, as of 3/14
GET_SETTINGS_URL = ScholarConf.SCHOLAR_SITE + '/scholar_settings?' \
+ 'sciifh=1&hl=en&as_sdt=0,5'
SET_SETTINGS_URL = ScholarConf.SCHOLAR_SITE + '/scholar_setprefs?' \
+ 'q=' \
+ '&scisig=%(scisig)s' \
+ '&inststart=0' \
+ '&as_sdt=1,5' \
+ '&as_sdtp=' \
+ '&num=%(num)s' \
+ '&scis=%(scis)s' \
+ '%(scisf)s' \
+ '&hl=en&lang=all&instq=&inst=569367360547434339&save='
# Older URLs:
# ScholarConf.SCHOLAR_SITE + '/scholar?q=%s&hl=en&btnG=Search&as_sdt=2001&as_sdtp=on
class Parser(ScholarArticleParser120726):
def __init__(self, querier):
ScholarArticleParser120726.__init__(self)
self.querier = querier
def handle_article(self, art):
self.querier.add_article(art)
def __init__(self):
self.articles = []
self.query = None
self.cjar = MozillaCookieJar()
# If we have a cookie file, load it:
if ScholarConf.COOKIE_JAR_FILE and \
os.path.exists(ScholarConf.COOKIE_JAR_FILE):
try:
self.cjar.load(ScholarConf.COOKIE_JAR_FILE,
ignore_discard=True)
ScholarUtils.log('info', 'loaded cookies file')
except Exception as msg:
ScholarUtils.log('warn', 'could not load cookies file: %s' % msg)
self.cjar = MozillaCookieJar() # Just to be safe
self.opener = build_opener(HTTPCookieProcessor(self.cjar))
self.settings = None # Last settings object, if any
def apply_settings(self, settings):
"""
Applies settings as provided by a ScholarSettings instance.
"""
if settings is None or not settings.is_configured():
return True
self.settings = settings
# This is a bit of work. We need to actually retrieve the
# contents of the Settings pane HTML in order to extract
# hidden fields before we can compose the query for updating
# the settings.
html = self._get_http_response(url=self.GET_SETTINGS_URL,
log_msg='dump of settings form HTML',
err_msg='requesting settings failed')
if html is None:
return False
# Now parse the required stuff out of the form. We require the
# "scisig" token to make the upload of our settings acceptable
# to Google.
soup = BeautifulSoup(html)
tag = soup.find(name='form', attrs={'id': 'gs_settings_form'})
if tag is None:
ScholarUtils.log('info', 'parsing settings failed: no form')
return False
tag = tag.find('input', attrs={'type':'hidden', 'name':'scisig'})
if tag is None:
ScholarUtils.log('info', 'parsing settings failed: scisig')
return False
urlargs = {'scisig': tag['value'],
'num': settings.per_page_results,
'scis': 'no',
'scisf': ''}
if settings.citform != 0:
urlargs['scis'] = 'yes'
urlargs['scisf'] = '&scisf=%d' % settings.citform
html = self._get_http_response(url=self.SET_SETTINGS_URL % urlargs,
log_msg='dump of settings result HTML',
err_msg='applying setttings failed')
if html is None:
return False
ScholarUtils.log('info', 'settings applied')
return True
def send_query(self, query):
"""
This method initiates a search query (a ScholarQuery instance)
with subsequent parsing of the response.
"""
self.clear_articles()
self.query = query
html = self._get_http_response(url=query.get_url(),
log_msg='dump of query response HTML',
err_msg='results retrieval failed')
if html is None:
return
self.parse(html)
def get_citation_data(self, article):
"""
Given an article, retrieves citation link. Note, this requires that
you adjusted the settings to tell Google Scholar to actually
provide this information, *prior* to retrieving the article.
"""
if article['url_citation'] is None:
return False
if article.citation_data is not None:
return True
ScholarUtils.log('info', 'retrieving citation export data')
data = self._get_http_response(url=article['url_citation'],
log_msg='citation data response',
err_msg='requesting citation data failed')
if data is None:
return False
article.set_citation_data(data)
return True
def parse(self, html):
"""
This method allows parsing of provided HTML content.
"""
parser = self.Parser(self)
parser.parse(html)
def add_article(self, art):
self.get_citation_data(art)
self.articles.append(art)
def clear_articles(self):
"""Clears any existing articles stored from previous queries."""
self.articles = []
def save_cookies(self):
"""
This stores the latest cookies we're using to disk, for reuse in a
later session.
"""
if ScholarConf.COOKIE_JAR_FILE is None:
return False
try:
self.cjar.save(ScholarConf.COOKIE_JAR_FILE,
ignore_discard=True)
ScholarUtils.log('info', 'saved cookies file')
return True
except Exception as msg:
ScholarUtils.log('warn', 'could not save cookies file: %s' % msg)
return False
def _get_http_response(self, url, log_msg=None, err_msg=None):
"""
Helper method, sends HTTP request and returns response payload.
"""
if log_msg is None:
log_msg = 'HTTP response data follow'
if err_msg is None:
err_msg = 'request failed'
try:
ScholarUtils.log('info', 'requesting %s' % url)
req = Request(url=url, headers={'User-Agent': ScholarConf.USER_AGENT})
hdl = self.opener.open(req)
html = hdl.read()
ScholarUtils.log('debug', log_msg)
ScholarUtils.log('debug', '>>>>' + '-'*68)
ScholarUtils.log('debug', 'url: %s' % hdl.geturl())
ScholarUtils.log('debug', 'result: %s' % hdl.getcode())
ScholarUtils.log('debug', 'headers:\n' + str(hdl.info()))
ScholarUtils.log('debug', 'data:\n' + html)
ScholarUtils.log('debug', '<<<<' + '-'*68)
return html
except Exception as err:
ScholarUtils.log('info', err_msg + ': %s' % err)
return None
def txt(querier):
articles = querier.articles
for art in articles:
print(encode(art.as_txt()) + '\n')
def csv(querier, header=False, sep=';'):
articles = querier.articles
for art in articles:
result = art.as_csv(header=header, sep=sep)
print(encode(result))
header = False
def citation_export(querier):
articles = querier.articles
for art in articles:
print(art.as_citation() + '\n')
def main():
usage = """scholar.py [options] <query string>
A command-line interface to Google Scholar.
Examples:
# Retrieve one article written by Einstein on quantum theory:
scholar.py -c 1 --author "albert einstein" --phrase "quantum theory"
# Retrieve a BibTeX entry for that quantum theory paper:
scholar.py -c 1 -C 17749203648027613321 --citation bt
# Retrieve five articles written by Einstein after 1970 where the title
# does not contain the words "quantum" and "theory":
scholar.py -c 5 -a "albert einstein" -t --none "quantum theory" --after 1970"""
fmt = optparse.IndentedHelpFormatter(max_help_position=50, width=100)
parser = optparse.OptionParser(usage=usage, formatter=fmt)
group = optparse.OptionGroup(parser, 'Query arguments',
'These options define search query arguments and parameters.')
group.add_option('-a', '--author', metavar='AUTHORS', default=None,
help='Author name(s)')
group.add_option('-A', '--all', metavar='WORDS', default=None, dest='allw',
help='Results must contain all of these words')
group.add_option('-s', '--some', metavar='WORDS', default=None,
help='Results must contain at least one of these words')
group.add_option('-n', '--none', metavar='WORDS', default=None,
help='Results must contain none of these words')
group.add_option('-p', '--phrase', metavar='PHRASE', default=None,
help='Results must contain exact phrase')
group.add_option('-t', '--title-only', action='store_true', default=False,
help='Search title only')
group.add_option('-P', '--pub', metavar='PUBLICATIONS', default=None,
help='Results must have appeared in this publication')
group.add_option('--after', metavar='YEAR', default=None,
help='Results must have appeared in or after given year')
group.add_option('--before', metavar='YEAR', default=None,
help='Results must have appeared in or before given year')
group.add_option('-C', '--cluster-id', metavar='CLUSTER_ID', default=None,
help='Do not search, just use articles in given cluster ID')
group.add_option('-c', '--count', type='int', default=None,
help='Maximum number of results')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Output format',
'These options control the appearance of the results.')
group.add_option('--txt', action='store_true',
help='Print article data in text format (default)')
group.add_option('--csv', action='store_true',
help='Print article data in CSV form (separator is "|")')
group.add_option('--csv-header', action='store_true',
help='Like --csv, but print header with column names')
group.add_option('--citation', metavar='FORMAT', default=None,
help='Print article details in standard citation format. Argument Must be one of "bt" (BibTeX), "en" (EndNote), "rm" (RefMan), or "rw" (RefWorks).')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Miscellaneous')
group.add_option('--cookie-file', metavar='FILE', default=None,
help='File to use for cookie storage. If given, will read any existing cookies if found at startup, and save resulting cookies in the end.')
group.add_option('-d', '--debug', action='count', default=0,
help='Enable verbose logging to stderr. Repeated options increase detail of debug output.')
group.add_option('-v', '--version', action='store_true', default=False,
help='Show version information')
parser.add_option_group(group)
options, _ = parser.parse_args()
# Show help if we have neither keyword search nor author name
if len(sys.argv) == 1:
parser.print_help()
return 1
if options.debug > 0:
options.debug = min(options.debug, ScholarUtils.LOG_LEVELS['debug'])
ScholarConf.LOG_LEVEL = options.debug
ScholarUtils.log('info', 'using log level %d' % ScholarConf.LOG_LEVEL)
if options.version:
print 'This is scholar.py %s.' % ScholarConf.VERSION
return 0
if options.cookie_file:
ScholarConf.COOKIE_JAR_FILE = options.cookie_file
# Sanity-check the options: if they include a cluster ID query, it
# makes no sense to have search arguments:
if options.cluster_id is not None:
if options.author or options.allw or options.some or options.none \
or options.phrase or options.title_only or options.pub \
or options.after or options.before:
print 'Cluster ID queries do not allow additional search arguments.'
return 1
querier = ScholarQuerier()
settings = ScholarSettings()
if options.citation == 'bt':
settings.set_citation_format(ScholarSettings.CITFORM_BIBTEX)
elif options.citation == 'en':
settings.set_citation_format(ScholarSettings.CITFORM_ENDNOTE)
elif options.citation == 'rm':
settings.set_citation_format(ScholarSettings.CITFORM_REFMAN)
elif options.citation == 'rw':
settings.set_citation_format(ScholarSettings.CITFORM_REFWORKS)
elif options.citation is not None:
print 'Invalid citation link format, must be one of "bt", "en", "rm", or "rw".'
return 1
querier.apply_settings(settings)
if options.cluster_id:
query = ClusterScholarQuery(cluster=options.cluster_id)
else:
query = SearchScholarQuery()
if options.author:
query.set_author(options.author)
if options.allw:
query.set_words(options.allw)
if options.some:
query.set_words_some(options.some)
if options.none:
query.set_words_none(options.none)
if options.phrase:
query.set_phrase(options.phrase)
if options.title_only:
query.set_scope(True)
if options.pub:
query.set_pub(options.pub)
if options.after or options.before:
query.set_timeframe(options.after, options.before)
if options.count is not None:
options.count = min(options.count, ScholarConf.MAX_PAGE_RESULTS)
query.set_num_page_results(options.count)
querier.send_query(query)
if options.csv:
csv(querier)
elif options.csv_header:
csv(querier, header=True)
elif options.citation is not None:
citation_export(querier)
else:
txt(querier)
if options.cookie_file:
querier.save_cookies()
return 0
if __name__ == "__main__":
sys.exit(main())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment