Adding caching to dedupe queries for Newgrabber

dedupe.py.diff

diff --git a/dedupe.py b/dedupe.py
index 26230a6..1e346ff 100644
--- a/dedupe.py
+++ b/dedupe.py
@@ -11,6 +11,10 @@ import warcio
 from warcio.archiveiterator import ArchiveIterator
 from warcio.warcwriter import WARCWriter
 
+proxies = {
+  'http': 'http://127.0.0.1:8080',
+}
+
 if not warcio.__file__ == os.path.join(os.getcwd(), 'warcio', '__init__.pyc'):
     print('Warcio was not imported correctly.')
     print('Location: ' + warcio.__file__ + '.')
@@ -27,7 +31,7 @@ def ia_available(url, digest):
         try:
             tries += 1
             ia_data = requests.get('http://NewsGrabberDedupe.b-cdn.net/{hashed}' \
-                                   .format(hashed=hashed), timeout=60)
+                                   .format(hashed=hashed), timeout=60, proxies=proxies)
             if not ';' in ia_data.text:
                 return False
             return ia_data.text.split(';', 1)