Skip to content

Instantly share code, notes, and snippets.

@acdha
acdha / threaded-requests.py
Created May 15, 2015 18:23
Multi-threaded network I/O in Python 2/3
from __future__ import absolute_import, unicode_literals, print_function
from multiprocessing.pool import ThreadPool
import requests
pool = ThreadPool(processes=8)
for response in pool.imap_unordered(requests.get, urls):
print(response.status_code, response.url)
@acdha
acdha / requests-multiprocessing-test.py
Last active August 29, 2015 14:21
Tracking down sporadic failures in requests under multiprocessing
#!/usr/bin/env python
from __future__ import absolute_import, print_function, unicode_literals
import multiprocessing
import sys
import time
import requests
# encoding: utf-8
from __future__ import absolute_import
from itertools import count, izip_longest, tee
def chunked_iterator(iterable, chunk_size):
"""Given an iterator, yield individual items but consume them in chunk_size
batches so we can e.g. retrieve records from Solr in 50-100 batches
rather than the default 10
@acdha
acdha / initial results.txt
Last active March 18, 2025 09:12
Comparing performance adding rows to a large table using innerHTML, the DOM API, React JSX and, eventually, React.createElement
Render using innerHTML took:0.140s; tbody height=536364px
Render using DOM took:0.076s; tbody height=536364px
Render using React took:1.443s; tbody height=536364px
@acdha
acdha / post-checkout
Created March 24, 2015 21:03
Git post-checkout hook for projects which use npm
#!/bin/sh
echo "Installing npm dependencies"
npm install ~/Projects/bagger-js
--- py-svn.c 2015-02-24 10:10:21.000000000 -0500
+++ py-hg.c 2015-02-24 10:11:11.000000000 -0500
@@ -11,7 +11,7 @@
/* Ensure ob_item has room for at least newsize elements, and set
* ob_size to newsize. If newsize > ob_size on entry, the content
* of the new slots at exit is undefined heap trash; it's the caller's
- * responsiblity to overwrite them with sane values.
+ * responsibility to overwrite them with sane values.
* The number of allocated elements may grow, shrink, or stay the same.
* Failure is impossible if newsize <= self.allocated on entry, although
@acdha
acdha / gist:f1f506b47a38574f3a5f
Last active August 29, 2015 14:13
Quick mock example showing how to simulate errors
>>> from unittest import mock
>>> import json
>>> m_o = mock.mock_open()
>>> m_o.side_effect = OSError('Insufficient voltage')
>>> json.load(m_o('foo.json', 'rb'))
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/Cellar/python3/3.4.2_1/Frameworks/Python.framework/Versions/3.4/lib/python3.4/unittest/mock.py", line 896, in __call__
return _mock_self._mock_call(*args, **kwargs)
File "/usr/local/Cellar/python3/3.4.2_1/Frameworks/Python.framework/Versions/3.4/lib/python3.4/unittest/mock.py", line 952, in _mock_call
@acdha
acdha / es6-destructuring-assignment.js
Last active August 29, 2015 14:10
Things which work in Firefox, Safari and maybe even IE but not V8
/* jshint esnext:true */
var [foo, bar] = [1, 2];
console.log('Foo is', foo);
console.log('Bar is', bar);
@acdha
acdha / dump-ddc-translations.py
Created November 24, 2014 22:25
Experiment with rdflib to retrieve DDC translations from http://dewey.info using rdflib
#!/usr/bin/env python
# encoding: utf-8
from __future__ import absolute_import, print_function, unicode_literals
import rdflib
g = rdflib.Graph()
for code in ('6', '64', '641'):
@acdha
acdha / iterators.py
Last active April 23, 2020 21:24
django-haystack sync_index command which retrieves the full list of database and search record IDs so only records which are not present in the search index will be processed
# encoding: utf-8
from __future__ import absolute_import, division, print_function
from itertools import count, izip_longest, tee
def chunked_iterator(iterable, chunk_size):
"""Given an iterator, yield individual items but consume them in chunk_size
batches so we can e.g. retrieve records from Solr in 50-100 batches
rather than the default 10