This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python3 | |
# | |
# Decompressor/compressor for files in Mozilla's "mozLz4" format. Firefox uses this file format to | |
# compress e. g. bookmark backups (*.jsonlz4). | |
# | |
# This file format is in fact just plain LZ4 data with a custom header (magic number [8 bytes] and | |
# uncompressed file size [4 bytes, little endian]). | |
# | |
# This Python 3 script requires the LZ4 bindings for Python, see: https://pypi.python.org/pypi/lz4 | |
# |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf-8 -*- | |
import logging | |
import scrapy | |
import feedparser | |
class RSSSpider(scrapy.Spider): | |
name = "rss" | |
# Can pass some URLs on the commandline: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
# | |
# (c) 2013 "nyov" | |
set -e | |
# | |
# Script to download, update and append blocklist to /etc/hosts | |
# for more info, visit http://winhelp2002.mvps.org/hosts.htm | |
# |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# -*- coding: utf-8 -*- | |
from __future__ import print_function | |
import logging | |
from scrapy.utils.log import configure_logging | |
from scrapy.spiders import Spider | |
from scrapy.exceptions import CloseSpider | |
from scrapy.http import Request |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python3 | |
# by nyov, Public Domain | |
from __future__ import unicode_literals, print_function | |
import six | |
from six.moves import cPickle as pickle | |
# https://github.com/sigmavirus24/github3.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
from __future__ import print_function | |
import sys, os | |
# This test should show how urllib.proxy_bypass_environment() | |
# doesn't handle proxy environment variables correctly | |
# on *NIX systems. | |
# (It does not understand IPs or CIDR notations.) | |
# | |
# For a possible solution see the `requests` library: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# -*- coding: utf-8 -*- | |
# A spider example on using reactor.callLater() | |
# for delays and repetition. | |
# scrapy 0.24 | |
import scrapy | |
from twisted.internet import reactor, defer |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
DROP TABLE IF EXISTS fruits; | |
CREATE TABLE fruits ( | |
id SERIAL NOT NULL, | |
name TEXT, | |
valid_from TIMESTAMP NOT NULL, | |
valid_to TIMESTAMP NOT NULL | |
); | |
DROP TRIGGER IF EXISTS fruits_before ON fruits; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python | |
import rados | |
# choose whatever you have, lzma, zlib, snappy... | |
# lzo seems to nicely represent the bug | |
compressor = __import__('lzo', fromlist=['']) | |
def _complete(completion, data_read): | |
print "oncomplete called:" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from scrapy.http import Request | |
from scrapy.exceptions import CloseSpider | |
from scrapy.selector import Selector | |
class MySpider(Spider): | |
name = '' | |
allowed_domains = [ | |
] | |
start_urls = [ |
NewerOlder