clarksun

# coding=utf-8
# https://github.com/GuozhuHe/webspider/blob/master/webspider/utils/cache.py
import logging
import pickle
from functools import wraps

import redis

from common import config

clarksun

# https://github.com/GuozhuHe/webspider/blob/master/webspider/utils/http_tools.py
def get_proxys(pages=4):
  """获取代理"""
  proxy_list = []
  url = 'http://www.xicidaili.com/wn/'
  headers = generate_http_header()
  headers.update(
    {
      'Referer': 'http://www.xicidaili.com/wn/',
      'Host': 'www.xicidaili.com',

clarksun

# tornado.util.ObjectDict
# Makes a dictionary behave like an object, with attribute-style access.
import typing
_ObjectDictBase = typing.Dict[str, typing.Any]

class ObjectDict(_ObjectDictBase):
  """Makes a dictionary behave like an object, with attribute-style access.
  """
  def __getattr__(self, name):
    # type: (str) -> Any

clarksun

# -*- coding: utf-8 -*-
"""
*batch*
/root/.virtualenvs/amazon/bin/python -m amazon.frontier.worker.db --config amazon.frontier.settings.prod.hbase.workersettings --no-incoming --no-scoring

*incoming*
/root/.virtualenvs/amazon/bin/python -m amazon.frontier.worker.db --config amazon.frontier.settings.prod.hbase.workersettings --no-batches --no-scoring --partition-id 0
/root/.virtualenvs/amazon/bin/python -m amazon.frontier.worker.db --config amazon.frontier.settings.prod.hbase.workersettings --no-batches --no-scoring --partition-id 1
/root/.virtualenvs/amazon/bin/python -m amazon.frontier.worker.db --config amazon.frontier.settings.prod.hbase.workersettings --no-batches --no-scoring --partition-id 2
/root/.virtualenvs/amazon/bin/python -m amazon.frontier.worker.db --config amazon.frontier.settings.prod.hbase.workersettings --no-batches --no-scoring --partition-id 3

clarksun

import os
import time
import string
import pickle

from operator import itemgetter

from nltk.corpus import stopwords as sw
from nltk.corpus import wordnet as wn
from nltk import wordpunct_tokenize

clarksun

Keybase proof

I hereby claim:

• I am clarksun on github.
• I am sunwei (https://keybase.io/sunwei) on keybase.
• I have a public key whose fingerprint is F950 1F84 6804 11EC 6E3E 96EC 4EAA 26A7 9973 6081

To claim this, I am signing this object:

clarksun

import asyncio


def call_periodic(interval, callback, *args, **kwargs):
    # get loop as a kwarg or take the default one
    loop = kwargs.get('loop') or asyncio.get_event_loop()
    # record the loop's time when call_periodic was called
    start = loop.time()

    def run(handle):

clarksun

#!/usr/bin/python

import os
import sys
import boto3

# get an access token, local (from) directory, and S3 (to) directory
# from the command-line
local_directory, bucket, destination = sys.argv[1:4]

clarksun

# /Users/sunwei/.config/tmuxinator/mkdocs.yml
name: mkdocs-<%= @args[0] %>
root: /Users/sunwei/MyLib/practices/mkdocsP
on_project_start: mkdocs new <%= @args[0] %>
windows:
  - mkdocs:
    layout: main-vertical
    panes:
      - list:
        - cd <%= @args[0] %>

clarksun

So you want to create a pex that packages your script and its dependencies?

Ok - first to make our script! Call it foo.py:

import requests

if __name__ == '__main__':
  req = requests.get("https://raw.githubusercontent.com/pantsbuild/pex/master/README.rst")
  print req.text.split("\n")[0]