shiweifu

mime = {}

local mime_table = 
{
				[".html"] = {
								["mime"] = "text/html",
								["bin"] = false
							},
				[".xml"] = {
							["mime"] = "application/xml",

shiweifu

socket = require("socket")
httpcode = require("httpcode")
require("mime")
getMime = mime.getMime

assert(getMime)


function main( p )
	local port

shiweifu

class FileWrapper:
    """Wrapper to convert file-like objects to iterables"""

    def __init__(self, filelike, blksize=8192):
        self.filelike = filelike
        self.blksize = blksize
        if hasattr(filelike,'close'):
            self.close = filelike.close

shiweifu

# -*- coding: UTF-8 -*-
import httplib
import urllib2
import socket
import traceback
import re
import BeautifulSoup
import threading
from threadpool import ThreadPool
from Queue import Queue, Empty

shiweifu

def down_page_callable(**kw):
    try:
        _task_queue = kw["task_queue"]
        _url_list = kw["url_list"]
        _log_queue = kw["log_queue"]
        _deeph = kw["deeph"] 
        _filter_exts = kw["filter_exts"],
        _root = kw["root"]
    except KeyError, ke: