Skip to content

Instantly share code, notes, and snippets.

@alexnask
Created May 22, 2012 13:22
Show Gist options
  • Select an option

  • Save alexnask/2769013 to your computer and use it in GitHub Desktop.

Select an option

Save alexnask/2769013 to your computer and use it in GitHub Desktop.
Simple Http class for file grabbing
include time, stdlib
import os/Process
import net/TCPSocket
import threading/Thread
import structs/[ArrayList, HashMap, HashBag, Bag]
import text/json
import io/[File, FileWriter, Reader]
time: extern func(Pointer) -> UInt
srand: extern func(UInt)
rand: extern func -> Int
// Really, REALLY, simple class to handle HTTP requests
Http: class {
// Get the input string of a file from a url (used internally by getContents, getFile)
getStream: static func(url: String, skipHeaders: Bool = true) -> Reader {
// URL format: host/path (No ports, no request schemes, only spaces are URLencoded)
firstSlash := url find("/", 0)
host := url substring(0, firstSlash)
path := url substring(firstSlash) replaceAll(" ", "%20") // Yes, we want the slash in :)
// Start the connection
sock := TCPSocket new(host, 80)
sock connect()
socket := TCPReaderWriterPair new(sock)
// Send out request
socket out write("GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\nAccept-Encoding: identity, *;q=0\r\n\r\n" format(path, host))
// Return the reader stream, after skipping through the headers
if(skipHeaders) socket in skipUntil("\r\n\r")
socket in
}
// Get the contents of a file from the url
getContents: static func(url: String) -> String {
reader := This getStream(url)
data := reader readAll()
reader close()
data
}
// Save a file to a randomly generated name in the dir `prefix` and return the file's path
getFile: static func(prefix, url: String) -> String {
ext := url substring(url findAll(".") last() + 1)
// Get the input stream
reader := This getStream(url, false)
// Parse the headers and get the content-length
header := reader readLine()
length := 0
while(header && !header empty?()) {
if(header toLower() startsWith?("content-length: ")) length = header substring(16) toInt()
header = reader readLine() trim('\r')
}
// Read data of a maximum size of content-length
count := 0
buffer := Buffer new(length)
while(count < length) {
temp := Buffer new(1024) // Read in 1024 byte chunks
count += reader read(temp)
buffer append(temp)
gc_free(temp)
}
// Close the connection
reader close()
// De-allocate unneeded memory
gc_free(reader)
gc_free(header)
// Get a path of a file that does not exist
f: File = null
while((f = File new("%s/%d.%s" format(prefix, rand(), ext))) && f exists?()) {}
if(f) {
// Write our data
FileWriter new(f getPath()) write(buffer) . close()
return f getPath()
}
null
}
}
main: func(args: ArrayList<String>) -> Int {
srand(time(null))
"Starting... Make sure directory `covers` exists and is preferably empty" println()
Http getFile("covers", "www.undisclosedstudio.com/blog/wp-content/uploads/2011/01/whatever-dude-whatever.jpg")
0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment