Created
May 22, 2012 13:22
-
-
Save alexnask/2769013 to your computer and use it in GitHub Desktop.
Simple Http class for file grabbing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| include time, stdlib | |
| import os/Process | |
| import net/TCPSocket | |
| import threading/Thread | |
| import structs/[ArrayList, HashMap, HashBag, Bag] | |
| import text/json | |
| import io/[File, FileWriter, Reader] | |
| time: extern func(Pointer) -> UInt | |
| srand: extern func(UInt) | |
| rand: extern func -> Int | |
| // Really, REALLY, simple class to handle HTTP requests | |
| Http: class { | |
| // Get the input string of a file from a url (used internally by getContents, getFile) | |
| getStream: static func(url: String, skipHeaders: Bool = true) -> Reader { | |
| // URL format: host/path (No ports, no request schemes, only spaces are URLencoded) | |
| firstSlash := url find("/", 0) | |
| host := url substring(0, firstSlash) | |
| path := url substring(firstSlash) replaceAll(" ", "%20") // Yes, we want the slash in :) | |
| // Start the connection | |
| sock := TCPSocket new(host, 80) | |
| sock connect() | |
| socket := TCPReaderWriterPair new(sock) | |
| // Send out request | |
| socket out write("GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\nAccept-Encoding: identity, *;q=0\r\n\r\n" format(path, host)) | |
| // Return the reader stream, after skipping through the headers | |
| if(skipHeaders) socket in skipUntil("\r\n\r") | |
| socket in | |
| } | |
| // Get the contents of a file from the url | |
| getContents: static func(url: String) -> String { | |
| reader := This getStream(url) | |
| data := reader readAll() | |
| reader close() | |
| data | |
| } | |
| // Save a file to a randomly generated name in the dir `prefix` and return the file's path | |
| getFile: static func(prefix, url: String) -> String { | |
| ext := url substring(url findAll(".") last() + 1) | |
| // Get the input stream | |
| reader := This getStream(url, false) | |
| // Parse the headers and get the content-length | |
| header := reader readLine() | |
| length := 0 | |
| while(header && !header empty?()) { | |
| if(header toLower() startsWith?("content-length: ")) length = header substring(16) toInt() | |
| header = reader readLine() trim('\r') | |
| } | |
| // Read data of a maximum size of content-length | |
| count := 0 | |
| buffer := Buffer new(length) | |
| while(count < length) { | |
| temp := Buffer new(1024) // Read in 1024 byte chunks | |
| count += reader read(temp) | |
| buffer append(temp) | |
| gc_free(temp) | |
| } | |
| // Close the connection | |
| reader close() | |
| // De-allocate unneeded memory | |
| gc_free(reader) | |
| gc_free(header) | |
| // Get a path of a file that does not exist | |
| f: File = null | |
| while((f = File new("%s/%d.%s" format(prefix, rand(), ext))) && f exists?()) {} | |
| if(f) { | |
| // Write our data | |
| FileWriter new(f getPath()) write(buffer) . close() | |
| return f getPath() | |
| } | |
| null | |
| } | |
| } | |
| main: func(args: ArrayList<String>) -> Int { | |
| srand(time(null)) | |
| "Starting... Make sure directory `covers` exists and is preferably empty" println() | |
| Http getFile("covers", "www.undisclosedstudio.com/blog/wp-content/uploads/2011/01/whatever-dude-whatever.jpg") | |
| 0 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment