Skip to content

Instantly share code, notes, and snippets.

@kzar
Created October 25, 2011 12:14
Show Gist options
  • Save kzar/1312517 to your computer and use it in GitHub Desktop.
Save kzar/1312517 to your computer and use it in GitHub Desktop.
Clojure scraper - java.lang.OutOfMemoryError: Java heap space
(ns logs.views.ajax
(:import java.net.URL)
(:require [logs.views.common :as common]
[noir.content.pages :as pages]
[net.cgrand.enlive-html :as html]
[clj-time.core :as time]
[clj-time.format :as time-format])
(:use [clojure.string :only [trim]]
[noir.core]
[hiccup.core]
[hiccup.page-helpers]))
(def log-url "http://*****************")
(def ericsson-date-format (time-format/formatter "yyyy-MM-dd' at 'HH:mm:ss"))
(def ericsson-timezone (time/time-zone-for-offset -5))
(defn fetch [url]
(html/html-resource (java.net.URL. url)))
(defn rows [resource]
(html/select resource [:tr]))
(defn parse-date [date-string]
(let [dt (-> (time-format/parse ericsson-date-format date-string)
(time/from-time-zone ericsson-timezone))]
(time-format/unparse ericsson-date-format dt)))
(defn parse-row [row]
(let [row (map (comp trim html/text) (html/select row [:td]))]
(conj (rest row) (parse-date (first row)))))
(defn fetch-logs []
(map parse-row (rest (rows (fetch log-url)))))
(defpage [:get "/ajax"] {}
(let [log-entries (fetch-logs)]
(if (empty? log-entries)
(common/json-response 503 "Unavaliable")
(common/json-response 200 {:aaData log-entries}))))
(ns logs.views.common
(:use [clojure.data.json :only [json-str]]))
(defn json-response [code m]
{:status code
:headers {"Content-Type" "application/json"}
:body (json-str m)})
java.lang.OutOfMemoryError: Java heap space
at sun.net.www.http.ChunkedInputStream.processRaw(ChunkedInputStream.java:336)
at sun.net.www.http.ChunkedInputStream.readAheadNonBlocking(ChunkedInputStream.java:493)
at sun.net.www.http.ChunkedInputStream.readAhead(ChunkedInputStream.java:584)
at sun.net.www.http.ChunkedInputStream.hurry(ChunkedInputStream.java:741)
at sun.net.www.http.ChunkedInputStream.closeUnderlying(ChunkedInputStream.java:198)
at sun.net.www.http.ChunkedInputStream.close(ChunkedInputStream.java:722)
at java.io.FilterInputStream.close(FilterInputStream.java:155)
at sun.net.www.protocol.http.HttpURLConnection$HttpInputStream.close(HttpURLConnection.java:2743)
at net.cgrand.enlive_html$load_html_resource$fn__2545.invoke(enlive_html.clj:53)
at net.cgrand.enlive_html$load_html_resource.invoke(enlive_html.clj:53)
at net.cgrand.enlive_html$fn__2569.invoke(enlive_html.clj:102)
at clojure.lang.MultiFn.invoke(MultiFn.java:167)
at net.cgrand.enlive_html$html_resource.invoke(enlive_html.clj:69)
at logs.views.ajax$fetch.invoke(ajax.clj:18)
at logs.views.ajax$fetch_logs.invoke(ajax.clj:33)
at logs.views.ajax$GET__ajax.invoke(ajax.clj:36)
at logs.views.ajax$fn__3213.invoke(ajax.clj:35)
at compojure.core$if_route$fn__296.invoke(core.clj:39)
at compojure.core$if_method$fn__289.invoke(core.clj:24)
at compojure.core$routing$fn__311.invoke(core.clj:98)
at clojure.core$some.invoke(core.clj:2388)
at compojure.core$routing.doInvoke(core.clj:98)
at clojure.lang.RestFn.applyTo(RestFn.java:139)
at clojure.core$apply.invoke(core.clj:602)
at compojure.core$routes$fn__315.invoke(core.clj:103)
at noir.server.handler$init_routes$fn__2259.invoke(handler.clj:48)
at noir.request$wrap_request_map$fn__2200.invoke(request.clj:14)
at ring.middleware.keyword_params$wrap_keyword_params$fn__441.invoke(keyword_params.clj:21)
at ring.middleware.nested_params$wrap_nested_params$fn__478.invoke(nested_params.clj:64)
at ring.middleware.params$wrap_params$fn__418.invoke(params.clj:76)
at ring.middleware.multipart_params$wrap_multipart_params$fn__511.invoke(multipart_params.clj:102)
at ring.middleware.session$wrap_session$fn__666.invoke(session.clj:40)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment