nlacasse

{"name":"opal.io","hostname":"opal.io","pid":1517,"component":"bullhorn","level":30,"response":{"req":{"domain":null,"_events":{},"_maxListeners":10,"output":[],"outputEncodings":[],"writable":true,"_last":false,"chunkedEncoding":false,"shouldKeepAlive":true,"useChunkedEncodingByDefault":true,"sendDate":false,"_hasBody":true,"_trailer":"","finished":true,"agent":{"domain":null,"_events":{},"_maxListeners":10,"options":{},"requests":{},"sockets":{},"maxSockets":5},"method":"POST","path":"/1/push","_headers":{"host":"api.parse.com","accept-encoding":"gzip, deflate","cookie":"","x-parse-application-id":"2MjUjBT5YdWcK0R1lbMs46U5tgB9VLeCO59F5sWZ","x-parse-rest-api-key":"xTufL3jqNsx7jd5nvuDzo3yxnnPNpPdjM0WB3HQA","content-type":"application/json","content-length":257},"_headerNames":{"host":"Host","accept-encoding":"Accept-Encoding","cookie":"Cookie","x-parse-application-id":"X-Parse-Application-Id","x-parse-rest-api-key":"X-Parse-REST-API-Key","content-type":"content-type","content-length":"Content-Length"},"_heade

nlacasse

{
   "venue" : {
      "node-id" : 22222,
      "feedback-support" : 1.5
   },
   "friends-of-friends" : [
      {
         "fb-event-coattendance" : 0,
         "fb-name" : "Rand",
         "fb-mentions-interactions" : 19,

nlacasse

Mar 6, 2013 12:45:33 AM com.sun.jersey.spi.container.ContainerResponse mapMappableContainerException
SEVERE: The RuntimeException could not be mapped to a response, re-throwing to the HTTP container
org.neo4j.kernel.DeadlockDetectedException: Don't panic.
        
A deadlock scenario has been detected and avoided. This means that two or more transactions, which were holding locks, were wanting to await locks held by one another, which would have resulted in a deadlock between these transactions. This exception was thrown instead of ending up in that deadlock.
        
See the deadlock section in the Neo4j manual for how to avoid this: http://docs.neo4j.org/chunked/stable/transactions-deadlocks.html
        
Details: 'Transaction(1687136, owner:"1519904647@qtp-2139633649-941")[STATUS_ACTIVE,Resources=1] can't wait on resource RWLock[Relationship[85805897]] since => Transaction(1687136, owner:"1519904647@qtp-2139633649-941")[STATUS_ACTIVE,Resources=1] <-[:HELD_BY]- RWLock[Node[35079086]] <-[:WAITING_FOR]- Trans

nlacasse

    ;; Connect retweet
    (when-let [retweeted-status (:retweeted_status status)]
      (let [retweeted-status-node (upsert-status! retweeted-status)]
        (upconnect-entity-nodes! retweeted-status-node status-node "retweet"))
      ;; Retweeted from user
      (if-let [retweeted-user (:user retweeted-status)]
        (let [retweeted-user-node (upsert-user! retweeted-user)]
          (upconnect-entity-nodes! status-node retweeted-user-node "retweeted-from-user"))))

nlacasse

    ;; Connect retweet
    (when-let [retweeted-status (:retweeted_status status)]
      (let [retweeted-status-node (upsert-status! retweeted-status)]
        (upconnect-entity-nodes! retweeted-status-node status-node "retweet"))
      ;; Retweeted from user
      (if-let [retweeted-user (:user retweeted-status)]
        (let [retweeted-user-node (upsert-user! retweeted-user)]
          (upconnect-entity-nodes! status-node retweeted-user-node "retweeted-from-user"))))

nlacasse

1360880797.773173 "sismember" "feeds" "newuser"
1360880797.773992 "MULTI"
1360880797.774022 "lpush" "feed.ids:newuser" "d851f290-d8ce-4f63-ad3c-f95f401f5bdf"
1360880797.774038 "incr" "feed.publishes:newuser"
1360880797.774052 "hset" "feed.items:newuser" "d851f290-d8ce-4f63-ad3c-f95f401f5bdf" "{\"uuid\":\"fb3760fa-a6a5-4e89-ac32-c393bab95b2a\",\"name\":\"Okie Kupid\",\"emails\":[{\"value\":\"[email protected]\",\"valid\":false}],\"services\":[{\"service\":\"facebook\",\"service_id\":\"100004601530540\",\"token\":\"AAAHCBLxw6KgBAFoSnCoqWHc4vlRlqJ7PbKdUhZCZAuq4nkbYqX8OGW7yCjDb1k4OipPYXTDgB5wmjjuBUx99rlUA8GgACvObWcPjjh0QZDZD\",\"secret\":null,\"expires\":\"2013-04-15T22:25:18.591Z\",\"permissions\":{\"email\":true,\"read_stream\":true,\"user_relationships\":true,\"user_location\":true,\"user_likes\":true,\"user_activities\":true,\"user_interests\":true,\"user_events\":true,\"user_photos\":true,\"user_status\":true,\"friends_relationships\":true,\"friends_location\":true,\"friends_likes\":true,\"friends_acti

nlacasse

(defn insert-tuple!
  [inserter tuple]
  ; Get the resource and label names out of the tuple.
  (let [[resource-1 label resource-2 & _ ] tuple
        ; Upsert the resource nodes.
        node-1 (insert-resource-node! inserter resource-1)
        node-2 (insert-resource-node! inserter resource-2)]
    ; Connect the nodes with an edge.
    (connect-resource-nodes! inserter node-1 node-2 label)))

nlacasse

(ns opal.dbpedia
  (:use [clojure.tools.logging :only [log]])
  (:require [clojure.java.io :as io])
  (:import [uk.ac.manchester.cs.owl.owlapi.turtle.parser TurtleParser]
           [org.neo4j.unsafe.batchinsert BatchInserters
                                         LuceneBatchInserterIndexProvider]
           [org.neo4j.graphdb DynamicRelationshipType]))

;; PARSING METHODS

nlacasse

(defn -main [graph-path & files]
  (let [inserter (BatchInserters/inserter graph-path)]
    (doseq [file files]
      (log :debug (str "Loading file: " file))
      (let [c (atom 0)]
        (doseq [tuple (parse-file file)]
          (if (= (mod @c 10000) 0)
            (log :debug (str file ": " @c)))
          (swap! c inc)
          (insert-tuple! inserter tuple))))

nlacasse

(defn insert-tuple!
  [inserter tuple]
  ; Get the resource and label names out of the tuple.
  (let [[resource-1 label resource-2 & _ ] tuple
        ; Upsert the resource nodes.
        node-1 (insert-resource-node! inserter resource-1)
        node-2 (insert-resource-node! inserter resource-2)]
    ; Connect the nodes with an edge.
    (connect-resource-nodes! inserter node-1 node-2 label)))