skyscraper_with_redis.clj

(ns io.codehopper.fun-with-redis
  (:require [taoensso.carmine :as car :refer (wcar)]
            [skyscraper.core :as sky]
            [skyscraper.cache :as sky.cache]
            [reaver]))

(def seed
  [{:url "https://clojuredays.org",
    :processor :page
    :page 2022}])

(def redis {:pool {} :spec {:uri "redis://127.0.0.1:6379"}})

(defn redis-cache
  [redis-conn]
  (reify
    sky.cache/CacheBackend
    (save-blob [_ key blob metadata]
      (car/wcar redis-conn
                (car/set key {:blob blob
                              :meta metadata})))
    (load-blob [_ key]
      (car/wcar redis-conn
                (car/get key)))
    java.io.Closeable
    (close [_]
      nil)))

(sky/defprocessor :page
  :cache-template "dcd/:page"
  :skyscraper.db/columns [:sponsor :year]
  :process-fn (fn [doc ctx]
                (concat
                 (reaver/extract-from doc
                                      "#sponsors a"
                                      [:sponsor :year]
                                      ".sponsor" (reaver/attr :href)
                                      ".sponsor" (constantly (str (:page ctx))))
                 )))

(defn run [cache]
  (sky/scrape! seed
               :html-cache cache
               :db-file "/tmp/dcd.db"
               :parse-fn sky/parse-reaver))

(comment
  (def _x (redis-cache redis))
  (sky.cache/save-blob _x :cache-key "I'm a blob" {:meta "too-meta"})
  (sky.cache/load-blob _x :cache-key)

  (car/wcar redis
            (car/get "dcd/1"))

  (car/wcar redis
            (car/keys "*"))

  (run (redis-cache redis)))