gistfile1.txt

(ns foo.proxy
  (:require [hickory.core :as hickory]
            [hickory.select :as select]
            [clojure.tools.logging :as log]
            [clj-http.client :as client]))

(defonce proxies (atom []))

(defn get-proxies []
  @proxies)

(def url-page "https://www.sslproxies.org/")

(def row-selector
  (select/descendant
   (select/id :proxylisttable)
   (select/tag :tr)))

(defn parse-tree []
  (-> url-page client/get :body hickory.core/parse hickory.core/as-hickory))

(defn parse-proxy [tr]
  [(-> tr :content first :content first)
   (-> tr :content second :content first Integer/parseInt)])

(def remove-nil (partial remove nil?))

(defn fetch-proxies []
  (let [tree (parse-tree)
        trs (select/select row-selector tree)
        trs (-> trs rest butlast)]
    (vec
     (remove-nil
      (for [tr trs]
        (try
          (parse-proxy tr)
          (catch Exception e)))))))

(defn update-proxes []
  (reset! proxies (fetch-proxies))
  true)

(defonce worker (atom nil))

(defn ^String exc->str
  "Returns a message string for an exception instance."
  [^Exception e]
  (let [class (-> e .getClass .getCanonicalName)
        message (-> e .getMessage (or "<no message>"))]
    (format "%s: %s" class message)))

(defn worker-beat []
  (while true
    (try
      (log/info "Updating proxies...")
      (update-proxes)
      (log/infof "Proxies updated, count: %s" (count (get-proxies)))
      (catch Exception e
        (log/errorf "Error while updating proxies: %s" (exc->str e)))
      (finally
        (Thread/sleep (* 1000 60 1))))))

(defn worker-start []
  (log/info "Starting proxies worker")
  (reset! worker (future (worker-beat))))

(defn worker-stop []
  (log/info "Stopping proxies worker")
  (when-let [f @worker]
    (future-cancel f)))