NPM todays updates

one.clj

(def npm (slurp "https://registry.npmjs.org/-/all/static/today.json"))

(distill* '[cheshire "LATEST"] {:verbose false})

(require '[cheshire.core :as json])

(require '[clojure.walk :refer [keywordize-keys]]
         '[clojure.core.reducers :as r])

(def npm-data
  (->> npm json/parse-string (map keywordize-keys)))

(defn filter-keys [data]
  ;; collect reducer output
  (into [] (r/map #(let [[pkg desc ver dt repo readme tags] %
                         repo (if (not (nil? repo))
                                (-> (string/replace repo #"^git\+" "")
                                    (string/replace #"\.git$" "")
                                    (string/replace #"^https://github" "https://raw.githubusercontent")
                                    (str "/" ver "/" readme))
                                repo)]
                     (hash-map :package pkg
                               :description desc
                               :latest-version ver
                               :datetime-modified dt
                               ;; assumes they all nicely tag their repos, which they probably don't
                               ;; master is another good location
                               :latest-readme repo
                               :readme-filename readme
                               :tagged-keywords tags
                               ))
                  ;; map over map with reducers
                  (r/map (juxt :name :description
                               #(get-in % [:dist-tags :latest])
                               #(get-in % [:time :modified])
                               #(get-in % [:repository :url])
                               :readmeFilename
                               :keywords
                               ) data))))

(def filtered-data
  (-> npm-data filter-keys))


(def tag-frequencies
  (->> (map :tagged-keywords filtered-data)
       (remove nil?) flatten
       frequencies
       (into (sorted-map))
       ))

(defn sort-map-by-vals
  [m]
  (into (sorted-map-by (fn [key1 key2]
                         (compare [(get m key2) key2]
                                  [(get m key1) key1])))
      m))

(sort-map-by-vals tag-frequencies)
(def updated-today (count filtered-data))