Queries Maven Central to build a dependency graph of tracked artifacts

dep-graph-db.clj

(ns dep-graph-db
  "Queries Maven Central to build a dependency graph of tracked artifacts"
  (:require [datomic.api :as d]
            [clojure.java.io :as io]
            [clojure.string :as str]
            [clojure.xml :as xml]
            [clojure.pprint :as pp]))

(def tracked-artifacts
  #{"com.datomic/peer"
    "com.datomic/java-io"
    "com.datomic/ion-dev"
    "com.datomic/local"
    "com.datomic/query-support"
    "com.datomic/client-cloud"
    "com.datomic/client-pro"
    "com.datomic/client"
    "com.datomic/io-stats"
    "com.datomic/client-api"
    "com.datomic/ion"
    "com.datomic/client-impl-shared"
    "com.datomic/memcache-asg-java-client"
    "com.datomic/ion-resolver"
    "com.datomic/query-stats"
    "com.datomic/tools.ops"
    "com.datomic/client.wire-specs"
    "com.datomic/client-impl-cloud"
    "com.datomic/client-impl-pro"
    "com.datomic/clojure-admin-protocols"
    "com.datomic/clojure-client-protocols"
    "com.datomic/s3-access-keys"
    "com.datomic/clj-client"
    "com.datomic/simulant"
    "com.datomic/datomic-lucene-core"
    "com.datomic/ion-http-direct"
    "com.cognitect/nano-impl"
    "com.cognitect/s3-creds"
    "com.cognitect/http-client"
    "com.cognitect/s3-libs"
    "com.cognitect/hmac-authn"
    "com.cognitect/caster"
    "com.cognitect/transit-clj"
    "com.cognitect/transit-java"
    "com.cognitect/transit-cljs"
    "com.cognitect/transit-js"
    "com.cognitect/transcriptor"
    "com.cognitect/anomalies"
    "com.cognitect/cognitect-anomalies"
    "com.cognitect/transducers-js"
    "com.cognitect/transducers-java"})

(def maven-central "https://repo1.maven.org/maven2")
(defn maven-metadata
  "Fetches Maven Central metadata for an artifact. Returns a map:
   {:artifact/id \"com.datomic/client\" :artifact/latest \"1.0.139\" :artifact/deps [...]}"
  [artifact]
  (let [find-tag (fn [node tag]
                   (first (filter #(= (:tag %) tag) (:content node))))
        text (fn [node tag]
               (first (:content (find-tag node tag))))
        [group art] (str/split artifact #"/")
        path (str (str/replace group "." "/") "/" art)
        version (try
                  (let [url (str maven-central "/" path "/maven-metadata.xml")
                        tree (xml/parse (io/input-stream (java.net.URI. url)))]
                    (text (find-tag tree :versioning) :latest))
                  (catch Exception _ nil))
        deps (when version
               (try
                 (let [url (str maven-central "/" path "/" version "/" art "-" version ".pom")
                       tree (xml/parse (io/input-stream (java.net.URI. url)))
                       deps-node (find-tag tree :dependencies)]
                   (->> (some->> deps-node :content (filter #(= (:tag %) :dependency)))
                        (keep (fn [node]
                                (let [a (str (text node :groupId) "/" (text node :artifactId))]
                                  (when (tracked-artifacts a) a))))
                        vec))
                 (catch Exception _ [])))]
    (cond-> {:artifact/id artifact}
      version (assoc :artifact/latest version)
      (seq deps) (assoc :artifact/deps deps))))

(defn build-datoms
  "Turns maven-metadata maps into a set of [entity attribute value] triples.

   Each artifact becomes an entity:
     [1 :artifact/id      \"com.datomic/client\"]
     [1 :artifact/latest  \"1.0.139\"]

   Each dependency edge gets its own entity with two datoms:
     [2 :dep/artifact \"com.cognitect/http-client\"]  — what is depended on
     [2 :dep/parent   1]                              — who depends on it"
  []
  (let [eid (atom 0)
        next-eid #(swap! eid inc)]
    (into #{}
          (mapcat
           (fn [artifact]
             (let [{:artifact/keys [id latest deps]} (maven-metadata artifact)
                   artifact-eid (next-eid)]
               (concat
                ;; the artifact itself
                [[artifact-eid :artifact/id id]]
                (when latest
                  [[artifact-eid :artifact/latest latest]])
                ;; one entity per dependency edge, linking back to the artifact
                (mapcat (fn [dep]
                          (let [dep-eid (next-eid)]
                            [[dep-eid :dep/artifact dep]
                             [dep-eid :dep/parent artifact-eid]]))
                        deps))))
           tracked-artifacts))))

(defonce db (atom nil))

(defn init!
  "Fetch Maven metadata and build the datom set."
  []
  (reset! db (build-datoms)))

(comment
  (require '[dev.nu.morse :as morse]
           '[clojure.main :as main]
           '[datomic.api :as d]
           '[dep-graph-db :as g])
  (morse/launch-in-proc)
  (main/repl
   :eval
   (fn [form]
     (let [v (eval form)]
       (morse/submit form v)
       v)))

  (g/init!)
  g/tracked-artifacts

  ;; what is direct dependcy of an artifact?
  (d/q '[:find ?name
         :in $ ?root
         :where
         [?artifact-eid :artifact/id ?root]
         [?deps :dep/parent ?artifact-eid]
         [?deps :dep/artifact ?name]]
       @g/db "com.datomic/client-impl-shared")
  ;; #{["com.cognitect/hmac-authn"] ["com.cognitect/anomalies"] ["com.cognitect/http-client"] ["com.cognitect/transit-clj"]}

  ;; what is the upstream dependency of this artifact?
  (d/q '[:find ?name
         :in $ ?root
         :where
         [?deps :dep/artifact ?root]
         [?deps :dep/parent ?a]
         [?a :artifact/id ?name]]
       @g/db "com.cognitect/http-client")
  ;; #{["com.datomic/client-impl-shared"] ["com.datomic/client"] ["com.datomic/clj-client"]}

  ;; who depends on it, transitively?
  (d/q '[:find ?who ?version
         :keys artifact published-version
         :in $ % ?root
         :where
         (depends-on ?who ?root)
         [?a :artifact/id ?who]
         [?a :artifact/latest ?version]]
       @g/db
       '[;; base: ?who directly depends on ?lib
         [(depends-on ?who ?lib)
          [?d :dep/artifact ?lib]
          [?d :dep/parent ?a]
          [?a :artifact/id ?who]]
         ;; recursive: ?who depends on something that depends on ?lib
         [(depends-on ?who ?lib)
          (depends-on ?mid ?lib)
          (depends-on ?who ?mid)]]
       "com.cognitect/http-client")
  ;; [{:artifact "com.datomic/client-impl-cloud", :published-version "0.8.35"}
  ;; {:artifact "com.datomic/client", :published-version "1.0.139"}
  ;; {:artifact "com.datomic/client-impl-shared", :published-version "1.0.106"}
  ;; {:artifact "com.datomic/clj-client", :published-version "0.8.606"}
  ;; {:artifact "com.datomic/client-pro", :published-version "1.0.82"}
  ;; {:artifact "com.datomic/local", :published-version "1.0.291"}
  ;; {:artifact "com.datomic/client-cloud", :published-version "1.0.131"}
  ;; {:artifact "com.datomic/client-impl-pro", :published-version "0.8.11"}]
  )