namespace_usage.clj

(require 'clojure.pprint 'clojure.set)

(def clojuresphere-top "http://www.clojuresphere.com/api/projects?limit=500&sort=dependents&output=clojure")

(defn exec
  [& shell-tokens]
  (.. Runtime getRuntime (exec (into-array String shell-tokens)) waitFor))

(defn namespace-usage
  [git-path clone-path namespaces]
  (let [projects (binding [*read-eval* false]
                   (-> (slurp clojuresphere-top)
                     read-string
                     :projects))
        projects (for [{{:keys [owner name]} :github :as project} projects
                       :when (and owner name)]
                   {:project project
                    :project-id (symbol owner name)
                    :git-url (str "git://github.com/" owner "/" name ".git")
                    :clone-path (str clone-path java.io.File/separator owner "-" name)})
        projects (take 250 projects)
        queue (java.util.concurrent.LinkedBlockingQueue. projects)
        clone #(future (loop []
                         (when-let [{:keys [project-id git-url clone-path]}
                                    (.poll queue 0 java.util.concurrent.TimeUnit/SECONDS)]
                           (when-not (.exists (java.io.File. clone-path))
                             (println "Cloning" project-id)
                             (exec git-path "clone" git-url clone-path))
                           (recur))))]
    (mapv deref (doall (repeatedly 5 clone)))
    
    (println "Searching for namespace usage...")
    (let [namespace-hits (apply merge-with clojure.set/union
                           (for [ns namespaces
                                 {:keys [project-id clone-path]} projects]
                             (when (zero? (exec "grep" "-Rq" ns clone-path))
                               {ns #{project-id}})))]
      (clojure.pprint/print-table
        (map (fn [[ns projects]] {"Namespace" ns
                                  "Number of top-250 Clojure projects using" (count projects)})
          namespace-hits)))))

usage

(namespace-usage "/path/to/git" "/clone/destination/path"
  #{"clojure.test"
    "expectations"
    "lazytest"
    "midje"})

; ... work being done
|       Namespace | Number of top-250 Clojure projects using |
|-----------------+------------------------------------------|
|    clojure.test |                                      161 |
|    expectations |                                       13 |
|        lazytest |                                       11 |
|           midje |                                       13 |