JSON to Datascrript

json-to-datascript.cljs

(ns ^:figwheel-hooks jsonviewer.core
  '
  An attempt to convert arbitrary json to datascript by;
  
  1. converting to EDN
  2. walking the EDN to identify cardianlity and ref issue sin the nested data strucutre
  3. creating a new schema from (2)
  4. add db.id s to each nexted map
  5. using updated conn (with new schema), transact the nexted JSON.
  
  This approach works. However, there are still a fewissues:
  
  1. Duplication:  if your nested maps are the same they will still be given uique IDS and transacted. 
     This approach would require deduplication.
  2. File Uploading. Weve been assuming the use of the browser and the current File.Reader upload mechanism is async. 
      my current setup does not proerpyly handle this problem.
  '
  (:require
   [goog.dom :as gdom]
   [cljs.core.async :refer [put! chan <! >!]]
   [clojure.spec.alpha :as s]
   [reagent.core :as reagent :refer [atom]]
   [datascript.core :as d]
   [datascript.transit :as dt]
   [datsync.sync.client :refer [update-schema!]])
   
  (:require-macros 
   [cljs.core.async.macros :refer [go go-loop]]))


;; ----- Datascript DB --------------------------

(defonce conn (d/create-conn {}))


;; ----- EDN processing Functions --------------------------

(defn add-db-ids [data]
  'add :db/id keys to each submap in a dataset'
  (let [atm (atom 1)]
    (clojure.walk/prewalk
     (fn [m]
       (if (map? m)
         (assoc m :db/id (swap! atm inc))
         m))
     data)))

(defn get-data-types [data]
  "determine if nestd data is of tpye k:{} or k:[] 
   in which case we need to use this in our schema"
  (let [cardinality-many (atom #{})
        nested-entities (atom #{})
        test-fn (fn [x]
                  (when (sequential? x)
                    (when-let [[k v] x]
                      (do
                        (when (and (keyword? k) (sequential? v))
                          ;(console.log (str "cardinality: " k " " v " " x))
                          (swap! cardinality-many conj k))
                        (when (and (keyword? k) (map? v))
                          (swap! nested-entities conj k)))))
                  x)]
    (clojure.walk/postwalk test-fn data)
    {:cardinality-many @cardinality-many
     :nested-entities @nested-entities}))


(defn add-edn-to-conn!  [edn conn]
  'take input edn, determine schema, transact data into mutable datascript db, conn'
   (let [edn      (-> edn (dissoc :timings) (dissoc-in))
         schema-types       (get-data-types edn)
         cardinality-schema (into {}
                                  (for [k  (get schema-types :cardinality-many)]
                                    {k {:db/cardinality :db.cardinality/many}}))
         nested-entities     (into {}
                                   (for [k (get schema-types :nested-entities)]
                                     {k {:db/valueType :db.type/ref
                                         :db/cardinality :db.cardinality/many}}))
         schema (merge cardinality-schema nested-entities)
         modified-data (add-db-ids edn)
         nested-keys (get schema-types :nested-entities)]

    ;; update schema
     (console.log (clj->js schema-types))
     (console.log (clj->js schema))
     (update-schema! conn schema)
     
     ;; add data
     (d/transact! conn [modified-data])

    ;; need to dereplicate identical datoms here
    ;; for each nested key:
    ;;   look for duplicates
    ;;   keep top duplicate
    ;;   update entity reference sto first duplicate
    ;;   eliminate all other datoms/ids
    ;(when check-data 
     conn))
  
  
(defn  data-to-edn [e conn]
  "handle the uploaded event data, process it, and log in the db"
  {:pre [d/conn? conn]}
  ;(console.log (-> e .-currentTarget .-result (js/JSON.parse) (js->clj :keywordize-keys true)))
  (let [edn
        (-> e
            .-target
            .-result
            (js/JSON.parse)
            (js->clj :keywordize-keys true))]
     (add-edn-to-conn! edn conn)))

 
(defn put-upload [e] 
  "initiate asyncronous upload" 
  (let [target (.-currentTarget e)
        file   (-> target .-files (aget 0))
        reader (js/FileReader.)]
    
    (set! (.-value target) "")
    (set! (.-onload reader) #(data-to-edn % conn))
    (.readAsText reader file)))


;; input component to allow users to upload file.
(defn input-component []
  [:input {:type "file" :id "file" :accept ".json" :name "file" :on-change put-upload}])


;; ------------------------- 
;; Views

(defn hello-world []
  [:div
   [:h2 "JSON Data in the Browser"]
   [input-component]
   [:div (dt/write-transit-str (:schema @conn))]   
   [:div (dt/write-transit-str @conn)]])   
 
(defn get-app-element []
  (gdom/getElement "app"))

(defn mount [el]
  (reagent/render-component [hello-world] el))

(defn mount-app-element []
  (when-let [el (get-app-element)]
    (mount el)))

;; conditionally start your application based on the presence of an "app" element
;; this is particularly helpful for testing this ns without launching the app
(mount-app-element)

;; specify reload hook with ^;after-load metadata
(defn ^:after-load on-reload []
  (mount-app-element))

I adapted this code per my schematic preferences. For nested objects, it uses :db/isComponent such that they will be pulled out during queries. Perhaps you (or others) will find it useful. Thanks!

defn get-temp-id [root idx]
  ;; cantor pairing function: https://www.cantorsparadise.com/cantor-pairing-function-e213a8a89c2b
  (- (- (+ (/ (* (+ root idx) (+ root idx 1)) 2) idx)) 1))

(defn- json->data-types [root json]
  "determine if nested data is of type k:{} or k:[]
   in which case we need to use this in our schema"
  (let [keys (keys-in json)
        test-fn (fn [{:keys [cardinality-many nested-entities key-map] :as acc} k]
                  (let [v (get-in json k)
                        qualified-key (if (= 1 (count k))
                                        (keyword (name root) (last k))
                                        (apply keyword (take-last 2 k)))
                        assoc-key #(assoc %1 :key-map (assoc key-map k qualified-key))]
                    (cond (map? v)
                          (assoc-key (assoc acc :nested-entities (conj nested-entities qualified-key)))
                          (sequential? v)
                          (assoc-key (assoc acc :cardinality-many (conj cardinality-many qualified-key)))
                          :else
                          acc)))]
    (reduce test-fn {:cardinality-many [] :nested-entities [] :key-map {}} keys)))

(def ^:private collection-schema {:db/cardinality :db.cardinality/many})
(def ^:private nested-obj-schema {:db/valueType :db.type/ref
                                  :db/isComponent true
                                  :db/cardinality :db.cardinality/one})

(defn- json->db-schema [root json]
  (let [{:keys [cardinality-many nested-entities key-map]} (json->data-types root json)
        collections (into {} (for [k cardinality-many] {k collection-schema}))
        nested-entities (into {} (for [k nested-entities] {k nested-obj-schema}))]
    {:schema (merge collections nested-entities)
     :key-map key-map}))

(defn json->db-ingest-operations [connection root json & {:keys [outer-idx start-idx]}]
  (let [{:keys [schema key-map]} (json->db-schema root json)
        ks (keys-in json)
        filtered-keys (filter key-map ks)
        sorted-keys (reverse (sort-by count filtered-keys))
        get-refs (fn [id ref-ids] (mapv #(do [:db/add id (%1 0) (%1 1)]) ref-ids))
        reduce-fn (fn [{:keys [ids res cur-idx] :as acc} key]
                    (let [attr (key-map key)
                          val (get-in json key)
                          schema (schema attr)]
                      (cond
                        (and (= nested-obj-schema schema) (not-empty val))
                        (let [parent (name attr)
                              keys-renamed (specter/transform
                                            [specter/MAP-KEYS]
                                            (partial keyword parent) val)
                              id (get-temp-id outer-idx cur-idx)
                              with-id (merge {:db/id id} keys-renamed)
                              ref-ids (ids key)
                              without-ref-bodies (apply (partial dissoc with-id) (keys ref-ids))
                              refs (get-refs id ref-ids)
                              ids (update ids (pop key) (fnil conj {}) [attr id])]
                          {:ids ids :res (concatv res [without-ref-bodies] refs) :cur-idx (+ 1 cur-idx)})
                        :else
                        acc)))
        {:keys [ids res cur-idx]} (reduce reduce-fn {:ids {} :res [] :cur-idx start-idx} sorted-keys)
        root-id (get-temp-id outer-idx cur-idx)
        with-root (concatv res [{:db/id root-id}]
                           (get-refs root-id (ids [])))]
    (update-schema! connection schema)
    with-root))

Also, pretty sure this does not suffer any "duplication" issue.