ClojureScript env-file parsing experiments

env-file

PLAIN=plain_value

DQUOTE1="a value" # ignored comment
DQUOTE2="val1"" val2 " # ignored comment
SQUOTE1='a value' # ignored comment
SQUOTE2='val1'' val2 ' # ignored comment

SHOULD_NOT_BE_PRESENT="value" true

HAS_EQ=value_has_=_sign

# A comment PLAIN=invalid
INSIDE_HASH1="before hash # after hash" # comment after
INSIDE_HASH2='before hash # after hash' # comment after
INSIDE_HASH3="before hash "#not_a_comment
INSIDE_HASH4='before hash '#not_a_comment

ADD_TO_PLAIN1=${ADD_TO_PLAIN1:-prefix-${PLAIN}-suffix}
ADD_TO_PLAIN2=${ADD_TO_PLAIN2:-prefix-$PLAIN-suffix}

HOME_BASED=${HOME_BASED:-${HOME:-/home/nonexistentuser}}

APPEND_EMPTY1="${APPEND_EMPTY1} ${PLAIN}"
APPEND_EMPTY1="${APPEND_EMPTY1} two"
APPEND_EMPTY1="${APPEND_EMPTY1} three"

APPEND_EMPTY2="$APPEND_EMPTY2 $PLAIN"
APPEND_EMPTY2="$APPEND_EMPTY2 two"
APPEND_EMPTY2="$APPEND_EMPTY2 three"

DEEP=${FOO:-abc${FOO:-foo}def${BAR:-A${BAZ:-baz}Z}ghi}

env-file.cljs

(ns env-file-experiment
  (:require [clojure.string :as S]
            [clojure.pprint :refer [pprint]]
            [promesa.core :as P]
            [clojure.set :refer [difference]]
            ["fs/promises" :as fs]
            ["child_process" :as child-process]
            ["util" :refer [promisify]]))

;;
;; Interpolate functions
;;
(defn match-group
  "Find the outermost sequence that start with left and ends with
  right while keeping track of embedded matches/depth. 
  Example: (match-group \"a{b{c}d}e\" \"{\" \"}\") -> [\"a\" \"b{c}d\" \"e\"]
  Returns: [before inside after]"
  [s left right]
  (let [start (S/index-of s left)
        [lcnt rcnt] [(count left) (count right)]]
    (if (nil? start)
      nil
      (loop [depth 1
             i (+ start lcnt)]
        (cond
          (= 0 depth) [(subs s 0 start) (subs s (+ start lcnt) (dec i)) (subs s i)]
          (>= i (count s)) nil
          (= (subs s i (+ i lcnt)) left)  (recur (inc depth) (+ i lcnt))
          (= (subs s i (+ i rcnt)) right) (recur (dec depth) (+ i rcnt))
          :else (recur depth (inc i)))))))


(def EXPANSION-RE #"^([a-zA-Z_][a-zA-Z0-9_]*)(:?[-+])?(.*)$")

(defn resolve-expansion
  "Resolves the shell style variable expansion expression."
  [expr env]
  (let [[_ var-name op word] (re-matches EXPANSION-RE expr)
        value  (get env var-name)
        is-set (contains? env var-name)
        non-empty (not-empty value)]
    (prn :expr expr :op op :word word)
    (condp = op
      ":-"  (if non-empty value word)
      "-"   (if is-set    value word)
      ":+"  (if non-empty word "")
      "+"   (if is-set    word "")
      ":?"  (if non-empty value (throw (js/Error. word)))
      "?"   (if is-set    value (throw (js/Error. word)))
            (if is-set    value ""))))

(defn interpolate
  "Recursively interpolates env variables in a string with support for
  defaults, alternatives, and nested expansions."
  [s env]
  (loop [result ""
         s      s]
    (if-let [[pre expr post] (match-group s "${" "}")]
      (let [expanded (interpolate (resolve-expansion expr env) env)]
        (recur (str result pre expanded) post))
      (str result s))))

(defn load-env
  [src start-env]
  (let [vars (->> (S/split-lines src)
                  (map S/trim)
                  (map #(S/replace % #"(^| )#.*$" ""))
                  (filter not-empty)
                  (map #(first (re-seq #"([^=]*)=(?:\"(.*)\"|'(.*)'|(.*))" %)))
                  (filter not-empty)
                  (reduce (fn [m [_ k v1 v2 v3]] (conj m [k (or v1 v2 v3)])) []))]
    (loop [env {}
           vars vars]
      (if (empty? vars)
        env
        (let [[[k v] & vars] vars
              lookup-env (merge start-env env)]
          (prn :k k :v v :-> (interpolate v lookup-env))
          (recur (assoc env k (interpolate v lookup-env)) vars))))))

(def exec-promise (promisify (.-exec child-process)))

(defn load-env-bash [path]
  "Source path (using bash) and return a map of modified env variables.
  Requires bash, sort, and comm on the PATH."
  [path]
  (let [bash-cmd (str "bash -c '_AAAAA=$(declare -x | sort);"
                      "set -a; source \"" path "\";"
                      "_ZZZZZ=$(declare -x | sort);"
                      "comm -13 <(echo \"${_AAAAA}\") <(echo \"${_ZZZZZ}\")"
                      "  | sed \"s/^declare -x //\"'")]
    (P/let [result (exec-promise bash-cmd)]
      (->> (S/split-lines (.-stdout result))
           (map #(first (re-seq #"([^=]*)=\"(.*)\"" %)))
           (reduce (fn [m [_ k v]] (assoc m k v)) {})))))

;;
;; Testing
;;

(defn exec
  "Executes a shell command with optional options and returns a Clojure map of the result."
  [cmd & [opts]]
  (P/let [opts (merge {:encoding "utf8" :stdio "pipe"} opts)
          res  (exec-promise cmd (clj->js opts))]
    (js->clj res)))

(defn do-test
  "Asynchronously tests the substitute function by comparing its output to Bash's parameter expansion."
  [s env]
  (P/let [env-vars (->> env
                        (map (fn [[k v]] (str k "=\"" v "\"")))
                        (S/join " "))
          escaped-s (-> s
                        (S/replace "\\" "\\\\")
                        (S/replace "\"" "\\\""))
          command   (str env-vars " bash -c 'echo [\"" escaped-s "\"]'")
          {:strs [stdout stderr]} (exec command)]
    (if (not (S/blank? stderr))
      (println "Error executing Bash:" stderr)
      (let [expected     (S/trim stdout)
            substitution (interpolate s env)
            result       (str "[" substitution "]")
            passed       (= result expected)]
        (println passed "result:" result "expected:" expected "str:" s "envVars:" env-vars)))))

;; ================================
;; Running Tests
;; ================================

(defn test-interpolate
  "Runs all substitution tests."
  []
  (let [tests [["abc${FOO:-XYZ}ghi" {}]
               ["abc${FOO:-XYZ}ghi}" {}]
               ["abc${FOO:-XYZ}ghi" {}]
               ["abc${FOO-XYZ}ghi" {}]
               ["abc${FOO:-XYZ}ghi" {"FOO" ""}]
               ["abc${FOO-XYZ}ghi" {"FOO" ""}]
               ["${FOO:-abc${FOO:-foo}def${BAR:-A${BAZ:-baz}Z}ghi}" {}]
               ["${FOO:-abc${FOO:-foo}def${BAR:-A${BAZ:-baz}Z}ghi}" {"FOO" "123"}]
               ["${FOO:-abc${FOO:-foo}def${BAR:-A${BAZ:-baz}Z}ghi}" {"BAR" "123"}]
               ["${FOO:-abc${FOO:-foo}def${BAR:-A${BAZ:-baz}Z}ghi}" {"BAZ" "123"}]
               ["abc${FOO:-foo}def${BAR:-A${BAZ:-baz}Z}ghi" {"FOO" "foo" "BAR" "AQUXZ" "BAZ" "qux"}]
               ["abc${FOO:-foo}def${BAR:-A${BAZ:-baz}Z}ghi" {"BAR" "BAR_VALUE" "BAZ" "BAZ_VALUE"}]
               ["abc${FOO:-foo}def" {"FOO" "X}YZ"}]
               ["${FOO:+bar}" {}]
               ["${FOO+bar}" {}]
               ["${FOO:+bar}" {"FOO" "value"}]
               ["${FOO+bar}" {"FOO" "value"}]
               ["${FOO:+bar}" {"FOO" ""}]
               ["${FOO+bar}" {"FOO" ""}] ]]
    (P/doseq [[s env] tests]
      (do-test s env))))

;; ================================
;; Execute Main
;; ================================

#_(test-interpolate)

(defn js->map [obj]
  (into {} (map js->clj (js/Object.entries obj))))

(P/let [path "dctest/test/env-file"
        raw (fs/readFile path)
        env (js->map js/process.env)
        res (load-env raw env)
        res-bash (load-env-bash path env)
        drops (difference (set (keys res-bash)) (set (keys res)))
        adds (difference (set (keys res)) (set (keys res-bash)))]
  (println "Should not be present:" adds)
  (println "Missing:" drops)
  (println)
  (doseq [[k v] (sort res)]
    (if (= v (get res-bash k))
      (println (str "Match:" k "=\"" v "\""))
      (println (str "Mis-Match:" k "=\"" v "\" (should be \"" (get res-bash k) "\")")))))