Simple Logical gates : forward and backward pass in neuron function (x, y) = sigmoid(a*x + b*y + c)

neural.clj

;; a single unit has forward value and backward gradient.
(defrecord Unit 
    [value gradient])

(defrecord Gate
    [^:Unit input-a ^:Unit input-b])

(defprotocol GateOps
  (forward [this])
  (backward [this back-grad]))

(extend-protocol GateOps
  Unit
  (forward [this]
    (-> this :value))
  (backward [this back-grad]
    ({this back-grad})))

(defrecord MultiplyGate [input-a input-b]
  GateOps
  (forward [this]
    (* (forward (-> this :input-a)) (forward (:input-b this))))

  (backward [this back-grad]
    (let [val-a (forward (-> this :input-a))
          val-b (forward (-> this :input-b))
          input-a (:input-a this)
          input-b (:input-b this)]
      (merge-with + (backward input-a (* val-b back-grad))
                    (backward input-b (* val-a back-grad))))))

(defrecord AddGate [input-a input-b]
  GateOps
  (forward [this]
    (+ (forward (:input-a this)) (forward (:input-b this))))

  (backward [this back-grad]
    (let [val-a (forward (-> this :input-a))
          val-b (forward (-> this :input-b))
          input-a (:input-a this)
          input-b (:input-b this)]
      (merge-with + (backward input-a (* 1.0 back-grad))
                    (backward input-b (* 1.0 back-grad))))))


(defn sig [x]
  (/ 1 (+ 1 (Math/pow Math/E (- x)))))

(defrecord SigmoidGate [gate]
  GateOps
  (forward [this]
    (sig (forward (:gate this))))

  (backward [this back-grad]
    (let [s (forward (:gate this))
          ds (* s (- 1 s))]
      (backward (:gate this) ds))))

should have written

user> (neurals.core/backward neurals.core/sigaxcby 1.0)