clojure-hadoop shuffle component example

step-example.clj

;; hadoop -jar compass.jar -step count-users
;; hadoop -jar compass.jar -flow count-user-flow

(defn map-count-users [k fmap]
  (ctx/increment-counter "Compass" "rows")
  (ctx/increment-counter "Compass" (format "src=%s" (:src (:userinfo fmap))))
  [])

(defn count-users-total [job]
  (job-counter-value job "Compass" "rows"))

(define-step count-users []
  :source (:hbase :users (make-constraints
			  (f/project :columns [[:userinfo :src]])))
  :map map-count-users
  :sink :null
  :reduce :none
  :post-hook count-users-total)

(define-flow count-user-flow []
  (do-step count-users))