pingles

package main

import (
	"github.com/pingles/go-metrics"
	"github.com/pingles/go-metrics/riemann"
	"log"
	"net"
	"time"
)

pingles

(ns async.example
  (:require [clojure.core.async :refer (chan go-loop <! >!)])
  (:import [java.util Date])
  (:gen-class))

(defn producer-process [channel]
  (go-loop []
    (>! channel (str "Hello, the time is " (Date.)))
    (Thread/sleep 500)
    (recur)))

pingles

$ lein uberjar
$ java -Djava.io.tmpdir=/mnt/bifrost-tmp \
       -Dlogback.configurationFile=/opt/uswitch/bifrost/etc/logback.xml \
       -server \
       -jar /opt/uswitch/bifrost/lib/*-standalone.jar \
       --config /opt/uswitch/bifrost/etc/config.edn

pingles

{:consumer-properties {"zookeeper.connect"  "localhost:2181"
                       "group.id"           "bifrost"
                       "auto.offset.reset"  "smallest" ; we explicitly commit offsets once files have
                                                       ; been uploaded to s3 so no need for auto commit
                       "auto.commit.enable" "false"}
 :topic-blacklist     #{"sometopic"} ; topics from :topic-blacklist will not be backed up
 :topic-whitelist     nil ; if :topic-whitelist is defined, only topics
                          ; from the whitelist will be backed up. The
                          ; value should be a set of strings.
 :rotation-interval   60000 ; milliseconds

pingles

{:table        "testing"
 :pk-columns   ["foo"]
 :columns      ["foo" "bar"]
 :jdbc-url     "jdbc:postgresql://foo.eu-west-1.redshift.amazonaws.com:5439/db?tcpKeepAlive=true&user=user&password=pwd"
 :options      ["DELIMITER '\\t'" "IGNOREHEADER 1" "ESCAPE" "TRIMBLANKS"]
 :data-pattern ".*tsv$"}

pingles

bucket
  ├── directory-a
  │   └── foo
  │       └── manifest.edn
  │       └── 0001.tsv
  │       └── 0002.tsv
  └── directory-b
      └── manifest.edn

pingles

{:s3 {:credentials   {:access-key ""
                      :secret-key ""}
      :bucket        "blueshift-data"
      :key-pattern   ".*"
      :poll-interval {:seconds 300}}
 :telemetry {:reporters [uswitch.blueshift.telemetry/log-metrics-reporter]}}

pingles

# you can run directly with leiningen:
$ lein run -- --config ./etc/config.edn

# or, build an uberjar
$ lein uberjar
$ java -Dlogback.configurationFile=./etc/logback.xml -cp "target/*-standalone.jar" uswitch.blueshift.main --config ./etc/config.edn

pingles

(let [ostream (ByteArrayOutputStream. 100)
      write (baldr.core/baldr-writer ostream)]
  (write (byte-array [1 2 3]))
  (write (byte-array [1 2 3 4]))
  (.close ostream)

  ;; now we can create a baldr-seq over the input stream
  ;; containing our 2 previously written records
  (baldr.core/baldr-seq (ByteArrayInputStream. (.toByteArray ostream))))

pingles

(reduce + [1 2 3])  ; this is nice and short
 
; this is a bit longer so break on the 2nd arg
(do-something-longer with-argument-1
                     (and argument-2
                          argument-3))