bitly-nsq-to-elk-example.go

// Running Instructions
// 1. Get a docker ELK instance up and running - I used https://github.com/deviantony/docker-elk
// 2. Compile this (fill in the right variables before)
// 3. Run this script
// 4. ???
// 5. Go to Kibana to see all the click messages

package main

import (
	"fmt"
	logstash "github.com/heatxsink/go-logstash"
	nsq "github.com/nsqio/go-nsq"
	errors "github.com/pkg/errors"
	"net/url"
	"os"
)

const (
	AccessToken string = "<access_token>" // provided to auth own account
	ApiKey      string = "<api_key>"
	Login       string = "<login>"
	URL         string = "https://api-ssl.bitly.com"
	Topic       string = "<topic>"
	Channel     string = "<channel>"
)

func getAuthToken() (string, error) {
  // should follow the proper Access token fetching workflow if using real (non-dev) OAuth Token
	return AccessToken, nil
}

func main() {
	accessToken, err := getAuthToken()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	cfg := nsq.NewConfig()
	cfg.TlsV1 = true
	cfg.AuthSecret = accessToken
	cfg.MaxInFlight = 1000

	c, newConsumerErr := nsq.NewConsumer(Topic, Channel, cfg)
	if newConsumerErr != nil {
		fmt.Println(errors.Wrap(newConsumerErr, "problem creating new consumer"))
		os.Exit(1)
	}

	c.AddHandler(nsq.HandlerFunc(func(m *nsq.Message) error {
		fmt.Println("Received Message")
		loggerWriteErr := writeToLogstash(string(m.Body))
		if loggerWriteErr != nil {
			fmt.Println(loggerWriteErr)
		}

		m.Finish()
		return nil
	}))

	baseURL := fmt.Sprintf("%s/v3/nsq/lookup?", URL)
	params := url.Values{}

	params.Add("access_token", accessToken)

	finalURL := baseURL + params.Encode()

	c.ConnectToNSQLookupd(finalURL)
	<-c.StopChan
}

func writeToLogstash(message string) error {
	logger := logstash.New("0.0.0.0", 5000, 5)
	_, err := logger.Connect()
	if err != nil {
		return errors.Wrap(err, "Error connecting to logstash")
	}
	loggerWriteErr := logger.Writeln(message)
	if loggerWriteErr != nil {
		return errors.Wrap(loggerWriteErr, "Couldn't write to Logstash")
	}
	return nil
}