Skip to content

Instantly share code, notes, and snippets.

@hivefans
hivefans / md5-example.go
Last active March 17, 2020 02:03 — forked from sergiotapia/md5-example.go
Golang - How to hash a string using MD5.|-|{"files":{"md5-example.go":{"env":"plain"}},"tag":"bigdata"}
import (
"crypto/md5"
"encoding/hex"
)
func GetMD5Hash(text string) string {
hasher := md5.New()
hasher.Write([]byte(text))
return hex.EncodeToString(hasher.Sum(nil))
}
@hivefans
hivefans / hgetall.go
Last active March 17, 2020 02:03 — forked from vireshas/hgetall.go
hgetall from go using redigo|-|{"files":{"hgetall.go":{"env":"plain"}},"tag":"bigdata"}
values, err := redis.String(c.Do("HGETALL", queue_id, "result"))
if err != nil {
fmt.Println("HGETALL", err)
}
for i := 0; i < len(values); i += 2 {
key, _ := redis.String(values[i], nil)
value, _ := redis.String(values[i+1], nil)
fmt.Println(" %s: %s", key, value)
@hivefans
hivefans / auto_relocate.py
Last active March 17, 2020 02:03 — forked from simonluijk/auto_relocate.py
|-|{"files":{"auto_relocate.py":{"env":"plain"}},"tag":"bigdata"}
#!/usr/bin/env python
import sys
from random import choice
from argparse import ArgumentParser
from pprint import pprint
from elasticsearch import Elasticsearch
def auto_relocate(host='localhost'):
@hivefans
hivefans / access.lua
Last active March 17, 2020 02:03 — forked from mariocesar/access.lua
Nginx Lua script redis based for Basic user authentication|-|{"files":{"vhost.conf":{"env":"plain"},"access.lua":{"env":"plain"}},"tag":"Uncategorized"}
function password_encode(password)
local bcrypt = require 'bcrypt'
return bcrypt.digest(password, 12)
end
function check_password(password, encoded_password)
local bcrypt = require 'bcrypt'
return bcrypt.verify(password, encoded_password)
end
@hivefans
hivefans / authorize.lua
Last active March 17, 2020 02:03 — forked from philwinder/authorize.lua
IVZ: Nginx config for using Lua as the authentication module. You must install nginx with lua support. See "openresty" for linux distros or the vagrant bootstrap shell script.|-|{"files":{"nginx_authorize_by_lua_kibana.conf":{"env":"plain"},"nginx_authorize_by_lua.conf":{"env":"plain"},"htpasswd":{"env":"plain"},"authorize.lua":{"env":"plain"}},…
--[[
Provides custom authorization for nginx.
See the `nginx_authorize_by_lua.conf` for the Nginx config. This lua file is referenced in the config
See testWebserverAccess.sh for unit tests.
To Run nginx (make sure you have the lua, config and htpasswd file):
$ /usr/local/openresty/nginx/sbin/nginx -c /etc/nginx/conf/nginx_authorize_by_lua.conf
Logs are available at: /usr/local/openresty/nginx/logs/lua.log
To write to the log:
@hivefans
hivefans / Korpse.scala
Last active March 17, 2020 02:03 — forked from ariens/Korpse.scala
Kafka Offset Range Provider/Storage Engine|-|{"files":{"KorpseExample.scala":{"env":"plain"},"Korpse.scala":{"env":"plain"}},"tag":"Uncategorized"}
package com.blackberry.bdp.korpse
import scala.collection.parallel.ParIterable
import scala.collection.mutable
import scala.collection.mutable.{ Map => MutableMap }
import org.apache.spark.streaming.kafka.OffsetRange
import kafka.api._
import kafka.api.OffsetResponse
@hivefans
hivefans / Commands.sh
Last active March 17, 2020 02:03 — forked from ottomata/Commands.sh
kafka failed broker tests|-|{"files":{"Commands.sh":{"env":"plain"},"sequence_check.sh":{"env":"plain"},"Results for 10 producers, 10 brokers, 1 failed broker":{"env":"plain"},"sequence_generate.sh":{"env":"plain"}},"tag":"Uncategorized"}
# Asynchronous (batched) producer:
./sequence_generate.sh 20000 10000 | bin/kafka-console-producer.sh --zookeeper analytics1003:2181 --topic test6
# Synchronous producer:
./sequence_generate.sh 20000 10000 | bin/kafka-producer-shell.sh --props config/producer.properties --topic test6
# Consumer
bin/kafka-consumer-shell.sh --props config/consumer.properties --topic test6 | tee -a /tmp/test6.log
# sequence check
@hivefans
hivefans / etchosts.sh
Last active March 17, 2020 02:03 — forked from mikeflynn/etchosts.sh
An /etc/hosts manager bash script (v1.1) -- Added import and export commands!|-|{"files":{"etchosts.sh":{"env":"plain"}},"tag":"bigdata"}
#!/bin/bash
HOSTSFILE="/etc/hosts"
BAKFILE="$HOSTSFILE.bak"
DOMAINREGEX="^[a-zA-Z0-9]{1}[a-zA-Z0-9\.\-]+$"
IPREGEX="^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"
URLREGEX="^https?:\/\/[a-zA-Z0-9]{1}[a-zA-Z0-9\/\.\-]+$"
backup()
{
@hivefans
hivefans / s.sh
Last active March 17, 2020 02:03 — forked from xurenlu/s.sh
ssh机器管理|-|{"files":{"s.sh":{"env":"plain"}},"tag":"bigdata"}
#! /bin/bash
#设置console的颜色
TPUT=/usr/bin/tput
RED=`$TPUT setaf 1`
GREEN=`$TPUT setaf 2`
YELLOW=`$TPUT setaf 3`
NORM=`$TPUT op`
HOSTFILE=~/.sshhosts
#输出红色文字,错误提示用
@hivefans
hivefans / gist:4d1aba8f2e554e95ddc7
Last active March 17, 2020 02:03 — forked from hiroyuki-sato/gist:c3996c03cfb8e554d6c8
Embulk how to |-|{"files":{"gistfile1.md":{"env":"plain"}},"tag":"bigdata"}

移動しました

Embulk

Embulkってなに?ということから、Embulkのプラグインを開発するところまでをまとめたページです。

Embulkとは