This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# herein we backup our indexes! this script should run at like 6pm or something, after logstash | |
# rotates to a new ES index and theres no new data coming in to the old one. we grab metadatas, | |
# compress the data files, create a restore script, and push it all up to S3. | |
TODAY=`date +"%Y.%m.%d"` | |
INDEXNAME="logstash-$TODAY" # this had better match the index name in ES | |
INDEXDIR="/usr/local/elasticsearch/data/logstash/nodes/0/indices/" | |
BACKUPCMD="/usr/local/backupTools/s3cmd --config=/usr/local/backupTools/s3cfg put" | |
BACKUPDIR="/mnt/es-backups/" | |
YEARMONTH=`date +"%Y-%m"` |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# node.js using PPA (for statsd) | |
sudo apt-get install python-software-properties | |
sudo apt-add-repository ppa:chris-lea/node.js | |
sudo apt-get update | |
sudo apt-get install nodejs npm | |
# Install git to get statsd | |
sudo apt-get install git |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
DEV_LOGGING = { | |
'version': 1, | |
'disable_existing_loggers': False, | |
'formatters': { | |
'basic': { | |
'format': '%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', | |
} | |
}, | |
'handlers': { | |
'console': { |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Latency Comparison Numbers (~2012) | |
---------------------------------- | |
L1 cache reference 0.5 ns | |
Branch mispredict 5 ns | |
L2 cache reference 7 ns 14x L1 cache | |
Mutex lock/unlock 25 ns | |
Main memory reference 100 ns 20x L2 cache, 200x L1 cache | |
Compress 1K bytes with Zippy 3,000 ns 3 us | |
Send 1K bytes over 1 Gbps network 10,000 ns 10 us | |
Read 4K randomly from SSD* 150,000 ns 150 us ~1GB/sec SSD |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original tweet size in JSON 2624 Msgpack = 1817 Gzip + Json = 1058 Gzip + Msgpack = 1116 LZ4 + Json = 1628 LZ4 + Msgpack = 1361 | |
Original tweet size in JSON 1863 Msgpack = 1443 Gzip + Json = 0783 Gzip + Msgpack = 0835 LZ4 + Json = 1153 LZ4 + Msgpack = 1040 | |
Original tweet size in JSON 2074 Msgpack = 1670 Gzip + Json = 0842 Gzip + Msgpack = 0894 LZ4 + Json = 1229 LZ4 + Msgpack = 1139 | |
Original tweet size in JSON 2025 Msgpack = 1617 Gzip + Json = 0845 Gzip + Msgpack = 0895 LZ4 + Json = 1238 LZ4 + Msgpack = 1143 | |
Original tweet size in JSON 2069 Msgpack = 1663 Gzip + Json = 0846 Gzip + Msgpack = 0901 LZ4 + Json = 1243 LZ4 + Msgpack = 1164 | |
Original tweet size in JSON 2035 Msgpack = 1634 Gzip + Json = 0852 Gzip + Msgpack = 0907 LZ4 + Json = 1247 LZ4 + Msgpack = 1167 | |
Original tweet size in JSON 1988 Msgpack = 1464 Gzip + Json = 0804 Gzip + Msgpack = 0862 LZ4 + Json = 1220 LZ4 + Msgpack = 1061 | |
Original tweet size in JSON 1910 Msgpack = 1502 Gzip + Json = 0775 Gzip + Msgpack = 0832 LZ4 + Json = 1154 LZ4 + Msgpack = 1060 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.lang.reflect.ParameterizedType; | |
import java.lang.reflect.Type; | |
import java.util.ArrayList; | |
import java.util.Collection; | |
import java.util.Iterator; | |
import com.google.gson.Gson; | |
import com.google.gson.JsonArray; | |
import com.google.gson.JsonDeserializationContext; | |
import com.google.gson.JsonDeserializer; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
-- show running queries (pre 9.2) | |
SELECT procpid, age(clock_timestamp(), query_start), usename, current_query | |
FROM pg_stat_activity | |
WHERE current_query != '<IDLE>' AND current_query NOT ILIKE '%pg_stat_activity%' | |
ORDER BY query_start desc; | |
-- show running queries (9.2) | |
SELECT pid, age(clock_timestamp(), query_start), usename, query | |
FROM pg_stat_activity | |
WHERE query != '<IDLE>' AND query NOT ILIKE '%pg_stat_activity%' |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
#Reconnaissance | |
yum install dsniff hping3 nc6 nc ncrack ngrep nmap nmap-frontend p0f sing scanssh scapy socat tcpdump unicornscan wireshark-gnome xprobe2 nbtscan tcpxtract firewalk hunt dnsenum iftop argus ettercap ettercap-gtk packETH iptraf pcapdiff etherape lynis netsniff-ng tcpjunk ssldump yersinia net-snmp openvas-client openvas-scanner | |
#Forensics | |
yum install ddrescue gparted hexedit testdisk foremost sectool-gui scanmem sleuthkit unhide examiner dc3dd afftools srm firstaidkit-plugin-all ntfs-3g ntfsprogs | |
#Web Application Testing | |
yum install httping nikto ratproxy lbd skipfish |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
server { | |
listen 8080 default_server deferred; | |
charset utf-8; | |
access_log /var/log/nginx/graphite.access.log; | |
error_log /var/log/nginx/graphite.error.log; | |
root /opt/graphite/webapp; | |
location /static/admin/ { |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# by TB : 2012-10-12 | |
if [ "$USER" != "root" ] | |
then | |
echo "You must be root to execute this script (in order to instruct the kernel to drop the cache)!" | |
exit 1 | |
fi | |
if [ -z "$1" ] |