beaver

gistfile1.txt

root@log:/etc/beaver# beaver -h
usage: beaver [-h] [-c CONFIG] [-C CONFD_PATH] [-d] [-D]
              [-f FILES [FILES ...]] [-F {json,msgpack,raw,rawjson,string}]
              [-H HOSTNAME] [-m {bind,connect}] [-l OUTPUT] [-p PATH] [-P PID]
              [-t {rabbitmq,redis,sqs,stdout,udp,zmq}] [-e] [-v] [--fqdn]

Beaver logfile shipper

optional arguments:
  -h, --help            show this help message and exit
  -c CONFIG, --configfile CONFIG
                        ini config file path
  -C CONFD_PATH, --confd-path CONFD_PATH
                        path to conf.d directory
  -d, --debug           enable debug mode
  -D, --daemonize       daemonize in the background
  -f FILES [FILES ...], --files FILES [FILES ...]
                        space-separated filelist to watch, can include globs
                        (*.log). Overrides --path argument
  -F {json,msgpack,raw,rawjson,string}, --format {json,msgpack,raw,rawjson,string}
                        format to use when sending to transport
  -H HOSTNAME, --hostname HOSTNAME
                        manual hostname override for source_host
  -m {bind,connect}, --mode {bind,connect}
                        bind or connect mode
  -l OUTPUT, --logfile OUTPUT, -o OUTPUT, --output OUTPUT
                        file to pipe output to (in addition to stdout)
  -p PATH, --path PATH  path to log files
  -P PID, --pid PID     path to pid file
  -t {rabbitmq,redis,sqs,stdout,udp,zmq}, --transport {rabbitmq,redis,sqs,stdout,udp,zmq}
                        log transport method
  -e, --experimental    use experimental version of beaver
  -v, --version         output version and quit
  --fqdn                use the machine's FQDN for source_host

    Beaver provides an lightweight method for shipping local log
    files to Logstash. It does this using either redis, stdin,
    zeromq as the transport. This means you'll need a redis,
    stdin, zeromq input somewhere down the road to get the events.

    Events are sent in logstash's json_event format. Options can
    also be set as environment variables.

    Please see the readme for complete examples.
    
root@log:/etc/beaver# tree 
.
├── beaver.conf
└── conf.d
    └── some.conf

1 directory, 2 files
root@log:/etc/beaver# cat beaver.conf 
[beaver]
transport: redis
redis_url: redis://log.[secret].com:6379/0
redis_namespace: logstash

root@log:/etc/beaver# cat conf.d/some.conf 
[/opt/log/*.log]
type: syslog
root@log:/etc/beaver# beaver -c /etc/beaver/beaver.conf -C /etc/beaver/conf.d -d
[2013-05-22 06:58:28,379] DEBUG   Logger level is DEBUG
[2013-05-22 06:58:28,379] DEBUG   Processing beaver portion of config file /etc/beaver/beaver.conf
[2013-05-22 06:58:28,381] DEBUG   [CONFIG] "rabbitmq_vhost" => "/"
[2013-05-22 06:58:28,382] DEBUG   [CONFIG] "sqs_aws_secret_key" => "None"
[2013-05-22 06:58:28,382] DEBUG   [CONFIG] "respawn_delay" => "3"
[2013-05-22 06:58:28,382] DEBUG   [CONFIG] "ssh_tunnel" => "None"
[2013-05-22 06:58:28,382] DEBUG   [CONFIG] "mqtt_clientid" => "mosquitto"
[2013-05-22 06:58:28,382] DEBUG   [CONFIG] "pid" => "None"
[2013-05-22 06:58:28,382] DEBUG   [CONFIG] "redis_url" => "redis://log.[secret].com:6379/0"
[2013-05-22 06:58:28,383] DEBUG   [CONFIG] "mqtt_host" => "localhost"
[2013-05-22 06:58:28,383] DEBUG   [CONFIG] "rabbitmq_exchange_durable" => "False"
[2013-05-22 06:58:28,383] DEBUG   [CONFIG] "max_failure" => "7"
[2013-05-22 06:58:28,383] DEBUG   [CONFIG] "rabbitmq_exchange" => "logstash-exchange"
[2013-05-22 06:58:28,383] DEBUG   [CONFIG] "globs" => "{}"
[2013-05-22 06:58:28,383] DEBUG   [CONFIG] "transport" => "redis"
[2013-05-22 06:58:28,384] DEBUG   [CONFIG] "redis_password" => "None"
[2013-05-22 06:58:28,384] DEBUG   [CONFIG] "mqtt_topic" => "/logstash"
[2013-05-22 06:58:28,384] DEBUG   [CONFIG] "rabbitmq_host" => "localhost"
[2013-05-22 06:58:28,384] DEBUG   [CONFIG] "mqtt_port" => "1883"
[2013-05-22 06:58:28,384] DEBUG   [CONFIG] "hostname" => "log"
[2013-05-22 06:58:28,384] DEBUG   [CONFIG] "ssh_tunnel_port" => "None"
[2013-05-22 06:58:28,384] DEBUG   [CONFIG] "daemonize" => "False"
[2013-05-22 06:58:28,385] DEBUG   [CONFIG] "max_queue_size" => "100"
[2013-05-22 06:58:28,385] DEBUG   [CONFIG] "zeromq_address" => "tcp://localhost:2120"
[2013-05-22 06:58:28,385] DEBUG   [CONFIG] "config" => "/etc/beaver/beaver.conf"
[2013-05-22 06:58:28,385] DEBUG   [CONFIG] "files" => "None"
[2013-05-22 06:58:28,385] DEBUG   [CONFIG] "zeromq_pattern" => "push"
[2013-05-22 06:58:28,385] DEBUG   [CONFIG] "discover_interval" => "15.0"
[2013-05-22 06:58:28,385] DEBUG   [CONFIG] "zeromq_hwm" => "None"
[2013-05-22 06:58:28,386] DEBUG   [CONFIG] "rabbitmq_ha_queue" => "False"
[2013-05-22 06:58:28,386] DEBUG   [CONFIG] "subprocess_poll_sleep" => "1"
[2013-05-22 06:58:28,386] DEBUG   [CONFIG] "format" => "json"
[2013-05-22 06:58:28,386] DEBUG   [CONFIG] "update_file_mapping_time" => "None"
[2013-05-22 06:58:28,386] DEBUG   [CONFIG] "ssh_remote_port" => "None"
[2013-05-22 06:58:28,386] DEBUG   [CONFIG] "udp_port" => "9999"
[2013-05-22 06:58:28,386] DEBUG   [CONFIG] "redis_namespace" => "logstash"
[2013-05-22 06:58:28,387] DEBUG   [CONFIG] "wait_timeout" => "5"
[2013-05-22 06:58:28,387] DEBUG   [CONFIG] "rabbitmq_queue" => "logstash-queue"
[2013-05-22 06:58:28,387] DEBUG   [CONFIG] "rabbitmq_key" => "logstash-key"
[2013-05-22 06:58:28,387] DEBUG   [CONFIG] "path" => "/var/log"
[2013-05-22 06:58:28,387] DEBUG   [CONFIG] "udp_host" => "127.0.0.1"
[2013-05-22 06:58:28,387] DEBUG   [CONFIG] "sincedb_path" => "None"
[2013-05-22 06:58:28,387] DEBUG   [CONFIG] "rabbitmq_exchange_type" => "direct"
[2013-05-22 06:58:28,388] DEBUG   [CONFIG] "rabbitmq_username" => "guest"
[2013-05-22 06:58:28,388] DEBUG   [CONFIG] "sqs_aws_region" => "us-east-1"
[2013-05-22 06:58:28,388] DEBUG   [CONFIG] "ssh_remote_host" => "None"
[2013-05-22 06:58:28,388] DEBUG   [CONFIG] "rabbitmq_queue_durable" => "False"
[2013-05-22 06:58:28,388] DEBUG   [CONFIG] "queue_timeout" => "60"
[2013-05-22 06:58:28,388] DEBUG   [CONFIG] "mqtt_keepalive" => "60"
[2013-05-22 06:58:28,388] DEBUG   [CONFIG] "fqdn" => "False"
[2013-05-22 06:58:28,389] DEBUG   [CONFIG] "zeromq_bind" => "connect"
[2013-05-22 06:58:28,389] DEBUG   [CONFIG] "ssh_key_file" => "None"
[2013-05-22 06:58:28,389] DEBUG   [CONFIG] "sqs_aws_queue" => "None"
[2013-05-22 06:58:28,389] DEBUG   [CONFIG] "debug" => "True"
[2013-05-22 06:58:28,389] DEBUG   [CONFIG] "output" => "None"
[2013-05-22 06:58:28,389] DEBUG   [CONFIG] "rabbitmq_password" => "guest"
[2013-05-22 06:58:28,389] DEBUG   [CONFIG] "sqs_aws_access_key" => "None"
[2013-05-22 06:58:28,390] DEBUG   [CONFIG] "rabbitmq_port" => "5672"
[2013-05-22 06:58:28,390] DEBUG   [CONFIG] "confd_path" => "/etc/beaver/conf.d"
[2013-05-22 06:58:28,394] INFO    Starting worker...
[2013-05-22 06:58:28,395] INFO    [ca01g23393] - watching logfile /var/log/alternatives.log
[2013-05-22 06:58:28,396] INFO    [ca01g2345f] - watching logfile /var/log/apport.log
[2013-05-22 06:58:28,396] INFO    [ca01g2340d] - watching logfile /var/log/kern.log
[2013-05-22 06:58:28,397] INFO    [ca01g20ec6] - watching logfile /var/log/atop.log
[2013-05-22 06:58:28,397] INFO    [ca01g20ec2] - watching logfile /var/log/fontconfig.log
[2013-05-22 06:58:28,397] INFO    [ca01g201da] - watching logfile /var/log/boot.log
[2013-05-22 06:58:28,398] INFO    [ca01g23391] - watching logfile /var/log/dpkg.log
[2013-05-22 06:58:28,398] INFO    [ca01g23568] - watching logfile /var/log/mail.log
[2013-05-22 06:58:28,399] INFO    [ca01g233f7] - watching logfile /var/log/cloud-init.log
[2013-05-22 06:58:28,399] INFO    [ca01g2340c] - watching logfile /var/log/ufw.log
[2013-05-22 06:58:28,400] INFO    [ca01g2358c] - watching logfile /var/log/auth.log
[2013-05-22 06:58:28,400] DEBUG   [ca01g23391] - getting start position /var/log/dpkg.log
[2013-05-22 06:58:28,400] DEBUG   [ca01g23391] - getting end position for /var/log/dpkg.log
[2013-05-22 06:58:28,404] DEBUG   [ca01g23391] - line count 2840 for /var/log/dpkg.log
[2013-05-22 06:58:28,404] DEBUG   [ca01g23393] - getting start position /var/log/alternatives.log
[2013-05-22 06:58:28,404] DEBUG   [ca01g23393] - getting end position for /var/log/alternatives.log
[2013-05-22 06:58:28,405] DEBUG   [ca01g23393] - line count 260 for /var/log/alternatives.log
[2013-05-22 06:58:28,405] DEBUG   [ca01g20ec2] - getting start position /var/log/fontconfig.log
[2013-05-22 06:58:28,405] DEBUG   [ca01g20ec2] - getting end position for /var/log/fontconfig.log
[2013-05-22 06:58:28,405] DEBUG   [ca01g20ec2] - line count 7 for /var/log/fontconfig.log
[2013-05-22 06:58:28,406] DEBUG   [ca01g2358c] - getting start position /var/log/auth.log
[2013-05-22 06:58:28,406] DEBUG   [ca01g2358c] - getting end position for /var/log/auth.log
[2013-05-22 06:58:28,417] DEBUG   [ca01g2358c] - line count 6955 for /var/log/auth.log
[2013-05-22 06:58:28,417] DEBUG   [ca01g201da] - getting start position /var/log/boot.log
[2013-05-22 06:58:28,417] DEBUG   [ca01g201da] - getting end position for /var/log/boot.log
[2013-05-22 06:58:28,417] DEBUG   [ca01g201da] - line count 32 for /var/log/boot.log
[2013-05-22 06:58:28,417] DEBUG   [ca01g20ec6] - getting start position /var/log/atop.log
[2013-05-22 06:58:28,418] DEBUG   [ca01g20ec6] - getting end position for /var/log/atop.log
[2013-05-22 06:58:28,418] DEBUG   [ca01g20ec6] - UnicodeDecodeError raised for /var/log/atop.log with encoding utf_8
[2013-05-22 06:58:28,431] DEBUG   [ca01g20ec6] - UnicodeDecodeError raised for /var/log/atop.log with encoding windows-1252
[2013-05-22 06:58:28,439] DEBUG   [ca01g20ec6] - line count 48 for /var/log/atop.log
[2013-05-22 06:58:28,439] DEBUG   [ca01g23568] - getting start position /var/log/mail.log
[2013-05-22 06:58:28,440] DEBUG   [ca01g23568] - getting end position for /var/log/mail.log
[2013-05-22 06:58:28,440] DEBUG   [ca01g23568] - line count 1 for /var/log/mail.log
[2013-05-22 06:58:28,440] DEBUG   [ca01g2340d] - getting start position /var/log/kern.log
[2013-05-22 06:58:28,440] DEBUG   [ca01g2340d] - getting end position for /var/log/kern.log
[2013-05-22 06:58:28,441] DEBUG   [ca01g2340d] - line count 249 for /var/log/kern.log
[2013-05-22 06:58:28,441] DEBUG   [ca01g2340c] - getting start position /var/log/ufw.log
[2013-05-22 06:58:28,441] DEBUG   [ca01g2340c] - getting end position for /var/log/ufw.log
[2013-05-22 06:58:28,441] DEBUG   [ca01g2340c] - line count 0 for /var/log/ufw.log
[2013-05-22 06:58:28,442] DEBUG   [ca01g2345f] - getting start position /var/log/apport.log
[2013-05-22 06:58:28,442] DEBUG   [ca01g2345f] - getting end position for /var/log/apport.log
[2013-05-22 06:58:28,442] DEBUG   [ca01g2345f] - line count 0 for /var/log/apport.log
[2013-05-22 06:58:28,442] DEBUG   [ca01g233f7] - getting start position /var/log/cloud-init.log
[2013-05-22 06:58:28,442] DEBUG   [ca01g233f7] - getting end position for /var/log/cloud-init.log
[2013-05-22 06:58:28,443] DEBUG   [ca01g233f7] - line count 129 for /var/log/cloud-init.log
[2013-05-22 06:58:28,443] INFO    Working...
[2013-05-22 06:58:28,443] INFO    Starting queue consumer
[2013-05-22 06:58:28,449] DEBUG   Iteration took 0.005844
[2013-05-22 06:58:28,453] DEBUG   Logging using the redis transport
[2013-05-22 06:58:28,554] DEBUG   Iteration took 0.000769
[2013-05-22 06:58:28,655] DEBUG   Iteration took 0.000727
[2013-05-22 06:58:28,756] DEBUG   Iteration took 0.000739
[2013-05-22 06:58:28,857] DEBUG   Iteration took 0.000668
[2013-05-22 06:58:28,958] DEBUG   Iteration took 0.000669
[2013-05-22 06:58:29,060] DEBUG   Iteration took 0.000829
[2013-05-22 06:58:29,161] DEBUG   Iteration took 0.000789
[2013-05-22 06:58:29,262] DEBUG   Iteration took 0.000663
[2013-05-22 06:58:29,363] DEBUG   Iteration took 0.000644
[2013-05-22 06:58:29,464] DEBUG   Iteration took 0.000927
[2013-05-22 06:58:29,565] DEBUG   Iteration took 0.000679
[2013-05-22 06:58:29,666] DEBUG   Iteration took 0.000668
[2013-05-22 06:58:29,767] DEBUG   Iteration took 0.000653
[2013-05-22 06:58:29,868] DEBUG   Iteration took 0.000752
[2013-05-22 06:58:29,969] DEBUG   Iteration took 0.000725
[2013-05-22 06:58:30,070] DEBUG   Iteration took 0.000749
[2013-05-22 06:58:30,171] DEBUG   Iteration took 0.000790
[2013-05-22 06:58:30,273] DEBUG   Iteration took 0.000767
^C[2013-05-22 06:58:30,288] INFO    SIGINT detected
[2013-05-22 06:58:30,289] INFO    Shutting down. Please wait...
[2013-05-22 06:58:30,289] INFO    Closing worker...
[2013-05-22 06:58:30,290] INFO    Shutdown complete.