Last active
December 25, 2015 21:58
-
-
Save munro/7045909 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$ cat orig_syslog_events | head -n 1 | java -jar logstash-1.2.1-flatjar.jar agent --pluginpath . -f logstash.rb | |
Exception in thread "LogStash::Runner" org.jruby.exceptions.RaiseException: (LoadError) no such file to load -- pg | |
at org.jruby.RubyKernel.require(org/jruby/RubyKernel.java:1054) | |
at Kernel.require(jar:file:/Users/rmunro/Desktop/logstash-1.2.1-flatjar.jar!/META-INF/jruby.home/lib/ruby/shared/rubygems/custom_require.rb:36) | |
at Kernel.require(file:/Users/rmunro/Desktop/logstash-1.2.1-flatjar.jar!/logstash/JRUBY-6970.rb:22) | |
at Kernel.require(file:/Users/rmunro/Desktop/logstash-1.2.1-flatjar.jar!/polyglot.rb:63) | |
at RUBY.register(/Users/rmunro/Desktop/logstash/outputs/postgresql.rb:41) | |
at org.jruby.RubyArray.each(org/jruby/RubyArray.java:1617) | |
at RUBY.outputworker(file:/Users/rmunro/Desktop/logstash-1.2.1-flatjar.jar!/logstash/pipeline.rb:208) | |
at RUBY.start_outputs(file:/Users/rmunro/Desktop/logstash-1.2.1-flatjar.jar!/logstash/pipeline.rb:140) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
source 'https://rubygems.org' | |
gem "pg", "~> 0.17.0" | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- encoding: utf-8 -*- | |
lib = File.expand_path('../lib/', __FILE__) | |
$:.unshift lib unless $:.include?(lib) | |
require 'bundler/version' | |
Gem::Specification.new do |gem| | |
gem.authors = ["Ryan Munro"] | |
gem.email = ["[email protected]"] | |
gem.description = %q{logstash postgres plugin} | |
gem.summary = %q{logstash - postgres plugin} | |
gem.homepage = "http://logstash.net/" | |
gem.license = "Apache License (2.0)" | |
# gem.files = `git ls-files`.split($\) | |
# gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } | |
# gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) | |
gem.name = "logstash-output-postgres" | |
gem.require_paths = ["logstash"] | |
gem.version = Bundler::VERSION | |
# # Core dependencies | |
# gem.add_runtime_dependency "cabin", ["0.5.0"] #(Apache 2.0 license) | |
# gem.add_runtime_dependency "json" #(ruby license) | |
# gem.add_runtime_dependency "minitest" # for running the tests from the jar, (MIT license) | |
# gem.add_runtime_dependency "pry" #(ruby license) | |
# gem.add_runtime_dependency "stud" #(Apache 2.0 license) | |
# gem.add_runtime_dependency "clamp" # for command line args/flags (MIT license) | |
# gem.add_runtime_dependency "i18n" #(MIT license) | |
# # Web dependencies | |
# gem.add_runtime_dependency "ftw", ["~> 0.0.36"] #(Apache 2.0 license) | |
# gem.add_runtime_dependency "haml" #(MIT license) | |
# gem.add_runtime_dependency "rack" #(MIT license) | |
# gem.add_runtime_dependency "sass" #(MIT license) | |
# gem.add_runtime_dependency "sinatra" #(MIT license) | |
# gem.add_runtime_dependency "mime-types" #(GPL 2.0) | |
gem.add_runtime_dependency "pg" #(Apache 2.0 license) | |
# gem.add_runtime_dependency "metriks" #(MIT license) | |
# gem.add_runtime_dependency "onstomp" #(Apache 2.0 license) | |
# gem.add_runtime_dependency "redis" #(MIT license) | |
# gem.add_runtime_dependency "riak-client", ["1.0.3"] #(Apache 2.0 license) | |
# gem.add_runtime_dependency "riemann-client", ["0.2.1"] #(MIT license) | |
# gem.add_runtime_dependency "statsd-ruby", ["1.2.0"] #(MIT license) | |
# gem.add_runtime_dependency "uuidtools" # For generating amqp queue names (Apache 2.0 license) | |
# gem.add_runtime_dependency "xml-simple" #(ruby license?) | |
# gem.add_runtime_dependency "xmpp4r", ["0.5"] #(ruby license) | |
# gem.add_runtime_dependency "jls-lumberjack", [">=0.0.19"] #(Apache 2.0 license) | |
# gem.add_runtime_dependency "geoip", [">= 1.3.2"] #(GPL license) | |
# gem.add_runtime_dependency "beefcake", "0.3.7" #(MIT license) | |
# gem.add_runtime_dependency "php-serialize" # For input drupal_dblog (MIT license) | |
# gem.add_runtime_dependency "murmurhash3" #(MIT license) | |
# gem.add_runtime_dependency "rufus-scheduler", "~> 2.0.24" #(MIT license) | |
# gem.add_runtime_dependency "user_agent_parser", [">= 2.0.0"] #(MIT license) | |
# gem.add_runtime_dependency "snmp" #(ruby license) | |
# gem.add_runtime_dependency "varnish-rb" #(MIT license) | |
# gem.add_runtime_dependency "mail" #(MIT license) | |
# gem.add_runtime_dependency "rbnacl" #(MIT license) | |
# gem.add_runtime_dependency "sequel" #(MIT license) | |
# gem.add_runtime_dependency "jdbc-sqlite3" #(MIT license) | |
# gem.add_runtime_dependency "bindata", [">= 1.5.0"] #(ruby license) | |
# gem.add_runtime_dependency "twitter", "5.0.0.rc.1" #(MIT license) | |
# if RUBY_PLATFORM == 'java' | |
# gem.platform = RUBY_PLATFORM | |
# gem.add_runtime_dependency "jruby-elasticsearch", ["0.0.15"] #(BSD license) | |
# gem.add_runtime_dependency "jruby-httpclient" #(Apache 2.0 license) | |
# gem.add_runtime_dependency "bouncy-castle-java", "1.5.0147" #(MIT license) | |
# gem.add_runtime_dependency "jruby-openssl", "0.8.7" #(CPL/GPL/LGPL license) | |
# gem.add_runtime_dependency "jruby-win32ole" #(unknown license) | |
# gem.add_runtime_dependency "jdbc-mysql" # For input drupal_dblog (BSD license) | |
# gem.add_runtime_dependency "msgpack-jruby" #(Apache 2.0 license) | |
# else | |
# gem.add_runtime_dependency "excon" #(MIT license) | |
# gem.add_runtime_dependency "mysql2" # For input drupal_dblog (MIT license) | |
# gem.add_runtime_dependency "msgpack" #(Apache 2.0 license) | |
# end | |
# if RUBY_PLATFORM != 'java' | |
# gem.add_runtime_dependency "bunny", ["~> 0.10.4"] #(MIT license) | |
# else | |
# gem.add_runtime_dependency "hot_bunnies", ["~> 2.0.0.pre12"] #(MIT license) | |
# end | |
# if RUBY_VERSION >= '1.9.1' | |
# gem.add_runtime_dependency "cinch" # cinch requires 1.9.1+ #(MIT license) | |
# end | |
# # These are runtime-deps so you can do 'java -jar logstash.jar rspec <test>' | |
# gem.add_runtime_dependency "spoon" #(Apache 2.0 license) | |
# gem.add_runtime_dependency "mocha" #(MIT license) | |
# gem.add_runtime_dependency "shoulda" #(MIT license) | |
# gem.add_runtime_dependency "rspec" #(MIT license) | |
# gem.add_runtime_dependency "insist", "1.0.0" #(Apache 2.0 license) | |
# gem.add_runtime_dependency "rumbster" # For faking smtp in email tests (Apache 2.0 license) | |
# #Development Deps | |
# gem.add_development_dependency "coveralls" | |
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require "logstash/outputs/base" | |
require "logstash/namespace" | |
class LogStash::Outputs::Mongodb < LogStash::Outputs::Base | |
config_name "postgresql" | |
milestone 1 | |
# a MongoDB URI to connect to | |
# See http://docs.mongodb.org/manual/reference/connection-string/ | |
config :host, :validate => :string, :required => true | |
config :user, :validate => :string, :required => true | |
config :password, :validate => :string, :required => true | |
# The database to use | |
config :database, :validate => :string, :required => true | |
# The collection to use. This value can use %{foo} values to dynamically | |
# select a collection based on data in the event. | |
config :table, :validate => :string, :required => true | |
config :columns, :validate => :hash, :required => true | |
# # If true, store the @timestamp field in mongodb as an ISODate type instead | |
# # of an ISO8601 string. For more information about this, see | |
# # http://www.mongodb.org/display/DOCS/Dates | |
# config :isodate, :validate => :boolean, :default => false | |
# # Number of seconds to wait after failure before retrying | |
# config :retry_delay, :validate => :number, :default => 3, :required => false | |
# # If true, a _id field will be added to the document before insertion. | |
# # The _id field will use the timestamp of the event and overwrite an existing | |
# # _id field in the event. | |
# config :generateId, :validate => :boolean, :default => false | |
public | |
def register | |
require "pg" | |
@conn = PGconn.open( | |
:host => @host, | |
:user => @user, | |
:password => @password, | |
:dbname => @database | |
) | |
$stdout.write(@columns) | |
end # def register | |
public | |
def receive(event) | |
return unless output?(event) | |
# SELECT column_name, data_type | |
# FROM information_schema.columns | |
# WHERE table_catalog = 'tarot' AND table_name = 'track_event_all'; | |
# conn.prepare("statement1", " | |
# SELECT | |
# ") | |
# conn.exec_prepared('statement1', [ 11, 'J.R. "Bob" Dobbs', 'Too much is always better than not enough.' ]) | |
# begin | |
# if @isodate | |
# # the mongodb driver wants time values as a ruby Time object. | |
# # set the @timestamp value of the document to a ruby Time object, then. | |
# document = event.to_hash | |
# else | |
# document = event.to_hash.merge("@timestamp" => event["@timestamp"].to_json) | |
# end | |
# if @generateId | |
# document['_id'] = BSON::ObjectId.new(nil, event["@timestamp"]) | |
# end | |
# @db.collection(event.sprintf(@collection)).insert(document) | |
# rescue => e | |
# @logger.warn("Failed to send event to MongoDB", :event => event, :exception => e, | |
# :backtrace => e.backtrace) | |
# if e.error_code == 11000 | |
# # On a duplicate key error, skip the insert. | |
# # We could check if the duplicate key err is the _id key | |
# # and generate a new primary key. | |
# # If the duplicate key error is on another field, we have no way | |
# # to fix the issue. | |
# else | |
# sleep @retry_delay | |
# retry | |
# end | |
# end | |
end # def receive | |
end # class LogStash::Outputs::Mongodb |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment