Andrew Otto ottomata
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # grep -> sed the JSON event string and parse it with JQ | |
| curl -s https://stream.wikimedia.org/v2/stream/recentchange | | |
| grep data | | |
| sed 's/^data: //g' | | |
| jq . |
ottomata
/ gist:33f3fafab813ca55a3129ce97d836ee5
Created
February 15, 2017 15:00
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Origin: Wikimedia | |
| Label: Wikimedia | |
| Suite: jessie-cloudera | |
| Codename: jessie-cloudera | |
| AlsoAcceptFor: cloudera | |
| Version: 8 | |
| Architectures: source amd64 i386 | |
| Components: cloudera | |
| UDebComponents: cloudera | |
| Update: cloudera |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import java.beans.Transient | |
| import java.text.SimpleDateFormat | |
| import org.apache.avro.SchemaBuilder | |
| import org.apache.avro.generic.{GenericRecordBuilder, IndexedRecord} | |
| import org.json4s.JsonDSL._ | |
| import org.json4s._ | |
| import org.json4s.jackson.JsonMethods._ | |
| import org.wikimedia.analytics.refinery.core._ |
ottomata
/ GaugeCounter.js
Created
February 7, 2017 15:33
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| class GaugeCounter { | |
| constructor(statsd, flush_interval_ms) { | |
| this.statsd = statsd; | |
| this.flush_interval_ms = flush_interval_ms || 5000; | |
| this.counters = {}; | |
| this.start(); | |
| } | |
| modify(key, delta) { | |
| if (!key) { |
ottomata
/ api.php_cache_status.csv
Created
February 6, 2017 21:31
api.php cache_status counts for 2017-02-01
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| cache_status | _c1 | |
|---|---|---|
| - | 30 | |
| hit | 154443263 | |
| int | 49397862 | |
| pass | 301643378 | |
| miss | 54975448 |
ottomata
/ test_banner_impressions_joal_druid_tranquility.json
Last active
January 23, 2017 15:46
ottomata
/ tranquility-kafka.propeties.json
Created
January 19, 2017 18:38
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "dataSources" : [ | |
| { | |
| "spec" : { | |
| "dataSchema" : { | |
| "dataSource" : "webrequest_test1", | |
| "metricsSpec" : [ | |
| { "type" : "count", "name" : "count" } | |
| ], | |
| "granularitySpec" : { |
ottomata
/ eventstreams_spec.yaml
Created
December 13, 2016 19:40
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| swagger: '2.0' | |
| info: | |
| version: 0.0.1 | |
| title: WMF EventStreams | |
| description: Streaming WMF event data over HTTP SSE / EventSource | |
| termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use | |
| license: | |
| name: Apache2 | |
| url: http://www.apache.org/licenses/LICENSE-2.0 | |
| paths: |
ottomata
/ webrequest_text_holes_2016-12-07T18:00.txt
Created
December 8, 2016 15:41
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| hostname missing_start missing_end missing_count dt_before_missing dt_after_missing | |
| cp1068.eqiad.wmnet 1094084466 1094084466 1 2016-12-07T18:45:44 2016-12-07T18:45:44 | |
| cp1068.eqiad.wmnet 1094084494 1094084494 1 2016-12-07T18:45:44 2016-12-07T18:45:44 | |
| cp1068.eqiad.wmnet 1094084507 1094084507 1 2016-12-07T18:45:44 2016-12-07T18:45:44 | |
| cp1068.eqiad.wmnet 1094084542 1094084542 1 2016-12-07T18:45:44 2016-12-07T18:45:44 | |
| cp1068.eqiad.wmnet 1094084594 1094084594 1 2016-12-07T18:45:44 2016-12-07T18:45:44 | |
| cp1068.eqiad.wmnet 1094084663 1094084663 1 2016-12-07T18:45:44 2016-12-07T18:45:44 | |
| cp1068.eqiad.wmnet 1094084666 1094084666 1 2016-12-07T18:45:44 2016-12-07T18:45:44 | |
| cp1068.eqiad.wmnet 1094084685 1094084685 1 2016-12-07T18:45:44 2016-12-07T18:45:44 | |
| cp1068.eqiad.wmnet 1094084763 1094084763 1 2016-12-07T18:45:44 2016-12-07T18:45:44 |
ottomata
/ rdkafka-consume.js
Last active
November 15, 2016 17:23
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| "use strict"; | |
| var kafka = require('node-rdkafka'); | |
| var consumer = new kafka.KafkaConsumer({ | |
| 'metadata.broker.list': 'localhost:9092', | |
| 'group.id': 'test1' | |
| }); | |
| // Non-flowing mode | |
| consumer.connect(); |