ottomata

Downloaded ring-cors manually and put it in .m2/repositories from
https://clojars.org/repo/ring-cors/ring-cors/0.1.5/


~/atlas/apache-maven-3.6.1/bin/mvn -Dhttp.proxyHost=webproxy.eqiad.wmnet -Dhttp.proxyPort=8080 -Dhttps.proxyHost=webproxy.eqiad.wmnet -Dhttps.proxyPort=8080 -DskipTests package -Pdist,embedded-hbase-solr -pl \!:storm-bridge-shim


// use elasticsearch and berkeleydb

ottomata

#!/usr/bin/env node
'use strict';

const util = require('util');
const _        = require('lodash');
const yaml     = require('js-yaml');
const path = require('path');
const glob = require('glob');
const semver = require('semver');
const NodeGit = require('nodegit');

ottomata

# HELP eventgate_rdkafka_producer_broker_int_latency Kafka Producer per broker window metric
# TYPE eventgate_rdkafka_producer_broker_int_latency gauge
eventgate_rdkafka_producer_broker_int_latency{broker_hostname="kafka-jumbo1001_eqiad_wmnet",broker_id="1001",broker_port="9092",producer_type="guaranteed",quantile="0.50",service="eventgate-analytics"} 0
eventgate_rdkafka_producer_broker_int_latency{broker_hostname="kafka-jumbo1001_eqiad_wmnet",broker_id="1001",broker_port="9092",producer_type="guaranteed",quantile="0.75",service="eventgate-analytics"} 0
eventgate_rdkafka_producer_broker_int_latency{broker_hostname="kafka-jumbo1001_eqiad_wmnet",broker_id="1001",broker_port="9092",producer_type="guaranteed",quantile="0.90",service="eventgate-analytics"} 0
eventgate_rdkafka_producer_broker_int_latency{broker_hostname="kafka-jumbo1001_eqiad_wmnet",broker_id="1001",broker_port="9092",producer_type="guaranteed",quantile="0.95",service="eventgate-analytics"} 0
eventgate_rdkafka_producer_broker_int_latency{broker_hos

ottomata

#!/usr/bin/env node
'use strict';

# npm install eventsource
const EventSource = require('eventsource');

const url = `https://stream.wikimedia.org/v2/stream/recentchange?since=2019-04-28`;
let eventSource = new EventSource(url, options);

eventSource.onmessage = function(event) {

ottomata

[2019-04-15T19:08:08.791Z] TRACE: eventgate/140 on eventgate-analytics-staging-5ff6445bc5-l7mcn: kafka-jumbo1003.eqiad.wmnet:9092/1003: staging.test.event [0] 86 message(s) in xmit queue (84 added from partition queue) (producer_type=hasty, rdkafka_facility=TOPPAR, rdkafka_thread=kafka-jumbo1003.eqiad.wmnet:9092/bootstrap)
[2019-04-15T19:08:08.852Z] TRACE: eventgate/140 on eventgate-analytics-staging-5ff6445bc5-l7mcn: kafka-jumbo1003.eqiad.wmnet:9092/1003: staging.test.event [0]: Produce MessageSet with 86 message(s) (1449 bytes, ApiVersion 3, MsgVersion 2) (producer_type=hasty, rdkafka_facility=PRODUCE, rdkafka_thread=kafka-jumbo1003.eqiad.wmnet:9092/bootstrap)
[2019-04-15T19:08:08.903Z] TRACE: eventgate/140 on eventgate-analytics-staging-5ff6445bc5-l7mcn: kafka-jumbo1003.eqiad.wmnet:9092/1003: Sent ProduceRequest (v3, 1541 bytes @ 0, CorrId 21) (producer_type=hasty, rdkafka_facility=SEND, rdkafka_thread=kafka-jumbo1003.eqiad.wmnet:9092/bootstrap)
[2019-04-15T19:08:08.922Z] TRACE: eventgate/140 on eventgate-

ottomata

strace: Process 11811 attached
14:49:03.540108 write(3, "{\"type\":\"heartbeat\"}\n", 21) = 21
14:49:03.540265 epoll_ctl(4, EPOLL_CTL_MOD, 3, {EPOLLIN, {u32=3, u64=3}}) = 0
14:49:04.016152 recvmsg(3, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="{\"cmd\":\"NODE_HANDLE\",\"type\":\"net"..., iov_len=65536}], msg_iovlen=1, msg_control=[{cmsg_len=20, cmsg_level=SOL_SOCKET, cmsg_type=SCM_RIGHTS, cmsg_data=[53]}], msg_controllen=24, msg_flags=MSG_CMSG_CLOEXEC}, MSG_CMSG_CLOEXEC) = 131
14:49:04.016436 getsockname(53, {sa_family=AF_INET, sin_port=htons(8192), sin_addr=inet_addr("127.0.0.1")}, [128->16]) = 0
14:49:04.016521 getsockopt(53, SOL_SOCKET, SO_TYPE, [1], [4]) = 0
14:49:04.016873 write(3, "{\"cmd\":\"NODE_HANDLE_ACK\"}\n", 26) = 26
14:49:04.017088 write(3, "{\"cmd\":\"NODE_CLUSTER\",\"accepted\""..., 57) = 57
14:49:04.017433 setsockopt(53, SOL_TCP, TCP_NODELAY, [1], 4) = 0
14:49:04.017539 epoll_ctl(4, EPOLL_CTL_MOD, 3, {EPOLLIN, {u32=3, u64=3}}) = 0

ottomata

        hasty:
          conf:
            producer.poll.interval.ms: 10
        guaranteed:
          conf:
            producer.poll.interval.ms: 10

ottomata

pod_name=eventgate-analytics-staging-9fdf767c-lhb7s
sudo strace -tt -p $(sudo docker top $(sudo docker ps | grep $pod_name | grep nodejs | head -n 1 | awk '{print $1}')  | grep /usr/bin/node | head -n 1 | awk '{print $2}') 2>&1 | grep -v 'epoll_wait'

21:04:44.305682 read(53, "POST /v1/events?hasty=true HTTP/"..., 65536) = 406
21:04:44.377385 getpid()                = 140
21:04:44.484994 getpid()                = 140
21:04:44.515973 getpeername(53, {sa_family=AF_INET, sin_port=htons(47888), sin_addr=inet_addr("10.64.0.247")}, [128->16]) = 0
21:04:44.669330 write(53, "HTTP/1.1 202 1 events hastily re"..., 560) = 560
21:04:44.751896 write(1, "{\"name\":\"eventgate-analytics\",\"h"..., 586) = 586
21:04:45.008037 sendmsg(12, {msg_name={sa_family=AF_INET, sin_port=htons(9125), sin_addr=inet_addr("127.0.0.1")}, msg_namelen=16, msg_iov=[{iov_base="eventgate-analytics.v1_events.PO"..., iov_len=273}], msg_iovlen=1, msg_controllen=0, msg_flags=0}, 0) = 273

ottomata

/usr/bin/spark2-submit \
--name otto_test_refine_eventlogging_0 \
--class org.wikimedia.analytics.refinery.job.refine.Refine \
--master yarn \
--deploy-mode client \
--conf spark.driver.extraClassPath=/usr/lib/hadoop-mapreduce/hadoop-mapreduce-client-common.jar:/srv/deployment/analytics/refinery/artifacts/hive-jdbc-1.1.0-cdh5.10.0.jar:/srv/deployment/analytics/refinery/artifacts/hive-service-1.1.0-cdh5.10.0.jar \
--driver-java-options='-Drefine.log.level=DEBUG  -Drefinery.log.level=DEBUG -Dhttp.proxyHost=webproxy.eqiad.wmnet -Dhttp.proxyPort=8080 -Dhttps.proxyHost=webproxy.eqiad.wmnet -Dhttps.proxyPort=8080' \
/home/otto/refinery-source/refinery-job/target/refinery-job-0.0.89-SNAPSHOT.jar \
--database=otto_json_refine_test \
--hive_server_url=an-coord1001.eqiad.wmnet:10000 \

ottomata

sudo php tests/phpunit/phpunit.php --wiki=wiki extensions/CirrusSearch/tests/phpunit/UtilTest.php