nuria

curl -X POST -H "Content-Type: application/vnd.kafka.jsonschema.v2+json" \
 -H "Accept: application/vnd.kafka.v2+json" \
 --data '{"value_schema_id":100101,   "records": [{"value": {"name": "testUser", "planet":"moon"} } ] }' \
   "http://localhost:8082/topics/rest-proxy-test"

nuria

kafkacat -F ./kafkacat.auth -p 0  -t postgres.public.activities -o 31992 -u -c1 | hexdump -C


sasl.username=
sasl.password=

sasl.mechanisms=PLAIN
# Required for correctness in Apache Kafka clients prior to 2.6
#client.dns.lookup=use_all_dns_ips
# Best practice for Kafka producer to prevent data loss

nuria

kafkacat -F ./kafkacat.auth -p 0  -t dlq-lcc-dy0v1 -C   -f '\nKey (%K bytes): %k
  Value (%S bytes): %s
  Timestamp: %T
  Partition: %p
  Offset: %o
  Headers: %h\n'

nuria

[
  {
    "user_group": [
      "dashboarding"
    ],
    "query_group": [],
    "name": "dashboarding",
    "memory_percent_to_use": 70,
    "query_concurrency": 15,
    "concurrency_scaling": "auto"

nuria

// spark-shell --jars /home/otto/algebird-core_2.10-0.9.0.jar,/home/mforns/refinery-core-0.0.9.jar


import java.util.Date
import java.text.SimpleDateFormat
import org.wikimedia.analytics.refinery.core.PageviewDefinition
import org.wikimedia.analytics.refinery.core.Webrequest
import scala.math.pow
import org.apache.spark.rdd.RDD
import com.twitter.algebird.QTree

nuria

import org.apache.spark.sql.{ Encoders, SaveMode }

val readPath = s"/wmf/data/event/PrefUpdate/year=*/month=*/day=*/hour=*/*.parquet"

val propertyWhitelistFilter = s"event.property in ('skin', 'mfMode', 'mf_amc_optin', 'VectorSkinVersion', 'popups', 'popupsreferencepreviews', 'discussiontools-betaenable', 'betafeatures-auto-enroll' , 'echo-notifications-blacklist', 'email-blacklist', 'growthexperiments-help-panel-tog-help-panel', 'growthexperiments-homepage-enable', 'growthexperiments-homepage-pt-link')"

case class UserAgent(
    browser_family: String,
    browser_major: String,
    browser_minor: String,

nuria

use wmf;

select distinct base_name
from
mediarequest
where year=2019
and month=12
and day=1
and hour=1
and base_name like '%commons%'

nuria

curl -v -H 'Content-Type: text/plain'  -d'{"$schema": "/mediawiki/client/error/1.0.0", "meta": {"stream": "mediawiki.client.error"}, "message": "test event", "type": "TEST", "url": "http://otto-test.org", "user_agent": "otto test"}' 'https://intake-logging.wikimedia.org/v1/events?hasty=true'

nuria

mw.config.values.wgWMEClientErrorIntakeURL

/**
{"$schema": "/mediawiki/client/error/1.0.0", "meta": {"stream": "mediawiki.client.error"}, "message": "test event", "type": "TEST", "url": "http://otto-test.org", "user_agent": "otto test"}
**/
mw.track( 'global.error', {
				errorMessage: 'ayayaya',
				url: 'https://intake-logging.wikimedia.org/v1/events?hasty=true',
				lineNumber: 1,
				columnNumber: 1,

nuria

import java.net.URLDecoder
import java.net.URLEncoder
import org.apache.spark.sql.functions._ 

val urlDecoder = (u: String) => URLDecoder.decode(u.replaceAll("%(?![0-9a-fA-F]{2})", "%25").replaceAll("\\+", "%2B"), "UTF-8")
val urlEncoder = (u: String) => URLEncoder.encode(u, "UTF-8")
val countSlashes = (u: String) => u.count(_ == '/')