noemi-dresden

version: "3"
services:
  zookeeper-1:
    image: confluentinc/cp-zookeeper:latest
    hostname: zookeeper-1
    ports:
      - "12181:12181"
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 12181

noemi-dresden

docker run --name spark-app \
       -e ENABLE_INIT_DAEMON=false \
       --link spark-master:spark-master \
       --network=same-network-as-master-and-worker \
       spark-app

noemi-dresden

version: "3.1"
services:
  spark-master:
    image: bde2020/spark-master:2.4.0-hadoop2.7
    container_name: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
    environment:
      - INIT_DAEMON_STEP=setup_spark

noemi-dresden

mappings:
- match: '*.*.jvm.*.*'
  name: jvm_memory_usage
  labels:
    application: $1
    executor_id: $2
    mem_type: $3
    qty: $4

- match: '*.*.jvm.pools.*.*'

noemi-dresden

graphite_exporter:
    image: prom/graphite-exporter
    command: "--graphite.mapping-config=/tmp/graphite_mapping.conf"
    container_name: graphite_exporter
    volumes:
      - path/to-your/graphite_mapping.conf:/tmp/graphite_mapping.conf
    ports:
      - "9108:9108"
      - "9109:9109"
      - "9109:9109/udp"

noemi-dresden

*.sink.graphite.class=org.apache.spark.metrics.sink.GraphiteSink
*.sink.graphite.host=graphite_exporter
*.sink.graphite.port=9109
*.sink.graphite.period=10
*.sink.graphite.unit=seconds

# Enable JvmSource for instance master, worker, driver and executor
master.source.jvm.class=org.apache.spark.metrics.source.JvmSource

worker.source.jvm.class=org.apache.spark.metrics.source.JvmSource

noemi-dresden

docker run --name spark-app \
       -e ENABLE_INIT_DAEMON=false \
       --link spark-master:spark-master \
       -v path/to-your/metrics.properties:/spark/conf/metrics.properties \
       --network=same-network-as-master-and-worker \
       spark-app

noemi-dresden

version: "3.1"
services:
  spark-master:
    image: bde2020/spark-master:2.4.0-hadoop2.7
    container_name: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
    volumes:
      - ${PWD}/spark/metrics.properties:/spark/conf/metrics.properties

noemi-dresden

apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus-ip:9090

noemi-dresden

Fix /bin/sh^M : bad interpreter when running a shell script

1. Open the script using vi or vim
2. Type esc
3. Type :set fileformat=unix
4. Save with :wq!