Template - Oshinko - Spark - Kafka for Openshift using RHEL images

spark-template.yaml

apiVersion: v1
kind: Template
labels:
  template: spark-kafka-persistent
metadata:
  annotations:
    description: Application template for Spark and kafka
    tags: spark,kafka
    version: "1.0"
  name: spark-kafka-persistent
objects:
- apiVersion: v1
  kind: ImageStream
  metadata:
    name: kafka
- apiVersion: v1
  kind: ImageStream
  metadata:
    name: oshinko-webui
- apiVersion: v1
  kind: ImageStream
  metadata:
    name: spark
- apiVersion: v1
  kind: DeploymentConfig
  metadata:
    name: kafka-deploy
  spec:
    replicas: 1
    selector:
      deploymentConfig: kafka-deploy
    strategy:
      type: Rolling
    template:
      metadata:
        labels:
          application: ${APPLICATION_NAME}
          deploymentConfig: kafka-deploy
      spec:
        containers:
        - args:
          - config/server.properties
          - --override
          - advertised.host.name=kafka
          - --override
          - log.segment.bytes=10485760
          - --override
          - log.retention.bytes=10485760
          command:
          - bin/kafka-server-start.sh
          image: ${REGISTRY}/${NAMESPACE}/kafka:${KAFKA_VER}
          imagePullPolicy: Always
          name: kafka
          ports:
          - containerPort: 9092
            protocol: TCP
          volumeMounts:
          - mountPath: /tmp/kafka-logs
            name: kafka-logs
        - args:
          - config/zookeeper.properties
          command:
          - bin/zookeeper-server-start.sh
          image: ${REGISTRY}/${NAMESPACE}/kafka:${KAFKA_VER}
          imagePullPolicy: Always
          name: zookeeper
          ports:
          - containerPort: 2181
            protocol: TCP
          volumeMounts:
          - mountPath: /tmp/zookeeper
            name: zookeeper
        volumes:
        - emptyDir: {}
          name: kafka-logs
        - emptyDir: {}
          name: zookeeper
    triggers:
    - type: ConfigChange
- apiVersion: v1
  kind: DeploymentConfig
  metadata:
    name: oshinko-deploy
  spec:
    replicas: 1
    selector:
      deploymentConfig: oshinko-web
    strategy:
      type: Rolling
    template:
      metadata:
        labels:
          application: ${APPLICATION_NAME}
          deploymentConfig: oshinko-web
      spec:
        containers:
        - name: oc-proxy
          image: docker.io/crobby/oc-proxy
          imagePullPolicy: IfNotPresent
          args:
            - proxy
            - "-p"
            - '8001'
            - "--address=0.0.0.0"
            - "--disable-filter=true"
            - "--api-prefix=/proxy"
          ports:
            - name: oc-proxy-port
              containerPort: 8001
              protocol: TCP
          volumes:
            - name: test-volume
              emptyDir: {}
        - name: oshinko-web
          env:
          - name: OSHINKO_SPARK_IMAGE
            value: ${REGISTRY}/${NAMESPACE}/spark:${SPARK_VER}
          - name: OSHINKO_REFRESH_INTERVAL
            value: "5"
          image: ${REGISTRY}/${NAMESPACE}/oshinko-webui:latest
          imagePullPolicy: Always
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /
              port: 8080
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          ports:
          - containerPort: 8080
            name: o-web-port
            protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /
              port: 8080
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
        serviceAccount: oshinko
    triggers:
    - type: ConfigChange
- apiVersion: v1
  kind: BuildConfig
  metadata:
    name: kafka-build
  spec:
    output:
      to:
        kind: ImageStreamTag
        name: kafka:${KAFKA_VER}
    source:
      dockerfile: |
        FROM registry.access.redhat.com/rhel7:$RHEL_VER
        RUN mkdir -p /opt/kafka \
          && cd /opt/kafka \
          && yum -y install --setopt=tsflags=nodocs java-$JDK_VER-openjdk-headless tar \
          && curl -s https://www.mirrorservice.org/sites/ftp.apache.org/kafka/$KAFKA_VER/kafka_$SCALA_VER-$KAFKA_VER.tgz | tar -xz --strip-components=1 \
          && yum clean all
        RUN chmod -R a=u /opt/kafka
        WORKDIR /opt/kafka
        VOLUME /tmp/kafka-logs /tmp/zookeeper
        EXPOSE 2181 2888 3888 9092
      git:
        ref: master
        uri: https://github.com/mattf/openshift-kafka
      type: Git
    strategy:
      dockerStrategy:
        env:
        - name: KAFKA_VER
          value: ${KAFKA_VER}
        - name: SCALA_VER
          value: ${SCALA_VER}
        - name: JDK_VER
          value: ${JDK_VER}
        - name: RHEL_VER
          value: ${RHEL_VER}
        from:
          kind: DockerImage
          name: registry.access.redhat.com/rhel7:${RHEL_VER}
      type: Docker
    triggers:
    - type: ConfigChange
- apiVersion: v1
  kind: BuildConfig
  metadata:
    labels:
      application: ${APPLICATION_NAME}
      build: oshinko-build
    name: oshinko-build
  spec:
    output:
      to:
        kind: ImageStreamTag
        name: oshinko-webui:latest
    source:
      dockerfile: |
        FROM registry.access.redhat.com/rhel7:$RHEL_VER
        RUN INSTALL_PKGS="tar bzip2 wget git rh-nodejs6 rh-nodejs6-nodejs-nodemon rh-nodejs6-npm" && \
            yum install -y --setopt=tsflags=nodocs --enablerepo=rhel-server-rhscl-7-rpms $INSTALL_PKGS && \
            rpm -V $INSTALL_PKGS && \
            yum clean all -y
        RUN mkdir -p /usr/src/app
        WORKDIR /usr/src/app
        COPY ./package.json /usr/src/app/
        COPY ./bower.json /usr/src/app/
        COPY ./scripts/launch.sh /usr/src/app/
        RUN export CLI_REPO=crobby && \
            export CLI_VER=v0.2.x.1 && \
            pushd /tmp && \
            wget https://github.com/${CLI_REPO}/oshinko-cli/releases/download/${CLI_VER}/oshinko-cli_${CLI_VER}_linux_amd64.tar.gz && \
            tar -zxvf oshinko-cli_${CLI_VER}_linux_amd64.tar.gz && \
            mv oshinko-cli_linux_amd64 /usr/src/app/oshinko && \
            chmod +x /usr/src/app/oshinko && rm -rf /tmp/oshinko-cli* && \
            popd
        RUN echo '{ "allow_root": true, "directory": "app/bower_components" }' > /usr/src/app/.bowerrc
        RUN scl enable rh-nodejs6 "npm install &&  npm install -g bower && bower install"
        COPY . /usr/src/app
        RUN sed -i '2s#^#source /opt/rh/rh-nodejs6/enable#' /usr/src/app/launch.sh
        RUN chmod a+rwX -R .
        EXPOSE 8080
        CMD [ "/usr/src/app/launch.sh" ]
      git:
        ref: master
        uri: https://github.com/radanalyticsio/oshinko-webui
      type: Git
    strategy:
      dockerStrategy:
        env:
        - name: RHEL_VER
          value: ${RHEL_VER}
        from:
          kind: DockerImage
          name: registry.access.redhat.com/rhel7:${RHEL_VER}
      type: Docker
    triggers:
    - type: ConfigChange
- apiVersion: v1
  kind: BuildConfig
  metadata:
    labels:
      application: ${APPLICATION_NAME}
      build: spark-build
    name: spark-build
  spec:
    nodeSelector: null
    output:
      to:
        kind: ImageStreamTag
        name: spark:${SPARK_VER}
    source:
      dockerfile: |
        FROM registry.access.redhat.com/rhel7:$RHEL_VER
        USER root
        ARG DISTRO_LOC=https://archive.apache.org/dist/spark/spark-$SPARK_VER/spark-$SPARK_VER-bin-without-hadoop.tgz
        ARG DISTRO_NAME=spark-$SPARK_VER-bin-without-hadoop
        RUN yum install -y --setopt=tsflags=nodocs epel-release tar java-$JDK_VER-openjdk-headless && \
            yum clean all
        RUN cd /opt && \
            curl $DISTRO_LOC  | \
                tar -zx && \
            ln -s $DISTRO_NAME spark
        RUN yum install -y nss_wrapper numpy && yum clean all
        ENV PATH=$PATH:/opt/spark/bin
        ENV SPARK_HOME=/opt/spark
        COPY scripts /tmp/scripts
        COPY metrics /opt/spark
        RUN [ "bash", "-x", "/tmp/scripts/spark/install" ]
        RUN rm -rf /tmp/scripts
        USER 185
        WORKDIR /tmp
        ENTRYPOINT ["/entrypoint"]
        CMD ["/opt/spark/bin/launch.sh"]
      git:
        ref: master
        uri: https://github.com/radanalyticsio/openshift-spark
      type: Git
    strategy:
      dockerStrategy:
        env:
        - name: SPARK_VER
          value: ${SPARK_VER}
        - name: JDK_VER
          value: ${JDK_VER}
        - name: RHEL_VER
          value: ${RHEL_VER}
        from:
          kind: DockerImage
          name: registry.access.redhat.com/rhel7:${RHEL_VER}
      type: Docker
    triggers:
    - type: ConfigChange
- apiVersion: v1
  kind: PersistentVolumeClaim
  metadata:
    labels:
      application: ${APPLICATION_NAME}
    name: ${APPLICATION_NAME}-vol-claim
  spec:
    accessModes:
    - ReadWriteMany
    resources:
      requests:
        storage: 10Gi
    selector:
      matchLabels:
        name: ${APPLICATION_NAME}-vol
- apiVersion: v1
  kind: ServiceAccount
  metadata:
    name: oshinko
- apiVersion: v1
  kind: Service
  metadata:
    name: kafka-svc
  spec:
    ports:
    - name: kafka
      port: 9092
      protocol: TCP
      targetPort: 9092
    - name: zookeeper
      port: 2181
      protocol: TCP
      targetPort: 2181
    selector:
      deploymentConfig: kafka-deploy
    type: ClusterIP
- apiVersion: v1
  kind: Service
  metadata:
    labels:
      application: ${APPLICATION_NAME}
    name: oshinko-web
  spec:
    ports:
    - name: o-web-port
      port: 8080
      protocol: TCP
      targetPort: 8080
    selector:
      deploymentConfig: oshinko-web
    type: ClusterIP
- apiVersion: v1
  kind: RoleBinding
  metadata:
    name: edit
  roleRef:
    name: edit
  subjects:
  - kind: ServiceAccount
    name: oshinko
    namespace: ${NAMESPACE}
  userNames:
  - system:serviceaccount:${NAMESPACE}:oshinko
parameters:
- description: Namespace name.
  name: NAMESPACE
  required: true
- description: Registry Hostname (usually is docker-registry.default.svc.cluster.local)
  name: REGISTRY
  value: docker-registry.default.svc.cluster.local
- description: The name for the application.
  name: APPLICATION_NAME
  required: true
  value: spark
- description: The minor version of RHEL 7.
  name: RHEL_VER
  required: true
  value: "7.3"
- description: The version of Kafka.
  name: KAFKA_VER
  required: true
  value: "0.10.1.1"
- description: The version of Spark.
  name: SPARK_VER
  required: true
  value: "2.1.0"
- description: The version of Java JDK.
  name: JDK_VER
  required: true
  value: "1.8.0"
- description: The version of Scala.
  name: SCALA_VER
  required: true
  value: "2.11"