arkadius

#!/bin/bash

set -e

RESULT=$(psql -tc "SELECT 1 FROM pg_database WHERE datname = '$DB';")

if [ -z "$RESULT" ]; then
 echo "Database $DB does not exist. Creating..."
 psql -v ON_ERROR_STOP=1 <<-EOSQL
   CREATE USER $USER WITH ENCRYPTED PASSWORD '$PASSWORD';

arkadius

nessie:
  image: projectnessie/nessie:0.76.6
  depends_on:
    postgres:
      condition: service_healthy
  networks:
    - nussknacker-iceberg-net 
  ports:
    - 19120:19120
  environment:

arkadius

catalogConfiguration {
  "type": "iceberg",
  "catalog-impl": "org.apache.iceberg.nessie.NessieCatalog"
  "io-impl": "org.apache.iceberg.aws.s3.S3FileIO"
  "uri": "http://nessie:19120/api/v2"
  "warehouse": "s3://warehouse"
  "s3.endpoint": "http://minio:9000"
}

arkadius

ARG NUSSKNACKER_VERSION

FROM touk/nussknacker:${NUSSKNACKER_VERSION}_scala-2.12

ARG FLINK_VERSION
ARG FLINK_VERSION_SHORT
ARG HADOOP_VERSION
ARG FLINK_HADOOP_VERSION
ARG AWS_SDK_VERSION
ARG ICEBERG_VERSION

arkadius

build:
 context: ./designer/
 args:
   NUSSKNACKER_VERSION: ${NUSSKNACKER_VERSION:?NUSSKNACKER_VERSION must be defined}
   FLINK_VERSION: "1.19.1-scala_2.12-java11"
   FLINK_VERSION_SHORT: 1.19
   HADOOP_VERSION: 2.8.3
   FLINK_HADOOP_VERSION: "2.8.3-10.0"
   AWS_SDK_VERSION: 2.26.10
   ICEBERG_VERSION: 1.6.1

arkadius

AWS_ACCESS_KEY_ID: admin
AWS_SECRET_ACCESS_KEY: password
AWS_REGION: us-east-1
AWS_DEFAULT_REGION: us-east-1

arkadius

ARG FLINK_VERSION_SHORT
ARG HADOOP_VERSION
ARG FLINK_HADOOP_VERSION
ARG AWS_SDK_VERSION
ARG ICEBERG_VERSION

RUN curl -L "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-common/${HADOOP_VERSION}/hadoop-common-${HADOOP_VERSION}.jar" \
   -o ${FLINK_HOME}/lib/hadoop-common-${HADOOP_VERSION}.jar

RUN curl -L "https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/${FLINK_HADOOP_VERSION}/flink-shaded-hadoop-2-uber-${FLINK_HADOOP_VERSION}.jar" \

arkadius

classloader.parent-first-patterns.additional: com.codahale.metrics;org.apache.iceberg;org.projectnessie

arkadius

docker compose exec flink-jobmanager sql-client.sh

arkadius

CREATE CATALOG nessie_catalog WITH (
  'type'='iceberg',
  'catalog-impl'='org.apache.iceberg.nessie.NessieCatalog',
  'io-impl'='org.apache.iceberg.aws.s3.S3FileIO',
  'uri'='http://nessie:19120/api/v2',
  'warehouse' = 's3://warehouse',
  's3.endpoint'='http://minio:9000'
);

USE CATALOG nessie_catalog;