Big Data Platform Docker Swarm Diagrams

big_data_platform_docker_swarm_diagrams.txt

########## Overall Architecture ##########

flowchart TB

    %% Define service clusters

    subgraph Storage ["Storage Layer"]
        direction LR
        postgres["PostgreSQL<br>(Hive Metastore DB)"]
        minio["MinIO<br>S3-compatible<br>Object Storage"]
        airflow_postgresql["PostgreSQL<br>(Airflow DB)"]
        airflow_redis["Redis<br>(Airflow Queue)"]
    end

    subgraph DataProcessing ["Data Processing Layer"]
        direction LR
        spark_master["Spark Master"]
        spark_worker["Spark Worker<br>(2 replicas)"]
        trino_coordinator["Trino Coordinator"]
        trino_worker["Trino Worker<br>(2 replicas)"]
        hive_metastore["Hive Metastore"]
    end

    subgraph Orchestration ["Orchestration Layer"]
        direction LR
        airflow_scheduler["Airflow Scheduler"]
        airflow_webserver["Airflow Webserver"]
        airflow_worker["Airflow Worker<br>(2 replicas)"]
    end

    subgraph Interface ["Interface Layer"]
        direction LR
        jupyter["Jupyter Lab"]
        superset["Superset"]
    end

    subgraph BucketCreation ["Bucket Creation Services"]
        direction LR
        createbucket["createbucket<br>(data bucket)"]
        createspoolingbucket["createspoolingbucket<br>(spooling bucket)"]
        createdagsbucket["createdagsbucket<br>(dags bucket)"]
    end

    subgraph Volumes ["Persistent Volumes"]
        direction LR
        minio_data[(minio_data)]
        postgresql_data[(postgresql_data)]
        hive_metastore_db[(hive_metastore_db)]
        airflow_scheduler_data[(airflow_scheduler_data)]
        airflow_webserver_data[(airflow_webserver_data)]
        airflow_worker_data[(airflow_worker_data)]
        s3data[(s3data)]
        superset_home[(superset_home)]
    end

    %% S3 Volume Service
    s3vol["s3vol<br>S3-to-Volume Sync"]

    %% Connect Storage to Volumes
    minio --> minio_data
    postgres --> hive_metastore_db
    airflow_postgresql --> postgresql_data

    %% Bucket creation connections
    minio --> BucketCreation

    %% S3Vol connections
    minio --> s3vol
    createdagsbucket --> s3vol
    s3vol --> s3data

    %% Connect data volumes to services
    s3data --> airflow_scheduler
    s3data --> airflow_webserver
    s3data --> airflow_worker
    airflow_scheduler --> airflow_scheduler_data
    airflow_webserver --> airflow_webserver_data
    airflow_worker --> airflow_worker_data
    superset --> superset_home

    %% Connect Processing services
    spark_master <--> spark_worker
    trino_coordinator <--> trino_worker
    postgres --> hive_metastore

    %% Connect Orchestration services
    airflow_postgresql --> Orchestration
    airflow_redis --> Orchestration

    %% Connect Interface services
    spark_master --> jupyter
    minio --> jupyter

    %% Other dependencies
    spark_master --> |hive-site.xml| hive_metastore
    createspoolingbucket --> trino_coordinator

    %% AWS/S3 environment connections
    minio --> |AWS credentials<br>in environment| jupyter
    minio --> |AWS_ENDPOINT_URL| Orchestration

    classDef storageNode fill:#f9d,stroke:#333,stroke-width:2px
    classDef processingNode fill:#bbf,stroke:#333,stroke-width:2px
    classDef orchestrationNode fill:#bfb,stroke:#333,stroke-width:2px
    classDef interfaceNode fill:#fbb,stroke:#333,stroke-width:2px
    classDef volumeNode fill:#ddd,stroke:#333,stroke-width:1px,stroke-dasharray: 5 5
    classDef bucketServiceNode fill:#ffd,stroke:#333,stroke-width:2px
    classDef syncNode fill:#efe,stroke:#333,stroke-width:2px

    class minio,postgres,airflow_postgresql,airflow_redis storageNode
    class spark_master,spark_worker,trino_coordinator,trino_worker,hive_metastore processingNode
    class airflow_scheduler,airflow_webserver,airflow_worker orchestrationNode
    class jupyter,superset interfaceNode
    class minio_data,postgresql_data,hive_metastore_db,airflow_scheduler_data,airflow_webserver_data,airflow_worker_data,s3data,superset_home volumeNode
    class createbucket,createspoolingbucket,createdagsbucket bucketServiceNode
    class s3vol,BucketCreation syncNode

########## Dependencies ##########

graph TD
    %% Core Services (no dependencies)
    minio[Minio]
    airflow-postgresql[Airflow-PostgreSQL]
    airflow-redis[Airflow-Redis]
    postgres[Postgres]
    spark-master[Spark-Master]

    %% Bucket Creation Services (depend on Minio)
    createspoolingbucket[Createspoolingbucket] --> minio
    createbucket[Createbucket] --> minio
    createdagsbucket[Createdagsbucket] --> minio

    %% S3 Volume Service (depends on Minio and Createdagsbucket)
    s3vol[S3vol] --> minio
    s3vol --> createdagsbucket

    %% Airflow Services (depend on Airflow-PostgreSQL and Airflow-Redis)
    airflow-scheduler[Airflow-Scheduler] --> airflow-postgresql
    airflow-scheduler --> airflow-redis
    airflow-webserver[Airflow-Webserver] --> airflow-postgresql
    airflow-webserver --> airflow-redis
    airflow-worker[Airflow-Worker] --> airflow-postgresql
    airflow-worker --> airflow-redis

    %% Hive Metastore (depends on Postgres)
    hive-metastore[Hive-Metastore] --> postgres

    %% Trino Services (depend on Createspoolingbucket and Trino-Coordinator)
    trino-coordinator[Trino-Coordinator] --> createspoolingbucket
    trino-worker[Trino-Worker] --> trino-coordinator

    %% Jupyter (depends on Spark-Master and Minio)
    jupyter[Jupyter] --> spark-master
    jupyter --> minio

    %% Superset (no dependencies)
    superset[Superset]

    %% Spark Worker (no explicit depends_on, but logically tied to Spark-Master)
    spark-worker[Spark-Worker]

    %% Styling for clarity
    classDef core fill:#f9f,stroke:#333,stroke-width:2px;
    classDef bucket fill:#bbf,stroke:#333,stroke-width:2px;
    classDef airflow fill:#bfb,stroke:#333,stroke-width:2px;
    classDef trino fill:#fbf,stroke:#333,stroke-width:2px;
    class minio,airflow-postgresql,airflow-redis,postgres,spark-master core;
    class createspoolingbucket,createbucket,createdagsbucket,s3vol bucket;
    class airflow-scheduler,airflow-webserver,airflow-worker airflow;
    class trino-coordinator,trino-worker trino;

########## Mounts ##########

flowchart TD
    %% Host Machine Physical Storage
    subgraph "Host Machine Physical Storage"
        direction LR
        
        local_volumes["Docker Named Volumes<br>(Local Driver)"] --> |"stored in<br>/var/lib/docker/volumes/"| minio_data[("minio_data")]
        local_volumes --> postgresql_data[("postgresql_data")]
        local_volumes --> hive_metastore_db[("hive_metastore_db")]
        local_volumes --> airflow_scheduler_data[("airflow_scheduler_data")]
        local_volumes --> airflow_webserver_data[("airflow_webserver_data")]
        local_volumes --> airflow_worker_data[("airflow_worker_data")]
        local_volumes --> spark_history_logs[("spark_history_logs")]
        local_volumes --> s3data[("s3data")]
        local_volumes --> superset_home[("superset_home")]
        
        host_directories["Host Machine Directories<br>(Bind Mounted)"] --> notebooks_dir["./notebooks<br>directory"]
        host_directories --> spark_configs["./configs/spark/<br>spark-defaults.conf"]
        host_directories --> hive_configs["./Dockerfiles/hive-metastore/conf/<br>hive-site.xml"]
        host_directories --> trino_coord_configs["./configs/trino/coordinator/etc<br>directory"]
        host_directories --> trino_worker_configs["/bigDataPlatform/configs/trino/worker/etc<br>directory"]
    end
    
    %% MinIO Object Storage
    subgraph "MinIO Object Storage"
        direction LR
        
        object_storage["S3-Compatible<br>Virtual Buckets"] --> dags_bucket["s3://dags"]
        object_storage --> data_bucket["s3://data"]
        object_storage --> spooling_bucket["s3://spooling"]
    end
    
    %% Services with Mounted Directories
    minio_service["Minio Service<br>(/data)"]
    s3vol_service["S3vol Service<br>(/data)"]
    airflow_scheduler_service["Airflow-Scheduler<br>(/bitnami, /opt/bitnami/airflow/dags)"]
    airflow_webserver_service["Airflow-Webserver<br>(/bitnami, /opt/bitnami/airflow/dags)"]
    airflow_worker_service["Airflow-Worker<br>(/bitnami, /opt/bitnami/airflow/dags)"]
    airflow_postgresql_service["Airflow-PostgreSQL<br>(/bitnami/postgresql)"]
    postgres_service["Postgres<br>(/var/lib/postgresql/data)"]
    jupyter_service["Jupyter<br>(/home/jovyan/work)"]
    spark_master_service["Spark-Master<br>(/opt/bitnami/spark/conf/spark-defaults.conf,<br>/opt/bitnami/spark/conf/hive-site.xml)"]
    trino_coordinator_service["Trino-Coordinator<br>(/etc/trino)"]
    trino_worker_service["Trino-Worker<br>(/etc/trino)"]
    superset_service["Superset<br>(/app/superset_home)"]

    %% Connections between systems
    minio_data --> |"mounted to"| minio_service
    minio_data --> |"stores objects for"| object_storage
    s3data --> |"synced with"| dags_bucket
    s3data --> |"mounted to"| s3vol_service
    s3data --> |"mounted to"| airflow_scheduler_service
    s3data --> |"mounted to"| airflow_webserver_service
    s3data --> |"mounted to"| airflow_worker_service
    airflow_scheduler_data --> |"mounted to"| airflow_scheduler_service
    airflow_webserver_data --> |"mounted to"| airflow_webserver_service
    airflow_worker_data --> |"mounted to"| airflow_worker_service
    postgresql_data --> |"mounted to"| airflow_postgresql_service
    hive_metastore_db --> |"mounted to"| postgres_service
    superset_home --> |"mounted to"| superset_service
    
    notebooks_dir --> |"bind mounted to"| jupyter_service
    spark_configs --> |"bind mounted to"| spark_master_service
    hive_configs --> |"bind mounted to"| spark_master_service
    trino_coord_configs --> |"bind mounted to"| trino_coordinator_service
    trino_worker_configs --> |"bind mounted to"| trino_worker_service
    
    %% Volume driver specification
    volume_driver["All volumes use 'local' driver<br>specified in docker-compose.yml:<br>volumes:<br>  minio_data:<br>    driver: local"]
    volume_driver --- local_volumes
    
    %% Notes and clarifications
    note1["S3 buckets (dags, data, spooling)<br>are virtual and stored inside minio_data"]
    note1 --- object_storage
    
    note2["./notebooks, ./configs/, and<br>/bigDataPlatform/configs/<br>are actual directories in<br>your project folder"]
    note2 --- host_directories
    
    %% Excluded mounts and services
    excluded["Excluded from Diagram:<br>- Volume: spark_history_logs<br><br>  Reason: Defined but not mounted to any service<br><br>- Services: createspoolingbucket, createbucket,<br>  createdagsbucket, spark-worker, airflow-redis,<br>  hive-metastore<br><br>  Reason: No volume or bind mounts specified"]
    
    
    %% Styling for clarity
    classDef localVolume fill:#f9d,stroke:#333,stroke-width:2px
    classDef objectStorage fill:#bfb,stroke:#333,stroke-width:2px
    classDef hostDir fill:#fbb,stroke:#333,stroke-width:2px
    classDef service fill:#bbf,stroke:#333,stroke-width:2px
    classDef note fill:#eee,stroke:#333,stroke-width:1px,stroke-dasharray: 5 5
    
    class minio_data,postgresql_data,hive_metastore_db,airflow_scheduler_data,airflow_webserver_data,airflow_worker_data,spark_history_logs,s3data,superset_home localVolume
    class object_storage,dags_bucket,data_bucket,spooling_bucket objectStorage
    class notebooks_dir,spark_configs,hive_configs,trino_coord_configs,trino_worker_configs hostDir
    class minio_service,s3vol_service,airflow_scheduler_service,airflow_webserver_service,airflow_worker_service,airflow_postgresql_service,postgres_service,jupyter_service,spark_master_service,trino_coordinator_service,trino_worker_service,superset_service service
    class note1,note2,volume_driver,excluded note

########## Sync Sequence ##########

sequenceDiagram
    participant Airflow as Airflow Scheduler/Worker
    participant S3Vol as s3vol (elementar/s3-volume)
    participant MinIO as minio (S3 Bucket: dags)
    participant LocalVol as Local Volume (s3data:/data)

    Note over S3Vol: Container Startup (--force-restore)
    S3Vol->>MinIO: GET s3://dags → /data (initial sync)
    MinIO-->>S3Vol: DAG files (e.g., pipeline.py)
    S3Vol->>LocalVol: Write to /data (mounted to s3data)

    loop During Runtime
        Airflow->>LocalVol: Read/write DAGs (e.g., /opt/bitnami/airflow/dags)
        Note over Airflow,LocalVol: Changes cached locally (not synced yet)
    end

    Note over S3Vol: Graceful Shutdown (or BACKUP_INTERVAL=2m)
    S3Vol->>LocalVol: Scan for changes in /data
    S3Vol->>MinIO: PUT modified files → s3://dags
    MinIO-->>S3Vol: 200 OK (sync confirmed)

########## Env ##########

graph LR
    %% Define node styles
    classDef storage fill:#f9d6a5,stroke:#e69138,stroke-width:2px
    classDef service fill:#d5e8d4,stroke:#82b366,stroke-width:2px
    classDef config fill:#d0e0ff,stroke:#3c78d8,stroke-width:2px
    classDef secrets fill:#ff9999,stroke:#cc0000,stroke-width:2px
    
    %% MinIO Configuration
    minio["MinIO"]:::storage
    
    minio_creds["MINIO_ROOT_USER: minioadmin<br>MINIO_ROOT_PASSWORD: minioadmin"]:::secrets
    
    minio --- minio_creds
    
    %% S3 Volume Sync
    s3vol["S3 Volume Sync"]:::service
    
    s3vol_config["BACKUP_INTERVAL: 2m<br>AWS_ACCESS_KEY_ID: minioadmin<br>AWS_SECRET_ACCESS_KEY: minioadmin<br>ENDPOINT_URL: http://minio:9000"]:::config
    
    s3vol --- s3vol_config
    
    %% Spark Configuration
    spark_master["Spark Master"]:::service
    
    spark_master_config["SPARK_MODE: master<br>SPARK_RPC_AUTHENTICATION_ENABLED: no<br>SPARK_RPC_ENCRYPTION_ENABLED: no<br>SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED: no<br>SPARK_SSL_ENABLED: no"]:::config
    
    spark_master --- spark_master_config
    
    %% Spark Worker
    spark_worker["Spark Workers"]:::service
    
    spark_worker_config["SPARK_MODE: worker<br>SPARK_MASTER_URL: spark://spark-master:7077<br>SPARK_WORKER_MEMORY: 2G<br>SPARK_WORKER_CORES: 2<br>SPARK_RPC_AUTHENTICATION_ENABLED: no<br>SPARK_RPC_ENCRYPTION_ENABLED: no<br>SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED: no<br>SPARK_SSL_ENABLED: no"]:::config
    
    spark_worker --- spark_worker_config
    
    %% Jupyter Configuration
    jupyter["Jupyter Lab"]:::service
    
    jupyter_config["JUPYTER_ENABLE_LAB: yes<br>SPARK_MASTER: spark://spark-master:7077<br>PYSPARK_PYTHON: /usr/bin/python3<br>PYSPARK_DRIVER_PYTHON: /usr/bin/python3<br>AWS_ACCESS_KEY_ID: minioadmin<br>AWS_SECRET_ACCESS_KEY: minioadmin<br>AWS_ENDPOINT_URL: http://minio:9000"]:::config
    
    jupyter --- jupyter_config
    
    %% Airflow Configuration
    airflow_webserver["Airflow Webserver"]:::service
    
    airflow_shared_config["AIRFLOW_DATABASE_HOST: airflow-postgresql<br>AIRFLOW_DATABASE_PORT_NUMBER: 5431<br>AIRFLOW_DATABASE_NAME: bitnami_airflow<br>AIRFLOW_DATABASE_USERNAME: bn_airflow<br>AIRFLOW_DATABASE_PASSWORD: bitnami1<br>AIRFLOW_EXECUTOR: CeleryExecutor<br>REDIS_HOST: airflow-redis<br>REDIS_PORT_NUMBER: 6379<br>AIRFLOW_LOAD_EXAMPLES: yes<br>AIRFLOW_USERNAME: user<br>AIRFLOW_PASSWORD: bitnami<br>AWS_ENDPOINT_URL: http://minio-ingress:9000"]:::config
    
    %% PostgreSQL Configuration
    airflow_postgresql["PostgreSQL<br>(Airflow)"]:::storage
    
    postgresql_config["POSTGRESQL_PORT_NUMBER: 5431<br>POSTGRESQL_DATABASE: bitnami_airflow<br>POSTGRESQL_USERNAME: bn_airflow<br>POSTGRESQL_PASSWORD: bitnami1"]:::secrets
    
    airflow_postgresql --- postgresql_config
    
    %% Hive Metastore PostgreSQL
    postgres["PostgreSQL<br>(Hive)"]:::storage
    
    postgres_config["POSTGRES_USER: hive<br>POSTGRES_PASSWORD: hive123<br>POSTGRES_DB: metastore_db"]:::secrets
    
    postgres --- postgres_config
    
    %% Redis Configuration
    airflow_redis["Redis"]:::storage
    
    redis_config["ALLOW_EMPTY_PASSWORD: yes"]:::config
    
    airflow_redis --- redis_config
    
    %% Superset Configuration
    superset["Superset"]:::service
    
    superset_config["SUPERSET_PORT: 9777<br>ADMIN_USERNAME: admin<br>ADMIN_FIRSTNAME: Superset<br>ADMIN_LASTNAME: Admin<br>ADMIN_EMAIL: admin@superset.com<br>ADMIN_PASSWORD: admin<br>LOAD_EXAMPLES: true"]:::config
    
    superset --- superset_config
    
    %% Airflow Component Types
    airflow_webserver --- airflow_shared_config
    airflow_webserver_type["AIRFLOW_COMPONENT_TYPE: webserver"]:::config
    airflow_webserver --- airflow_webserver_type
    
    airflow_scheduler["Airflow Scheduler"]:::service
    airflow_scheduler --- airflow_shared_config
    airflow_scheduler_type["AIRFLOW_COMPONENT_TYPE: scheduler"]:::config
    airflow_scheduler --- airflow_scheduler_type
    
    airflow_worker["Airflow Workers"]:::service
    airflow_worker --- airflow_shared_config
    airflow_worker_type["AIRFLOW_COMPONENT_TYPE: worker"]:::config
    airflow_worker --- airflow_worker_type

########## Network ##########

flowchart LR
    subgraph "Storage Services"
        minio["MinIO (S3-compatible Storage)"]
        minio -.-> |9000: API Endpoint| port9000[Port 9000]
        minio -.-> |9001: Web Console| port9001[Port 9001]
    end

    subgraph "Spark Services"
        spark["Spark Master"]
        spark -.-> |8080: Web UI| port8080[Port 8080]
        spark -.-> |7077: Spark Job Submission| port7077[Port 7077]
        spark -.-> |4040: Spark Application UI| port4040[Port 4040]
        
        sparkw["Spark Workers"]
        sparkw -.-> |8081-8082: Worker Web UIs| port8081_8082[Ports 8081-8082]
    end

    subgraph "User Interfaces"
        jupyter["Jupyter Lab"]
        jupyter -.-> |8888: Jupyter Web UI| port8888[Port 8888]
        
        airflow["Airflow Webserver"]
        airflow -.-> |8090: Airflow UI| port8090[Port 8080]
        
        superset["Superset"]
        superset -.-> |8089: Superset UI| port8089[Port 9777]
        
        trino["Trino Coordinator"]
        trino -.-> |9090: Trino UI| port9090[Port 9090]
    end

    subgraph "Metadata Services"
        hive["Hive Metastore"]
        hive -.-> |9083: Thrift Service| port9083[Port 9083]
        
        postgres["PostgreSQL"]
        postgres -.-> |5431: PostgreSQL for Airflow| port5431[Port 5431]
        
        redis["Redis"]
        redis -.-> |6379: Redis for Airflow| port6379[Port 6379]
    end

    class port9000,port9001,port8080,port7077,port4040,port8081_8082,port8888,port8090,port8089,port9090,port9083,port5431,port6379 port

    classDef port fill:#f9f,stroke:#333,stroke-width:1px