c4 model data platform

gistfile1.txt

workspace {
    model {
        # People/Actors
        businessUser = person "Business User" "Consumes data insights and reports"
        dataEngineer = person "Data Engineer" "Develops and maintains data pipelines"
        dataScientist = person "Data Scientist" "Develops ML models and performs advanced analytics"
        dataAnalyst = person "Data Analyst" "Performs data analysis and creates reports"

        group "Data Platform" {
            # Software Systems - External
            aws = softwareSystem "AWS" "Cloud services and storage" "External"
            azure = softwareSystem "Azure" "Cloud services and storage" "External"
            gcp = softwareSystem "GCP" "Cloud services and storage" "External"
            onPrem = softwareSystem "On-Premises Systems" "Legacy data systems" "External"

            # Main System
            dataPlatform = softwareSystem "Enterprise Data Platform" "Centralizes data management and processing" {
                # Containers
                apiGateway = container "API Gateway" "Kong" "Manages API access and security"
                dataCatalog = container "Data Catalog" "DataHub" "Metadata management and data discovery"
                dataLake = container "Data Lake" "Delta Lake" "Raw and processed data storage"
                dataWarehouse = container "Data Warehouse" "Snowflake" "Business-ready data storage"
                pipelineService = container "Pipeline Service" "Apache Airflow" "Orchestrates data workflows" {
                    # Components
                    scheduler = component "Pipeline Scheduler" "Schedules and triggers pipeline execution"
                    executor = component "Task Executor" "Executes pipeline tasks"
                    connectors = component "Data Connectors" "Connects to various data sources"
                    transformer = component "Data Transformer" "Apache Spark" "Processes and transforms data"
                    validator = component "Data Validator" "Great Expectations" "Validates data quality"
                    logging = component "Logging Service" "ELK Stack" "Handles pipeline logging"
                    configManager = component "Configuration Manager" "Manages pipeline configurations"
                }
                dataGovernance = container "Data Governance" "Ensures data compliance and quality"
                monitoring = container "Monitoring" "Prometheus & Grafana" "System monitoring and alerting"
                security = container "Security Service" "Handles authentication and authorization"
            }
        }

        # Relationships - People to System
        businessUser -> dataPlatform "Consumes data insights"
        dataEngineer -> dataPlatform "Manages data pipelines"
        dataScientist -> dataPlatform "Performs analytics"
        dataAnalyst -> dataPlatform "Analyzes data"

        # Relationships - External Systems
        dataPlatform -> aws "Reads/writes data"
        dataPlatform -> azure "Reads/writes data"
        dataPlatform -> gcp "Reads/writes data"
        dataPlatform -> onPrem "Reads/writes data"

        # Relationships - Containers
        apiGateway -> security "Authenticates requests"
        apiGateway -> dataWarehouse "Routes queries"
        apiGateway -> dataCatalog "Routes metadata requests"
        
        pipelineService -> dataLake "Reads/writes data"
        pipelineService -> dataWarehouse "Loads processed data"
        pipelineService -> dataCatalog "Updates metadata"
        
        monitoring -> pipelineService "Monitors"
        dataGovernance -> dataLake "Enforces policies"
        dataGovernance -> dataWarehouse "Enforces policies"

        # Relationships - Components
        scheduler -> executor "Triggers tasks"
        executor -> connectors "Uses"
        executor -> transformer "Uses"
        executor -> validator "Uses"
        executor -> logging "Logs events"
        executor -> configManager "Reads configuration"
        
        connectors -> dataLake "Reads/writes data"
        transformer -> dataLake "Processes data"
        validator -> dataLake "Validates data"
    }

    views {
        systemContext dataPlatform "Context" {
            include *
            autoLayout tb
        }

        container dataPlatform "Containers" {
            include *
            autoLayout tb
        }

        component pipelineService "Components" {
            include *
            autoLayout tb
        }

        styles {
            element "Person" {
                shape Person
                background #08427b
            }
            element "External" {
                background #999999
            }
            element "Software System" {
                background #1168bd
            }
            element "Container" {
                background #438dd5
            }
            element "Component" {
                background #85bbf0
            }
        }
    }
}