clap + Figment (cli + config file + env variables) in rust

gistfile1.txt

# clap + Figment

for command line switches like: program --help, program --input "asdf" i want to use `clap` and for config file 
parsing like `pankat.toml` i want to use Figment.

after studying https://steezeburger.com/2023/03/rust-hierarchical-configuration/ i came up with my solution, 
which is quite similar, except it splits the clap arguments into default values and explicit values for the later merge.

now i can either use:

* default values from clap
* toml file configuration values
* environment variable values (not tested in this code)
* explicit values from clap

enjoy!

# pankat.toml

input = "documents/blog.lastlog.de"
output = "documents/output"
assets = "documents/assets"
wasm = "documents/wasm"
database = "documents"
brand = "lastlog.de/blog"
port = 5000
static_build_only = false
flat = false

# main.rs

mod articles;
mod auth;
mod config;
mod db;
mod error;
mod file_monitor;
mod handlers;
mod registry;
mod renderer;
use crate::config::*;
use crate::renderer::pandoc::check_pandoc;
use axum::{
    routing::{get, post},
    Router,
};
use clap::{Arg, ArgAction, Command};
use colored::Colorize;
use diesel_migrations::{embed_migrations, EmbeddedMigrations};
use std::collections::HashMap;
use tokio::signal;
use tokio::sync::broadcast;
use tower_http::cors::CorsLayer;

pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    tracing_subscriber::fmt::init();
    let matches = Command::new("pankat")
        .version(env!("CARGO_PKG_VERSION"))
        .author("Joachim Schiele <[email protected]>")
        .about("https://github.com/nixcloud/pankat - static site generator")
        .arg(
            Arg::new("input")
                .short('i')
                .long("input")
                .value_name("PATH")
                .help("Absolute path where the media/*.jpg and posts/*.md files of your blog are located")
        )
        .arg(
            Arg::new("output")
                .short('o')
                .long("output")
                .value_name("PATH")
                .help("Absolute path, where pankat 'maintains' the generated html files by adding/deleting/updating them")
                .default_value("documents/output")
        )
        .arg(
            Arg::new("assets")
                .short('a')
                .long("assets")
                .value_name("PATH")
                .help("An absolute assets path, where js/wasm/css/templates/lua/... files are stored")
                .default_value("documents/assets")
        )
        .arg(
            Arg::new("wasm")
                .short('w')
                .long("wasm")
                .help("The bundled pankat-wasm executable built from rust")
                .value_name("PATH")
                .default_value("documents/wasm")
        )
        .arg(
            Arg::new("database")
                .short('d')
                .long("database")
                .value_name("PATH")
                .help("An absolute path where 'only' the database is stored (don't put this into output!)")
                .default_value("documents")
        )
        .arg(
            Arg::new("brand")
                .short('b')
                .long("brand")
                .value_name("URL")
                .help("A brand name shown on every page top left")
                .required(false)
                .default_value("lastlog.de/blog")
        )
        .arg(
            Arg::new("static")
                .short('s')
                .long("static")
                .help("Only build documents and exit (static blog generator)")
                .required(false)
                .action(ArgAction::SetTrue)
        )
        .arg(
            Arg::new("flat")
                .short('f')
                .long("flat")
                .help("Flatten the output directory like (foo/bar.mdwn -> bar.html)")
                .required(false)
                .action(ArgAction::SetTrue)
        )
        .arg(
            Arg::new("port")
                .short('p')
                .long("port")
                .value_name("PORT")
                .help("Port number where pankat listens for incoming connections for browser connections")
                .default_value("5000"),
        )
        .get_matches();

    let mut config_values: HashMap<String, ConfigValue> = HashMap::new();

    config_values.insert(
        "input".to_string(),
        ConfigValue {
            value: ConfigValueType::Path(
                matches
                    .get_one::<String>("input")
                    .map(|v| std::path::Path::new(v).into()),
            ),
            is_default: Some(clap::parser::ValueSource::DefaultValue)
                == matches.value_source("input"),
        },
    );

    config_values.insert(
        "output".to_string(),
        ConfigValue {
            value: ConfigValueType::Path(
                matches
                    .get_one::<String>("output")
                    .map(|v| std::path::Path::new(v).into()),
            ),
            is_default: Some(clap::parser::ValueSource::DefaultValue)
                == matches.value_source("output"),
        },
    );

    config_values.insert(
        "assets".to_string(),
        ConfigValue {
            value: ConfigValueType::Path(
                matches
                    .get_one::<String>("assets")
                    .map(|v| std::path::Path::new(v).into()),
            ),
            is_default: Some(clap::parser::ValueSource::DefaultValue)
                == matches.value_source("assets"),
        },
    );

    config_values.insert(
        "wasm".to_string(),
        ConfigValue {
            value: ConfigValueType::Path(
                matches
                    .get_one::<String>("wasm")
                    .map(|v| std::path::Path::new(v).into()),
            ),
            is_default: Some(clap::parser::ValueSource::DefaultValue)
                == matches.value_source("wasm"),
        },
    );

    config_values.insert(
        "database".to_string(),
        ConfigValue {
            value: ConfigValueType::Path(
                matches
                    .get_one::<String>("database")
                    .map(|v| std::path::Path::new(v).into()),
            ),
            is_default: Some(clap::parser::ValueSource::DefaultValue)
                == matches.value_source("database"),
        },
    );

    config_values.insert(
        "brand".to_string(),
        ConfigValue {
            value: ConfigValueType::String(matches.get_one::<String>("brand").map(|v| v.into())),
            is_default: Some(clap::parser::ValueSource::DefaultValue)
                == matches.value_source("brand"),
        },
    );

    config_values.insert(
        "port".to_string(),
        ConfigValue {
            value: ConfigValueType::Number(
                matches
                    .get_one::<String>("port")
                    .map(|port| port.parse::<u16>().unwrap()),
            ),
            is_default: Some(clap::parser::ValueSource::DefaultValue)
                == matches.value_source("port"),
        },
    );

    config_values.insert(
        "static".to_string(),
        ConfigValue {
            value: ConfigValueType::Bool(matches.get_one::<bool>("static").copied()),
            is_default: Some(clap::parser::ValueSource::DefaultValue)
                == matches.value_source("static"),
        },
    );

    config_values.insert(
        "flat".to_string(),
        ConfigValue {
            value: ConfigValueType::Bool(matches.get_one::<bool>("flat").copied()),
            is_default: Some(clap::parser::ValueSource::DefaultValue)
                == matches.value_source("flat"),
        },
    );

    let config = config::Config::new(config_values);

    config::Config::initialize(config).expect("Failed to initialize config");
    let cfg = config::Config::get();

    println!("-------------------------------------------------");
    println!("Input Path: {}", cfg.input.display());
    println!("Output Path: {}", cfg.output.display());
    println!("Assets Path: {}", cfg.assets.display());
    println!("WASM Path: {}", cfg.wasm.display());
    println!("Database Path: {}", cfg.database.display());
    println!("Port Number: {}", cfg.port);
    println!("Brand: {}", cfg.brand);
    println!("Static build only: {}", cfg.static_build_only);
    println!("Flat filename structure: {}", cfg.flat);
    println!("-------------------------------------------------");

    check_pandoc()?;

    // Initialize SQLite database with Diesel
    let pool = db::establish_connection_pool();

    articles::collect_garbage(&pool);
    articles::scan_articles(&pool);
    articles::build_articles(&pool);

    if cfg.static_build_only {
        println!("Static build only, exiting...");
        return Ok(());
    }

    // Setup broadcast channel for shutdown coordination
    let (shutdown_tx, _) = broadcast::channel::<()>(1);

    // FIXME adapt this concept for with_state
    // struct AppState {
    //     pool: Pool<ConnectionManager<SqliteConnection>>,
    // }
    // https://docs.rs/axum/latest/axum/#sharing-state-with-handlers

    // Initialize file monitoring
    let monitor_handle =
        file_monitor::spawn_async_monitor(pool.clone(), cfg.input.clone(), shutdown_tx.subscribe())
            .map_err(|e| Box::<dyn std::error::Error + Send + Sync>::from(e))?;

    // Create router
    let app = Router::new()
        .route("/posts/*path", get(handlers::serve_input))
        .route("/media/*path", get(handlers::serve_input))
        .route("/assets/*path", get(handlers::serve_internals))
        .route("/wasm/*path", get(handlers::serve_internals))
        .route("/api/auth/register", post(handlers::register))
        .route("/api/auth/login", post(handlers::login))
        .route("/api/protected", get(handlers::protected))
        .route("/api/ws", get(handlers::websocket_route))
        .route("/", get(handlers::serve_output))
        .route("/*path", get(handlers::serve_output))
        .layer(CorsLayer::permissive())
        .with_state(pool.clone());

    // Start server
    let address_config = format!("[::]:{}", cfg.port);
    let addr = address_config.parse::<std::net::SocketAddr>().unwrap();

    // Create a listener with retry logic
    let listener = match tokio::net::TcpListener::bind(addr).await {
        Ok(listener) => {
            let l = format!("Listening on: {}", address_config);
            println!("{}", l.green());
            listener
        }
        Err(e) => {
            eprintln!("Failed to bind to port {}: {}", cfg.port, e);
            // If port is in use, try to clean up and exit
            println!("Initiating cleanup sequence...");
            if let Err(send_err) = shutdown_tx.send(()) {
                eprintln!("Error broadcasting shutdown signal: {}", send_err);
            }
            // Wait for monitor to cleanup
            if let Err(e) = monitor_handle.await {
                eprintln!("Error during monitor shutdown: {}", e);
            }
            return Err(Box::<dyn std::error::Error + Send + Sync>::from(e));
        }
    };

    // Start server with graceful shutdown
    let s = "Press Ctrl+C to stop the server...".yellow();
    println!("{s}");
    tokio::select! {
        result = axum::serve(listener, app) => {
            if let Err(e) = result {
                eprintln!("Server error: {}", e);
                // Initiate cleanup before returning error
                if let Err(send_err) = shutdown_tx.send(()) {
                    eprintln!("Error broadcasting shutdown signal: {}", send_err);
                }
            }
        }
        _ = signal::ctrl_c() => {
            println!("\nReceived Ctrl+C, initiating graceful shutdown...");
            // Broadcast shutdown signal to all tasks
            if let Err(e) = shutdown_tx.send(()) {
                eprintln!("Error broadcasting shutdown signal: {}", e);
            }
        }
    }

    // Wait for the file monitor to complete its cleanup
    println!("Waiting for file monitor to complete shutdown...");
    if let Err(e) = monitor_handle.await {
        eprintln!("Error during monitor shutdown: {}", e);
        return Err(Box::<dyn std::error::Error + Send + Sync>::from(e));
    }
    println!("Graceful shutdown complete");

    Ok(())
}

# config.rs

use clap::Parser;
use figment::{
    providers::{Env, Format, Serialized, Toml},
    Figment,
};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, OnceLock};

pub enum ConfigValueType {
    Path(Option<std::path::PathBuf>),
    String(Option<String>),
    Number(Option<u16>),
    Bool(Option<bool>),
}

pub struct ConfigValue {
    pub value: ConfigValueType,
    pub is_default: bool,
}

#[derive(Parser, Debug, Clone, Serialize, Deserialize)]
pub struct CliConfig {
    #[serde(skip_serializing_if = "::std::option::Option::is_none")]
    pub input: Option<PathBuf>,
    #[serde(skip_serializing_if = "::std::option::Option::is_none")]
    pub output: Option<PathBuf>,
    #[serde(skip_serializing_if = "::std::option::Option::is_none")]
    pub assets: Option<PathBuf>,
    #[serde(skip_serializing_if = "::std::option::Option::is_none")]
    pub wasm: Option<PathBuf>,
    #[serde(skip_serializing_if = "::std::option::Option::is_none")]
    pub database: Option<PathBuf>,
    #[serde(skip_serializing_if = "::std::option::Option::is_none")]
    pub brand: Option<String>,
    #[serde(skip_serializing_if = "::std::option::Option::is_none")]
    pub port: Option<u16>,
    #[serde(skip_serializing_if = "::std::option::Option::is_none")]
    pub static_build_only: Option<bool>,
    #[serde(skip_serializing_if = "::std::option::Option::is_none")]
    pub flat: Option<bool>,
}

#[derive(Parser, Debug, Clone, Serialize, Deserialize)]
pub struct Config {
    pub input: PathBuf,
    pub output: PathBuf,
    pub assets: PathBuf,
    pub wasm: PathBuf,
    pub database: PathBuf,
    pub brand: String,
    pub port: u16,
    pub static_build_only: bool,
    pub flat: bool,
}

enum config_creation_mode {
    only_default_values,
    only_set_values,
}

fn create_config(
    config_values: &HashMap<String, ConfigValue>,
    creation_mode: config_creation_mode,
) -> CliConfig {
    CliConfig {
        input: match config_values.get("input") {
            Some(cv) => match &cv.value {
                ConfigValueType::Path(p) => match creation_mode {
                    config_creation_mode::only_default_values => {
                        if cv.is_default {
                            p.clone()
                        } else {
                            None
                        }
                    }
                    config_creation_mode::only_set_values => {
                        if cv.is_default {
                            None
                        } else {
                            p.clone()
                        }
                    }
                },
                _ => None,
            },
            None => None,
        },
        output: match config_values.get("output") {
            Some(cv) => match &cv.value {
                ConfigValueType::Path(p) => match creation_mode {
                    config_creation_mode::only_default_values => {
                        if cv.is_default {
                            p.clone()
                        } else {
                            None
                        }
                    }
                    config_creation_mode::only_set_values => {
                        if cv.is_default {
                            None
                        } else {
                            p.clone()
                        }
                    }
                },
                _ => None,
            },
            None => None,
        },
        assets: match config_values.get("assets") {
            Some(cv) => match &cv.value {
                ConfigValueType::Path(p) => match creation_mode {
                    config_creation_mode::only_default_values => {
                        if cv.is_default {
                            p.clone()
                        } else {
                            None
                        }
                    }
                    config_creation_mode::only_set_values => {
                        if cv.is_default {
                            None
                        } else {
                            p.clone()
                        }
                    }
                },
                _ => None,
            },
            None => None,
        },
        wasm: match config_values.get("wasm") {
            Some(cv) => match &cv.value {
                ConfigValueType::Path(p) => match creation_mode {
                    config_creation_mode::only_default_values => {
                        if cv.is_default {
                            p.clone()
                        } else {
                            None
                        }
                    }
                    config_creation_mode::only_set_values => {
                        if cv.is_default {
                            None
                        } else {
                            p.clone()
                        }
                    }
                },
                _ => None,
            },
            None => None,
        },
        database: match config_values.get("database") {
            Some(cv) => match &cv.value {
                ConfigValueType::Path(p) => match creation_mode {
                    config_creation_mode::only_default_values => {
                        if cv.is_default {
                            p.clone()
                        } else {
                            None
                        }
                    }
                    config_creation_mode::only_set_values => {
                        if cv.is_default {
                            None
                        } else {
                            p.clone()
                        }
                    }
                },
                _ => None,
            },
            None => None,
        },
        brand: match config_values.get("brand") {
            Some(cv) => match &cv.value {
                ConfigValueType::String(p) => match creation_mode {
                    config_creation_mode::only_default_values => {
                        if cv.is_default {
                            p.clone()
                        } else {
                            None
                        }
                    }
                    config_creation_mode::only_set_values => {
                        if cv.is_default {
                            None
                        } else {
                            p.clone()
                        }
                    }
                },
                _ => None,
            },
            None => None,
        },
        port: match config_values.get("port") {
            Some(cv) => match &cv.value {
                ConfigValueType::Number(p) => match creation_mode {
                    config_creation_mode::only_default_values => {
                        if cv.is_default {
                            p.clone()
                        } else {
                            None
                        }
                    }
                    config_creation_mode::only_set_values => {
                        if cv.is_default {
                            None
                        } else {
                            p.clone()
                        }
                    }
                },
                _ => None,
            },
            None => None,
        },
        static_build_only: match config_values.get("static_build_only") {
            Some(cv) => match &cv.value {
                ConfigValueType::Bool(p) => match creation_mode {
                    config_creation_mode::only_default_values => {
                        if cv.is_default {
                            p.clone()
                        } else {
                            None
                        }
                    }
                    config_creation_mode::only_set_values => {
                        if cv.is_default {
                            None
                        } else {
                            p.clone()
                        }
                    }
                },
                _ => None,
            },
            None => None,
        },
        flat: match config_values.get("flat") {
            Some(cv) => match &cv.value {
                ConfigValueType::Bool(p) => match creation_mode {
                    config_creation_mode::only_default_values => {
                        if cv.is_default {
                            p.clone()
                        } else {
                            None
                        }
                    }
                    config_creation_mode::only_set_values => {
                        if cv.is_default {
                            None
                        } else {
                            p.clone()
                        }
                    }
                },
                _ => None,
            },
            None => None,
        },
    }
}

impl Config {
    pub fn new(config_values: HashMap<String, ConfigValue>) -> Self {
        let cli_only_default_values_config =
            create_config(&config_values, config_creation_mode::only_default_values);
        let cli_only_set_values_config =
            create_config(&config_values, config_creation_mode::only_set_values);

        let figment_config: Config = Figment::new()
            .merge(Serialized::defaults(cli_only_default_values_config))
            .merge(Toml::file("pankat.toml"))
            .merge(Env::prefixed("PANKAT_"))
            .merge(Serialized::defaults(cli_only_set_values_config))
            .extract()
            .unwrap();
        figment_config
    }

    pub fn get() -> &'static Arc<Config> {
        SINGLETON.get().expect("Config not initialized")
    }

    pub fn initialize(config: Config) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        #[cfg(all(not(test)))]
        {
            ensure_paths_exist(&config.input)?;
            ensure_paths_exist(&config.output)?;
            ensure_paths_exist(&config.assets)?;
            ensure_paths_exist(&config.wasm)?;
            ensure_paths_exist(&config.database)?;
        }

        let _ = SINGLETON
            .set(Arc::new(config))
            .map_err(|_| "Config can only be initialized once");
        Ok(())
    }
}

static SINGLETON: OnceLock<Arc<Config>> = OnceLock::new();

#[cfg(all(not(test)))]
fn ensure_paths_exist(path: &PathBuf) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    if !path.exists() {
        return Err(Box::<dyn std::error::Error + Send + Sync>::from(format!(
            "Path does not exist: '{}'. Create it manually!",
            path.display()
        )));
    }
    Ok(())
}

# tests.rs

#[cfg(test)]
mod tests {
    use crate::articles::ArticleWithTags;
    use crate::config;
    use crate::renderer::html::create_html_from_standalone_template_by_article;
    use crate::Config;
    use crate::ConfigValue;
    use std::collections::HashMap;
    use std::path::PathBuf;

    fn create_hacky_config() -> Config {
        let mut config_values: HashMap<String, ConfigValue> = HashMap::new();

        config_values.insert(
            "input".to_string(),
            ConfigValue {
                value: config::ConfigValueType::Path(Some("documents/blog.lastlog.de".into())),
                is_default: false,
            },
        );

        config_values.insert(
            "output".to_string(),
            ConfigValue {
                value: config::ConfigValueType::Path(Some("documents/output".into())),
                is_default: false,
            },
        );

        config_values.insert(
            "assets".to_string(),
            ConfigValue {
                value: config::ConfigValueType::Path(Some("documents/assets".into())),
                is_default: false,
            },
        );

        config_values.insert(
            "wasm".to_string(),
            ConfigValue {
                value: config::ConfigValueType::Path(Some("documents/wasm".into())),
                is_default: false,
            },
        );

        config_values.insert(
            "database".to_string(),
            ConfigValue {
                value: config::ConfigValueType::Path(Some("documents".into())),
                is_default: false,
            },
        );

        config_values.insert(
            "brand".to_string(),
            ConfigValue {
                value: config::ConfigValueType::String(Some("".to_string())),
                is_default: false,
            },
        );

        config_values.insert(
            "port".to_string(),
            ConfigValue {
                value: config::ConfigValueType::Number(Some(5000)),
                is_default: false,
            },
        );

        config_values.insert(
            "static".to_string(),
            ConfigValue {
                value: config::ConfigValueType::Bool(Some(false)),
                is_default: false,
            },
        );

        config_values.insert(
            "flat".to_string(),
            ConfigValue {
                value: config::ConfigValueType::Bool(Some(false)),
                is_default: false,
            },
        );

        let config = config::Config::new(config_values);

        config
    }

    #[test]
    fn test_create_html_from_standalone_template() {
        let config = create_hacky_config();
        config::Config::initialize(config).expect("Failed to initialize config");

        let article = ArticleWithTags {
            id: None,
            src_file_name: "documents/blog.lastlog.de/posts/test_src.md".to_string(),
            dst_file_name: "test_dst.html".to_string(),
            title: Some("Test NewArticle".to_string()),
            modification_date: None,
            summary: None,
            series: None,
            special_page: Some(true),
            draft: None,
            anchorjs: None,
            tocify: None,
            live_updates: None,
            tags: None,
        };
        let html_content = "<p>This is a test body.</p>".to_string();

        let result = create_html_from_standalone_template_by_article(article, html_content.clone());

        assert!(result.is_ok());
        let rendered_html = result.unwrap();
        println!("{}", rendered_html);
        assert!(rendered_html.contains(&html_content));
        assert!(rendered_html.contains("Test NewArticle"));
    }

    #[test]
    fn test_date_and_time() {
        use crate::renderer::utils::date_and_time;
        use chrono::NaiveDateTime;
        let date_time_str = "2024-04-12 20:53:00";
        let date_time = NaiveDateTime::parse_from_str(date_time_str, "%Y-%m-%d %H:%M:%S")
            .expect("Failed to parse date time");
        let formatted_date = date_and_time(&Some(date_time));
        assert_eq!(formatted_date, "12 apr 2024");

        let formatted_date_none = date_and_time(&None);
        assert_eq!(formatted_date_none, "");
    }
}