Convert a tox.ini to tox.toml or pyproject.toml file.

tox_ini_to_toml.py

#!/usr/bin/env python3

import argparse
import configparser
import os
import re
import shlex
import sys

from typing import Any

try:
    import tomlkit
    from tomlkit.items import Table
except ImportError:
    print(
        'This script requires the tomlkit package. Please install it via "pip install tomlkit".'
    )
    sys.exit(1)

# Mapping of INI option names to TOML option names
option_name_map: dict[str, str] = {
    "basepython": "base_python",
    "ignore_basepython_conflict": "ignore_base_python_conflict",
    "usedevelop": "use_develop",
    "skip_installation": "skip_install",
    "skipsdist": "no_package",
    "commands_pre": "commands_pre",
    "commands_post": "commands_post",
    "whitelist_externals": "allowlist_externals",
    "toxworkdir": "work_dir",
    "toxproject": "tox_root",
    "changedir": "change_dir",
    "passenv": "pass_env",
    "setenv": "set_env",
    "envdir": "env_dir",
    "envpython": "env_python",
    "envbindir": "env_bin_dir",
    "envlogdir": "env_log_dir",
    "envtmpdir": "env_tmp_dir",
    "deps": "deps",
    # Add more mappings as needed
}


def ini_to_toml(
    ini_file: str, output_file: None | str = None, use_pyproject: bool = False
) -> None:
    """
    Convert a tox.ini or setup.cfg INI file to a TOML format.

    Args:
        ini_file (str): Path to the INI file to convert.
        output_file (str | None): Path to the output TOML file. If None, output to stdout.
        use_pyproject (bool): If True, output TOML suitable for inclusion in pyproject.toml.

    Raises:
        FileNotFoundError: If the ini_file does not exist.
    """
    # Ensure the ini_file exists
    if not os.path.exists(ini_file):
        raise FileNotFoundError(f"The file {ini_file} does not exist.")

    # Load the config parser
    config = configparser.ConfigParser()
    config.optionxform = str  # preserve case sensitivity  # pyright: ignore[reportAttributeAccessIssue]

    try:
        with open(ini_file, "r", encoding="utf-8") as f:
            config.read_file(f)
    except FileNotFoundError:
        print(f"Error: INI file not found: {ini_file}")
        sys.exit(1)
    except configparser.Error as e:
        print(f"Error parsing INI file {ini_file}: {e}")
        sys.exit(1)
    except Exception as e:
        print(f"An unexpected error occurred while reading {ini_file}: {e}")
        sys.exit(1)

    # Build the TOML document
    try:
        if use_pyproject:
            # Start building [tool.tox] table
            toml_doc = tomlkit.document()
            tool = tomlkit.table()
            tox_table = tomlkit.table()
            toml_doc.add("tool", tool)
            tool.add("tox", tox_table)
        else:
            toml_doc = tomlkit.document()
            tox_table = toml_doc

        # Handle the [tox] section
        tox_options, per_env_settings = handle_tox_section(config)
        tox_table.update(tox_options)

        # Initialize per-environment configurations
        envs: dict[str, Table] = {}
        env_run_base = None
        env_pkg_base = None

        # Handle testenv sections
        env_run_base, env_pkg_base, envs = handle_testenv_sections(config, envs)

        # Add env_run_base and env_pkg_base to toml_doc
        if env_run_base:
            if use_pyproject:
                tox_table["env_run_base"] = env_run_base
            else:
                toml_doc["env_run_base"] = env_run_base
        if env_pkg_base:
            if use_pyproject:
                tox_table["env_pkg_base"] = env_pkg_base
            else:
                toml_doc["env_pkg_base"] = env_pkg_base

        # Add individual environments
        if envs:
            env_table = tomlkit.table()
            for env_name, env_options in envs.items():
                env_table.add(env_name, env_options)
            if use_pyproject:
                tox_table["env"] = env_table
            else:
                toml_doc["env"] = env_table

        # Write the TOML data to the output file or stdout using tomlkit
        toml_output = tomlkit.dumps(toml_doc)
        if output_file:
            try:
                with open(output_file, "w", encoding="utf-8") as f:
                    f.write(toml_output)
            except IOError as e:
                print(f"Error writing to output file {output_file}: {e}")
                sys.exit(1)
        else:
            print(toml_output)
    except Exception as e:
        print(f"An error occurred during conversion: {e}")
        sys.exit(1)


def handle_tox_section(
    config: configparser.ConfigParser,
) -> tuple[Table, dict[str, Table]]:
    """
    Handle the [tox] section of the ini file and return tox options and per-environment settings.

    Args:
        config (configparser.ConfigParser): The configuration parser object containing the parsed ini file.

    Returns:
        tuple[tomlkit.items.Table, dict[str, tomlkit.items.Table]]: tox options and per-environment settings.
    """
    tox_options = tomlkit.table()
    per_env_settings: dict[str, Table] = {}
    if "tox" in config.sections() or "tox:tox" in config.sections():
        tox_section = config["tox"] if "tox" in config.sections() else config["tox:tox"]
        tox_options_parsed, per_env_settings = parse_tox_options(
            tox_section, option_name_map
        )
        tox_options.update(tox_options_parsed)
    else:
        # No [tox] section found
        pass
    return tox_options, per_env_settings


def handle_testenv_sections(
    config: configparser.ConfigParser, envs: dict[str, Table]
) -> tuple[Table | None, Table | None, dict[str, Table]]:
    """
    Handle the [testenv] and [testenv:name] sections.

    Args:
        config (configparser.ConfigParser): The configuration parser object containing the parsed ini file.
        envs (dict[str, tomlkit.items.Table]): The existing environment configurations.

    Returns:
        tuple[tomlkit.items.Table | None, tomlkit.items.Table | None, dict[str, tomlkit.items.Table]]:
            env_run_base, env_pkg_base, and updated envs dictionary.
    """
    env_run_base: Table | None = None
    env_pkg_base: Table | None = None

    for section in config.sections():
        if section == "testenv" or section == "testenv:":
            # Base test environment configurations
            options, per_env_settings = parse_testenv_options(
                dict(config.items(section)), option_name_map
            )
            env_run_base = options
            for env_name, settings in per_env_settings.items():
                if env_name not in envs:
                    envs[env_name] = tomlkit.table()
                envs[env_name].update(settings)
        elif section.startswith("testenv:"):
            # Specific test environment
            env_name = section.split(":", 1)[1].strip()
            options, per_env_settings = parse_testenv_options(
                dict(config.items(section)), option_name_map
            )
            if env_name not in envs:
                envs[env_name] = tomlkit.table()
            envs[env_name].update(options)
            # Also update with any conditional settings
            for conditional_env_name, settings in per_env_settings.items():
                if conditional_env_name == env_name:
                    envs[env_name].update(settings)
        elif section == "pkgenv":
            # Base package environment configurations
            options, per_env_settings = parse_testenv_options(
                dict(config.items(section)), option_name_map
            )
            env_pkg_base = options
            for env_name, settings in per_env_settings.items():
                if env_name not in envs:
                    envs[env_name] = tomlkit.table()
                envs[env_name].update(settings)
        else:
            # Other sections are ignored or could be handled as needed
            pass

    return env_run_base, env_pkg_base, envs


def parse_tox_options(
    options: configparser.SectionProxy, option_name_map: dict[str, str]
) -> tuple[Table, dict[str, Table]]:
    """
    Parse options from the [tox] section.

    Args:
        options (configparser.SectionProxy): The [tox] section from configparser.
        option_name_map (dict[str, str]): Mapping of INI option names to TOML option names.

    Returns:
        tuple[tomlkit.items.Table, dict[str, tomlkit.items.Table]]: A tuple containing the tox options table and per-environment settings.
    """
    tox_options = tomlkit.table()
    per_env_settings: dict[str, Table] = {}
    for key, value in options.items():
        key_mapped = option_name_map.get(key, key)
        if key in ("env_list", "envlist"):
            # Split the environments and store as a list
            env_list: list[str] = []
            for line in value.strip().splitlines():
                line = line.strip()
                if not line or is_comment_line(line):
                    continue
                if is_conditional_assignment(line):
                    # Skip conditional assignments in env_list
                    continue
                envs_in_line = line.strip().split()
                env_list.extend(env.strip() for env in envs_in_line if env.strip())
            tox_options["env_list"] = env_list
        elif key == "requires":
            requires_list: list[str] = []
            for line in value.strip().splitlines():
                line = line.strip()
                if line and not is_comment_line(line):
                    requires_list.append(line)
            tox_options["requires"] = requires_list
        elif key in ("min_version", "minversion"):
            tox_options["min_version"] = value.strip()
        else:
            # Handle conditional assignments
            general_value, conditional_values = parse_conditional_settings(value)
            if general_value is not None:
                tox_options[key_mapped] = convert_value(general_value, key_mapped)
            for envs_list, cond_value in conditional_values.items():
                for env_name in envs_list:
                    if env_name not in per_env_settings:
                        per_env_settings[env_name] = tomlkit.table()
                    per_env_settings[env_name][key_mapped] = convert_value(
                        cond_value, key_mapped
                    )
    return tox_options, per_env_settings


def parse_testenv_options(
    options: dict[str, str], option_name_map: dict[str, str]
) -> tuple[Table, dict[str, Table]]:
    """
    Parse options from the [testenv] or [testenv:name] section.

    Args:
        options (dict[str, str]): Options from the section.
        option_name_map (dict[str, str]): Mapping of INI option names to TOML option names.

    Returns:
        tuple[tomlkit.items.Table, dict[str, tomlkit.items.Table]]: A tuple containing the testenv options table and per-environment settings.
    """
    env_options = tomlkit.table()
    per_env_settings: dict[str, Table] = {}
    for key, value in options.items():
        key_mapped = option_name_map.get(key, key)
        # Handle conditional assignments
        general_value, conditional_values = parse_conditional_settings(value)
        if key_mapped == "deps":
            if general_value is not None:
                deps_list = parse_deps(general_value)
                env_options["deps"] = deps_list
            for envs_list, cond_value in conditional_values.items():
                deps_list = parse_deps(cond_value)
                for env_name in envs_list:
                    if env_name not in per_env_settings:
                        per_env_settings[env_name] = tomlkit.table()
                    per_env_settings[env_name].setdefault("deps", []).extend(deps_list)
        elif key_mapped == "commands":
            if general_value is not None:
                commands_list = parse_commands(general_value)
                env_options["commands"] = commands_list
            for envs_list, cond_value in conditional_values.items():
                commands_list = parse_commands(cond_value)
                for env_name in envs_list:
                    if env_name not in per_env_settings:
                        per_env_settings[env_name] = tomlkit.table()
                    per_env_settings[env_name]["commands"] = commands_list
        elif key_mapped == "allowlist_externals":
            if general_value is not None:
                allowlist = parse_list_option(general_value)
                env_options["allowlist_externals"] = allowlist
            for envs_list, cond_value in conditional_values.items():
                allowlist = parse_list_option(cond_value)
                for env_name in envs_list:
                    if env_name not in per_env_settings:
                        per_env_settings[env_name] = tomlkit.table()
                    per_env_settings[env_name]["allowlist_externals"] = allowlist
        elif key_mapped == "set_env":
            if general_value is not None:
                setenv_dict = parse_setenv(general_value)
                env_options["set_env"] = setenv_dict
            for envs_list, cond_value in conditional_values.items():
                setenv_dict = parse_setenv(cond_value)
                for env_name in envs_list:
                    if env_name not in per_env_settings:
                        per_env_settings[env_name] = tomlkit.table()
                    per_env_settings[env_name]["set_env"] = setenv_dict
        elif key_mapped in (
            "skip_install",
            "use_develop",
            "recreate",
            "ignore_errors",
            "ignore_outcome",
            "no_package",
            "system_site_packages",
            "always_copy",
            "download",
            "parallel_show_output",
        ):
            # Boolean keys
            if general_value is not None:
                env_options[key_mapped] = str_to_bool(general_value)
            for envs_list, cond_value in conditional_values.items():
                bool_value = str_to_bool(cond_value)
                for env_name in envs_list:
                    if env_name not in per_env_settings:
                        per_env_settings[env_name] = tomlkit.table()
                    per_env_settings[env_name][key_mapped] = bool_value
        else:
            if general_value is not None:
                env_options[key_mapped] = convert_value(general_value, key_mapped)
            for envs_list, cond_value in conditional_values.items():
                for env_name in envs_list:
                    if env_name not in per_env_settings:
                        per_env_settings[env_name] = tomlkit.table()
                    per_env_settings[env_name][key_mapped] = convert_value(
                        cond_value, key_mapped
                    )
    return env_options, per_env_settings


def convert_value(value: str, key: str) -> Any:
    """
    Convert the value to the appropriate type based on the key.

    Args:
        value (str): The value to convert.
        key (str): The option key.

    Returns:
        Any: The converted value.
    """
    if key in (
        "skip_install",
        "use_develop",
        "recreate",
        "ignore_errors",
        "ignore_outcome",
        "no_package",
        "system_site_packages",
        "always_copy",
        "download",
        "parallel_show_output",
        "skip_missing_interpreters",
        "ignore_base_python_conflict",
    ):
        try:
            return str_to_bool(value)
        except ValueError as e:
            print(f"Invalid boolean value for '{key}': {e}")
            return value.strip()  # Return the original value if conversion fails
    elif key == "pass_env":
        return parse_list_option(value)
    else:
        return value.strip()


def parse_conditional_settings(
    value: str,
) -> tuple[str | None, dict[tuple[str, ...], str]]:
    """
    Parses a value that may contain conditional assignments.

    Args:
        value (str): The value to parse.

    Returns:
        tuple[str | None, dict[tuple[str, ...], str]]: A tuple of the general value and conditional values.
    """
    lines = value.strip().splitlines()
    general_lines: list[str] = []
    conditional_values: dict[tuple[str, ...], str] = {}

    for line in lines:
        line = line.strip()
        if not line or is_comment_line(line):
            continue
        if is_conditional_assignment(line):
            env_part, val_part = line.split(":", 1)
            env_names = [e.strip() for e in env_part.split(",")]
            val_part = val_part.strip()
            conditional_values[tuple(env_names)] = val_part
        else:
            general_lines.append(line)
    general_value = "\n".join(general_lines) if general_lines else None
    return general_value, conditional_values


def is_conditional_assignment(line: str) -> bool:
    """
    Check if a line is a conditional assignment.

    Args:
        line (str): The line to check.

    Returns:
        bool: True if the line is a conditional assignment, False otherwise.
    """
    return bool(
        ":" in line and not line.startswith("#") and re.match(r"^[\w\d_\-, ]+:", line)
    )


def is_comment_line(line: str) -> bool:
    """
    Check if a line is a comment line.

    Args:
        line (str): The line to check.

    Returns:
        bool: True if the line is a comment line, False otherwise.
    """
    return line.startswith("#")


def parse_deps(value: str) -> list[str]:
    """
    Parse the 'deps' option into a list.

    Args:
        value (str): The deps value to parse.

    Returns:
        list[str]: A list of dependencies.
    """
    deps_list: list[str] = []
    current_line = ""
    lines = value.strip().splitlines()
    for line in lines:
        line = line.strip()
        if not line or is_comment_line(line):
            continue
        if line.endswith("\\"):
            current_line += line.rstrip("\\").rstrip() + " "
        else:
            current_line += line
            if current_line:
                deps_list.append(current_line.strip())
            current_line = ""
    if current_line:
        deps_list.append(current_line.strip())
    return deps_list


def parse_commands(value: str) -> list[list[str]]:
    """
    Parse the 'commands' option into a list of commands.

    Args:
        value (str): The commands value to parse.

    Returns:
        list[list[str]]: A list of command arguments.
    """
    commands_list: list[list[str]] = []
    current_command = ""
    lines = value.strip().splitlines()
    for line in lines:
        line = line.rstrip()
        if not line or is_comment_line(line):
            continue
        if line.endswith("\\"):
            current_command += line.rstrip("\\").rstrip() + " "
        else:
            current_command += line
            if current_command:
                # Split the command into a list for TOML
                command_args = split_args(current_command)
                commands_list.append(command_args)
            current_command = ""
    if current_command:
        # Leftover command after loop
        command_args = split_args(current_command)
        commands_list.append(command_args)
    return commands_list


def parse_list_option(value: str) -> list[str]:
    """
    Parse a multi-line list option into a list of strings.

    Args:
        value (str): The value to parse.

    Returns:
        list[str]: A list of strings.
    """
    items: list[str] = []
    for line in value.strip().splitlines():
        line = line.strip()
        if not line or is_comment_line(line):
            continue
        items.append(line)
    return items


def parse_setenv(value: str) -> dict[str, str]:
    """
    Parse the 'setenv' option into a dictionary.

    Args:
        value (str): The setenv value to parse.

    Returns:
        dict[str, str]: A dictionary of environment variable assignments.
    """
    setenv_dict: dict[str, str] = {}
    for line in value.strip().splitlines():
        line = line.strip()
        if not line or is_comment_line(line):
            continue
        if "=" in line:
            k, v = line.split("=", 1)
            setenv_dict[k.strip()] = v.strip()
    return setenv_dict


def str_to_bool(value: str) -> bool:
    """
    Convert a string to a boolean value.

    Args:
        value (str): The string to convert.

    Returns:
        bool: The boolean value.

    Raises:
        ValueError: If the string cannot be converted to a boolean.
    """
    value_lower = value.strip().lower()
    if value_lower in ("true", "yes", "1", "on"):
        return True
    elif value_lower in ("false", "no", "0", "off"):
        return False
    else:
        raise ValueError(f"Invalid boolean value: '{value}'")


def split_args(command_str: str) -> list[str]:
    """
    Split a command string into arguments.

    Args:
        command_str (str): The command string to split.

    Returns:
        list[str]: A list of command arguments.
    """
    try:
        return shlex.split(command_str, posix=True)
    except ValueError:
        # For commands that may not parse correctly with shlex (e.g., Windows)
        return command_str.strip().split()


def main() -> None:
    """
    Main function to parse arguments and perform the conversion.
    """
    parser = argparse.ArgumentParser(description="Convert tox.ini to TOML format.")
    parser.add_argument("ini_file", help="Path to the tox.ini or setup.cfg file.")
    parser.add_argument(
        "-o",
        "--output",
        help="Output file to write the TOML data. Defaults to stdout.",
    )
    parser.add_argument(
        "--pyproject",
        action="store_true",
        help=(
            "Output TOML suitable for pyproject.toml (use [tool.tox] sections). "
            "Default outputs to tox.toml format."
        ),
    )
    args = parser.parse_args()

    if not os.path.exists(args.ini_file):
        print(f"Error: File not found: {args.ini_file}")
        sys.exit(1)

    try:
        ini_to_toml(args.ini_file, args.output, args.pyproject)
    except Exception as e:
        print(f"An error occurred: {e}")
        sys.exit(1)


if __name__ == "__main__":
    main()