Create Prompt Markdown Instructions (.md Files)

Prompt Markdown Instructions (.md Files)

#!/usr/bin/env python3
"""
html_to_ai_markdown.py

Convert every index.html/index.htm in a static HTML website to index.md.

Workflow:
1. Performs a complete dry-run test across all target files.
2. If dry-run passes, generates and applies index.md files.
3. Validates generated output.
4. Removes all `.bak` files upon success.
5. Restarts the Apache web server (systemctl restart apache2 / httpd).
"""

from __future__ import annotations

import argparse
import html
import re
import shutil
import subprocess
import sys
import tempfile
from dataclasses import dataclass, field
from html.parser import HTMLParser
from pathlib import Path


# ============================================================================
# CONFIGURATION
# ============================================================================

AUTHOR = "Karim (Masum)"
DATE = "2026-08-31"

HTML_NAMES = {"index.html", "index.htm"}

# HTML elements that should never appear in the generated AI Markdown.
REMOVE_TAGS = {
    "script",
    "style",
    "noscript",
    "template",
    "svg",
    "canvas",
    "nav",
    "footer",
    "header",
    "form",
    "iframe",
    "video",
    "audio",
    "source",
    "object",
    "embed",
}

# UI/container classes that normally represent website chrome.
REMOVE_CLASS_PATTERNS = (
    "footer",
    "navbar",
    "navigation",
    "breadcrumb",
    "cookie",
    "modal",
    "popup",
    "sidebar",
    "menu",
)


# ============================================================================
# HTML TREE
# ============================================================================

@dataclass
class Node:
    tag: str | None = None
    attrs: dict[str, str] = field(default_factory=dict)
    children: list["Node"] = field(default_factory=list)
    text: str | None = None


class SiteParser(HTMLParser):
    VOID = {
        "area",
        "base",
        "br",
        "col",
        "embed",
        "hr",
        "img",
        "input",
        "link",
        "meta",
        "param",
        "source",
        "track",
        "wbr",
    }

    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.root = Node("root")
        self.stack = [self.root]

    @property
    def current(self) -> Node:
        return self.stack[-1]

    def handle_starttag(self, tag: str, attrs) -> None:
        node = Node(
            tag=tag.lower(),
            attrs={
                str(key).lower(): str(value or "")
                for key, value in attrs
            },
        )
        self.current.children.append(node)
        if node.tag not in self.VOID:
            self.stack.append(node)

    def handle_startendtag(self, tag: str, attrs) -> None:
        node = Node(
            tag=tag.lower(),
            attrs={
                str(key).lower(): str(value or "")
                for key, value in attrs
            },
        )
        self.current.children.append(node)

    def handle_endtag(self, tag: str) -> None:
        tag = tag.lower()
        for index in range(len(self.stack) - 1, 0, -1):
            if self.stack[index].tag == tag:
                del self.stack[index:]
                return

    def handle_data(self, data: str) -> None:
        if data:
            self.current.children.append(Node(text=data))


def parse_html(source: str) -> Node:
    parser = SiteParser()
    parser.feed(source)
    parser.close()
    return parser.root


# ============================================================================
# HTML HELPERS
# ============================================================================

def clean_text(value: str) -> str:
    value = html.unescape(value or "")
    value = value.replace("\xa0", " ")
    return re.sub(r"[ \t\r\f\v]+", " ", value).strip()


def attr(node: Node, name: str) -> str:
    return node.attrs.get(name.lower(), "")


def classes(node: Node) -> set[str]:
    return set(attr(node, "class").lower().split())


def node_text(node: Node | None) -> str:
    if node is None:
        return ""
    if node.text is not None:
        return clean_text(node.text)
    return clean_text(
        " ".join(node_text(child) for child in node.children)
    )


def find_first(node: Node, tag: str) -> Node | None:
    for child in node.children:
        if child.text is not None:
            continue
        if child.tag == tag:
            return child
        found = find_first(child, tag)
        if found:
            return found
    return None


def find_all(node: Node, tag: str) -> list[Node]:
    result = []
    for child in node.children:
        if child.text is not None:
            continue
        if child.tag == tag:
            result.append(child)
        result.extend(find_all(child, tag))
    return result


def is_removed(node: Node) -> bool:
    if node.tag in REMOVE_TAGS:
        return True
    class_text = " ".join(classes(node))
    if any(pattern in class_text for pattern in REMOVE_CLASS_PATTERNS):
        return True
    role = attr(node, "role").lower()
    if role in {"navigation", "banner", "contentinfo"}:
        return True
    return False


def clone_content(node: Node) -> Node:
    copied = Node(
        tag=node.tag,
        attrs=dict(node.attrs),
        text=node.text,
    )
    if node.text is not None:
        return copied
    for child in node.children:
        if is_removed(child):
            continue
        copied.children.append(clone_content(child))
    return copied


# ============================================================================
# PAGE METADATA & CONVERSION
# ============================================================================

def get_content_root(root: Node) -> Node:
    return (
        find_first(root, "main")
        or find_first(root, "article")
        or find_first(root, "body")
        or root
    )


def get_title(root: Node) -> str:
    title = node_text(find_first(root, "title"))
    if title:
        return title
    h1 = node_text(find_first(root, "h1"))
    if h1:
        return h1
    return "Untitled Page"


def get_description(root: Node, content: Node) -> str:
    for meta in find_all(root, "meta"):
        name = attr(meta, "name").lower()
        prop = attr(meta, "property").lower()
        if name == "description" or prop == "og:description":
            value = clean_text(attr(meta, "content"))
            if value:
                return value
    for paragraph in find_all(content, "p"):
        value = node_text(paragraph)
        if len(value) >= 30:
            return value[:240].rstrip()
    return "Content and information provided on this page."


def yaml_quote(value: str) -> str:
    value = value.replace("\\", "\\\\").replace('"', '\\"')
    return f'"{value}"'


def front_matter(title: str, description: str) -> str:
    return (
        "---\n"
        f"title: {yaml_quote(title)}\n"
        f"author: {yaml_quote(AUTHOR)}\n"
        f"date: {DATE}\n"
        f"description: {yaml_quote(description)}\n"
        "---\n"
    )


def inline(node: Node) -> str:
    if node.text is not None:
        return clean_text(node.text)
    tag = node.tag or ""
    if tag == "br":
        return "\n"
    if tag == "a":
        text = inline_children(node)
        href = attr(node, "href").strip()
        return f"[{text}]({href})" if text and href else text
    if tag == "img":
        src = attr(node, "src").strip()
        alt = clean_text(attr(node, "alt"))
        return f"![{alt}]({src})" if src else ""
    if tag in {"strong", "b"}:
        text = inline_children(node)
        return f"**{text}**" if text else ""
    if tag in {"em", "i"}:
        node_classes = classes(node)
        if (
            tag == "i"
            and (
                attr(node, "aria-hidden").lower() == "true"
                or "fa" in node_classes
                or any(item.startswith("fa-") for item in node_classes)
            )
        ):
            return ""
        text = inline_children(node)
        return f"*{text}*" if text else ""
    if tag == "code":
        text = inline_children(node)
        return f"`{text.replace('`', r'\`')}`" if text else ""
    if tag in {"del", "s"}:
        text = inline_children(node)
        return f"~~{text}~~" if text else ""
    return inline_children(node)


def inline_children(node: Node) -> str:
    parts = [inline(child) for child in node.children]
    value = " ".join(part for part in parts if part)
    value = re.sub(r"[ \t]+", " ", value)
    value = re.sub(r" *\n *", "\n", value)
    return value.strip()


class Renderer:
    def __init__(self) -> None:
        self.lines: list[str] = []

    def blank(self) -> None:
        while self.lines and self.lines[-1] == "":
            self.lines.pop()
        self.lines.append("")

    def add(self, text: str = "") -> None:
        self.lines.append(text.rstrip())

    def render(self, node: Node) -> str:
        for child in node.children:
            self.render_node(child)
        while self.lines and not self.lines[-1].strip():
            self.lines.pop()
        return "\n".join(self.lines)

    def render_node(self, node: Node) -> None:
        if node.text is not None:
            return
        tag = node.tag or ""
        if re.fullmatch(r"h[1-6]", tag):
            text = inline_children(node)
            if text:
                self.blank()
                self.add("#" * int(tag[1]) + " " + text)
                self.blank()
            return
        if tag == "p":
            text = inline_children(node)
            if text:
                self.blank()
                self.add(text)
                self.blank()
            return
        if tag == "pre":
            code = find_first(node, "code")
            source = node_text(code or node).rstrip()
            language = ""
            if code:
                match = re.search(
                    r"(?:language|lang)-([\w+-]+)", attr(code, "class")
                )
                if match:
                    language = match.group(1)
            if source:
                self.blank()
                self.add("```" + language)
                self.lines.extend(source.splitlines())
                self.add("```")
                self.blank()
            return
        if tag == "blockquote":
            text = inline_children(node)
            if text:
                self.blank()
                for line in text.splitlines():
                    self.add("> " + line)
                self.blank()
            return
        if tag in {"ul", "ol"}:
            self.render_list(node, tag == "ol")
            return
        if tag == "table":
            self.render_table(node)
            return
        if tag == "hr":
            self.blank()
            self.add("---")
            self.blank()
            return
        if tag == "img":
            text = inline(node)
            if text:
                self.blank()
                self.add(text)
                self.blank()
            return
        for child in node.children:
            self.render_node(child)

    def render_list(self, node: Node, ordered: bool) -> None:
        items = [c for c in node.children if c.text is None and c.tag == "li"]
        if not items:
            return
        self.blank()
        number = 1
        for item in items:
            parts, nested = [], []
            for child in item.children:
                if child.text is not None:
                    val = clean_text(child.text)
                    if val:
                        parts.append(val)
                elif child.tag in {"ul", "ol"}:
                    nested.append(child)
                else:
                    val = inline(child)
                    if val:
                        parts.append(val)
            text = clean_text(" ".join(parts))
            if not text:
                continue
            prefix = f"{number}. " if ordered else "- "
            self.add(prefix + text)
            for sublist in nested:
                nested_renderer = Renderer()
                nested_renderer.render_list(sublist, sublist.tag == "ol")
                for line in nested_renderer.lines:
                    if line.strip():
                        self.add("  " + line)
            number += 1
        self.blank()

    def render_table(self, node: Node) -> None:
        rows = []
        for tr in find_all(node, "tr"):
            cells = [
                inline_children(c).replace("|", r"\|")
                for c in tr.children
                if c.text is None and c.tag in {"th", "td"}
            ]
            if cells:
                rows.append(cells)
        if not rows:
            return
        width = max(len(row) for row in rows)
        rows = [row + [""] * (width - len(row)) for row in rows]
        self.blank()
        self.add("| " + " | ".join(rows[0]) + " |")
        self.add("| " + " | ".join(["---"] * width) + " |")
        for row in rows[1:]:
            self.add("| " + " | ".join(row) + " |")
        self.blank()


def normalize_headings(body: str, title: str) -> str:
    result = []
    h1_seen = False
    for line in body.splitlines():
        match = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
        if not match:
            result.append(line)
            continue
        level = len(match.group(1))
        text = match.group(2).strip()
        if level == 1:
            if not h1_seen:
                result.append("# " + title)
                h1_seen = True
            else:
                result.append("## " + text)
        else:
            result.append("#" * level + " " + text)
    if not h1_seen:
        result.insert(0, "# " + title)
    return "\n".join(result)


def clean_body(body: str) -> str:
    body = re.sub(r"\n{3,}", "\n\n", body)
    lines = []
    for line in body.splitlines():
        if line.strip():
            lines.append(line.rstrip())
        elif lines and lines[-1] != "":
            lines.append("")
    while lines and not lines[-1].strip():
        lines.pop()
    return "\n".join(lines)


def generate_markdown(html_file: Path) -> str:
    source = html_file.read_text(encoding="utf-8", errors="replace")
    root = parse_html(source)
    title = get_title(root)
    original_content = get_content_root(root)
    description = get_description(root, original_content)
    content = clone_content(original_content)
    
    renderer = Renderer()
    body = renderer.render(content)
    body = normalize_headings(body, title)
    body = clean_body(body)
    
    return front_matter(title, description) + "\n" + body + "\n"


# ============================================================================
# VALIDATION
# ============================================================================

def parse_front_matter(markdown: str) -> tuple[dict[str, str], str]:
    match = re.match(r"\A---\r?\n(.*?)\r?\n---\r?\n?", markdown, re.DOTALL)
    if not match:
        return {}, markdown
    fields = {}
    for line in match.group(1).splitlines():
        field_match = re.match(r"^([A-Za-z][A-Za-z0-9_-]*):[ \t]*(.*)$", line)
        if not field_match:
            continue
        key, value = field_match.group(1), field_match.group(2).strip()
        if len(value) >= 2 and value[0] == '"' and value[-1] == '"':
            value = value[1:-1].replace('\\"', '"').replace("\\\\", "\\")
        fields[key] = value
    return fields, markdown[match.end():]


def validate_markdown(markdown: str, html_file: Path) -> list[str]:
    errors = []
    fields, body = parse_front_matter(markdown)

    if not fields:
        errors.append("Missing or invalid YAML front matter.")
        return errors

    for field_name in ("title", "author", "date", "description"):
        if not fields.get(field_name, "").strip():
            errors.append(f"Invalid or missing front matter field: {field_name}")

    if fields.get("author") != AUTHOR:
        errors.append(f'Invalid author: expected "{AUTHOR}", got "{fields.get("author")}".')
    if fields.get("date") != DATE:
        errors.append(f'Invalid date: expected {DATE}, got "{fields.get("date")}".')

    h1s = re.findall(r"^#\s+.+$", body, re.MULTILINE)
    if len(h1s) != 1:
        errors.append(f"Expected exactly one H1; found {len(h1s)}.")

    if not body.strip():
        errors.append("Markdown body is empty.")
    if re.search(r"\]\(\s*\)", body):
        errors.append("Empty Markdown link found.")
    if re.search(r"!\[[^\]]*\]\(\s*\)", body):
        errors.append("Empty Markdown image found.")
    if len(re.findall(r"^```", body, re.MULTILINE)) % 2:
        errors.append("Unclosed fenced code block.")

    body_no_code = re.sub(r"```.*?```", "", body, flags=re.DOTALL)
    for tag in ("script", "style", "iframe", "svg", "canvas"):
        if re.search(rf"<\s*{tag}(?:\s|>)", body_no_code, re.IGNORECASE):
            errors.append(f"Forbidden HTML element found: <{tag}>.")

    return errors


# ============================================================================
# FILE MANAGEMENT & SERVICES
# ============================================================================

def find_html_files(root: Path) -> list[Path]:
    files = []
    for path in root.rglob("*"):
        if path.is_file() and path.name.lower() in HTML_NAMES:
            relative = path.relative_to(root)
            if not any(part.startswith(".") for part in relative.parts):
                files.append(path)
    return sorted(files)


def atomic_write(path: Path, content: str) -> None:
    with tempfile.NamedTemporaryFile(
        "w",
        encoding="utf-8",
        dir=path.parent,
        prefix=f".{path.name}.",
        suffix=".tmp",
        delete=False,
    ) as temp:
        temp.write(content)
        temp.flush()
        temp_path = Path(temp.name)
    try:
        temp_path.replace(path)
    except Exception:
        temp_path.unlink(missing_ok=True)
        raise


def cleanup_backups(root: Path) -> int:
    """Find and remove all .bak files in root."""
    count = 0
    for bak_file in root.rglob("*.bak"):
        if bak_file.is_file():
            bak_file.unlink()
            count += 1
    return count


def restart_apache() -> bool:
    """Attempt to restart Apache service."""
    for service_cmd in [["systemctl", "restart", "apache2"], ["systemctl", "restart", "httpd"]]:
        try:
            res = subprocess.run(service_cmd, capture_output=True, text=True)
            if res.returncode == 0:
                print(f"[+] Apache restarted successfully via '{' '.join(service_cmd)}'.")
                return True
        except FileNotFoundError:
            continue
    print("[-] Error: Failed to restart Apache. 'systemctl' or apache service not found.")
    return False


# ============================================================================
# MAIN EXECUTION
# ============================================================================

def main() -> int:
    parser = argparse.ArgumentParser(
        description="Convert HTML index pages to AI-friendly Markdown."
    )
    parser.add_argument(
        "directory",
        nargs="?",
        default="/var/www/html",
        help="Website root directory. Default: /var/www/html",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Run validation only without modifying any files.",
    )
    args = parser.parse_args()

    target_dir = Path(args.directory).resolve()
    if not target_dir.exists() or not target_dir.is_dir():
        print(f"Error: Target directory '{target_dir}' does not exist.")
        return 1

    html_files = find_html_files(target_dir)
    if not html_files:
        print(f"No target index HTML files found in {target_dir}.")
        return 0

    print(f"[*] Target directory: {target_dir}")
    print(f"[*] Found {len(html_files)} HTML files.\n")

    # STEP 1: Dry-run testing phase
    print("=== PHASE 1: Executing Dry-Run Test ===")
    dry_run_failed = False
    for html_file in html_files:
        md_content = generate_markdown(html_file)
        errors = validate_markdown(md_content, html_file)
        if errors:
            dry_run_failed = True
            print(f"[-] [FAIL] {html_file.relative_to(target_dir)}")
            for err in errors:
                print(f"    - {err}")
        else:
            print(f"[+] [PASS] {html_file.relative_to(target_dir)}")

    if dry_run_failed:
        print("\n[-] Dry run failed with errors. Aborting operation.")
        return 1

    print("\n[+] Dry run passed successfully!")

    if args.dry_run:
        print("[*] --dry-run flag was set. Stopping execution before write.")
        return 0

    # STEP 2: Applying Markdown & Backups
    print("\n=== PHASE 2: Applying Markdown Conversion ===")
    for html_file in html_files:
        md_content = generate_markdown(html_file)
        output_file = html_file.parent / "index.md"
        
        # Create backup if file exists
        if output_file.exists():
            backup_path = output_file.with_name("index.md.bak")
            shutil.copy2(output_file, backup_path)

        atomic_write(output_file, md_content)
        print(f"[+] Written: {output_file.relative_to(target_dir)}")

    # STEP 3: Post-apply Validation
    print("\n=== PHASE 3: Validating Output ===")
    validation_failed = False
    for html_file in html_files:
        output_file = html_file.parent / "index.md"
        written_content = output_file.read_text(encoding="utf-8")
        errors = validate_markdown(written_content, html_file)
        if errors:
            validation_failed = True
            print(f"[-] [FAIL] {output_file.relative_to(target_dir)}")
            for err in errors:
                print(f"    - {err}")

    if validation_failed:
        print("\n[-] Validation failed after applying changes. Preserving backups.")
        return 1

    print("[+] All applied files validated successfully!")

    # STEP 4: Delete .bak Files
    print("\n=== PHASE 4: Cleaning Up Backup Files ===")
    deleted_count = cleanup_backups(target_dir)
    print(f"[+] Removed {deleted_count} .bak files.")

    # STEP 5: Restart Apache
    print("\n=== PHASE 5: Restarting Apache Service ===")
    restart_apache()

    print("\n[+] All tasks completed successfully.")
    return 0


if __name__ == "__main__":
    sys.exit(main())

nano html_to_ai_markdown.py
chmod +x html_to_ai_markdown.py
python3 ./html_to_ai_markdown.py /var/www/html --dry-run
python3 ./html_to_ai_markdown.py /var/www/html