aidoc.nim

import std / [strutils, os, parseopt, sets]
import "$nim" / compiler / [lexer, llstream, pathutils, parser, ast, options, idents, renderer, syntaxes, msgs]

const
  Usage = """
Nim AI Documentation Generator

Usage: nimaidoc [options] <input-file>

Options:
  -o, --output-dir:<dir>     Output directory (default: same as input)
  -h, --help                 Show this help message

Examples:
  nimaidoc raylib.nim
  nimaidoc -o:docs src/module.nim
"""
  HiddenPragmas = ["importc", "header"]

type
  AIDocGenerator = object
    outFile: File
    processedFiles: HashSet[string]

# ----------------------------------------------------------------------------
# Setup
# ----------------------------------------------------------------------------

proc createConf(): ConfigRef =
  result = newConfigRef()
  #result.notes.excl hintLineTooLong
  result.errorMax = 1000

proc parseFile(inputFile: string): PNode =
  let stream = llStreamOpen(AbsoluteFile inputFile, fmRead)
  if stream == nil:
    quit "cannot open file: " & inputFile
  else:
    var conf = createConf()
    let fileIdx = fileInfoIdx(conf, AbsoluteFile inputFile)
    var parser: Parser = default(Parser)
    syntaxes.openParser(parser, fileIdx, stream, newIdentCache(), conf)
    result = parseAll(parser)
    closeParser(parser)
    if conf.errorCounter > 0:
      quit QuitFailure

# ----------------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------------

proc getTypeStr(n: PNode): string =
  result = ""
  var r = initTokRender(n, {renderNoBody, renderNoComments, renderDocComments})
  var kind = tkEof
  var tok = ""
  while true:
    getNextTok(r, kind, tok)
    if kind == tkEof: break
    result.add tok

proc getComment(n: PNode): string =
  if n.comment.len > 0: n.comment.strip() else: ""

proc getName(n: PNode; exported: var bool): string =
  case n.kind
  of nkPostfix:
    exported = true
    result = getName(n[1], exported)
  of nkPragmaExpr: result = getName(n[0], exported)
  of nkSym: result = n.sym.name.s
  of nkIdent: result = n.ident.s
  of nkAccQuoted:
    for i in 0..<n.len: result.add(getName(n[i], exported))
  of nkOpenSymChoice, nkClosedSymChoice, nkOpenSym:
    result = getName(n[0], exported)
  else:
    result = ""

proc getPragmaSeq(n: PNode): seq[string] =
  result = @[]
  for i in 0..<n.len:
    let c = n[i]
    case c.kind
    of nkIdent:
      let name = c.ident.s
      if name notin HiddenPragmas:
        result.add name
    of nkExprColonExpr:
      if c[0].kind == nkIdent:
        let key = c[0].ident.s
        if key notin HiddenPragmas:
          let valNode = c[1]
          case valNode.kind
          of nkStrLit:
            result.add key & ": \"" & valNode.strVal & "\""
          of nkIdent:
            result.add key & ": " & valNode.ident.s
          else:
            let txt = getTypeStr(valNode).strip()
            if txt.len > 0:
              result.add key & ": " & txt
    else:
      let txt = getTypeStr(c).strip()
      if txt.len > 0:
        result.add txt

proc attachedPragmas(nameNode: PNode): seq[string] =
  if nameNode.kind == nkPragmaExpr and nameNode.len > 1 and nameNode[1].kind == nkPragma:
    return getPragmaSeq(nameNode[1])
  @[]

# ----------------------------------------------------------------------------
# Declarations (now assume 'name' is exported and non-empty)
# ----------------------------------------------------------------------------

proc processConst(gen: var AIDocGenerator; n: PNode; name: string) =
  let valueStr =
    if n.len > 2 and n[2].kind != nkEmpty: getTypeStr(n[2]).strip() else: ""
  let pragmas = attachedPragmas(n[0])
  let comment = getComment(n)

  gen.outFile.writeLine "<const name=\"" & name & "\">"
  if valueStr.len > 0:
    gen.outFile.writeLine "VALUE: " & valueStr
  if pragmas.len > 0:
    gen.outFile.writeLine "PRAGMAS:"
    for p in pragmas: gen.outFile.writeLine "  " & p
  if comment.len > 0:
    gen.outFile.writeLine "DESCRIPTION: " & comment
  gen.outFile.writeLine "</const>"
  gen.outFile.writeLine ""

proc processEnum(gen: var AIDocGenerator; n: PNode; name: string) =
  let enumTy = n[2]
  let pragmas = attachedPragmas(n[0])
  let comment = getComment(n)
  gen.outFile.writeLine "<enum name=\"" & name & "\">"
  if pragmas.len > 0:
    gen.outFile.writeLine "PRAGMAS:"
    for p in pragmas: gen.outFile.writeLine "  " & p
  gen.outFile.writeLine "MEMBERS:"
  for i in 1..<enumTy.len:
    let m = enumTy[i]
    if m.kind == nkEmpty: continue
    var memberName = ""
    var valueStr = ""
    var memberComment = ""
    case m.kind
    of nkEnumFieldDef:
      if m.len > 0:
        var dummy = false
        memberName = getName(m[0], dummy)
      if m.len > 1 and m[1].kind != nkEmpty:
        valueStr = getTypeStr(m[1]).strip()
      memberComment = getComment(m)
    of nkIdent:
      memberName = m.ident.s
      memberComment = getComment(m)
    else:
      continue
    if memberName.len == 0: continue
    var line = "  " & memberName
    if valueStr.len > 0: line.add " = " & valueStr
    if memberComment.len > 0: line.add "          # " & memberComment
    gen.outFile.writeLine line
  if comment.len > 0:
    gen.outFile.writeLine "DESCRIPTION: " & comment
  gen.outFile.writeLine "</enum>"
  gen.outFile.writeLine ""

proc processObject(gen: var AIDocGenerator; n: PNode; name: string) =
  var rhs = n[2]
  if rhs.kind == nkPragmaExpr and rhs.len > 0:
    rhs = rhs[0]
  let pragmas = attachedPragmas(n[0])
  let comment = getComment(n)

  gen.outFile.writeLine "<object name=\"" & name & "\">"
  if pragmas.len > 0:
    gen.outFile.writeLine "PRAGMAS:"
    for p in pragmas: gen.outFile.writeLine "  " & p
  if rhs.len > 2 and rhs[2].kind == nkRecList:
    let recList = rhs[2]
    if recList.len > 0:
      gen.outFile.writeLine "FIELDS:"
      for field in recList:
        if field.kind != nkIdentDefs or field.len < 3: continue
        let fieldType = getTypeStr(field[field.len-2]).strip()
        let fieldComment = getComment(field)
        for j in 0..<field.len-2:
          var fe = false
          # Keep exporting policy for fields: only include those marked with '*'
          let fname = getName(field[j], fe)
          if not fe: continue
          var line = "  " & fname & ": " & fieldType
          if fieldComment.len > 0:
            line.add "               # " & fieldComment
          gen.outFile.writeLine line
  if comment.len > 0:
    gen.outFile.writeLine "DESCRIPTION: " & comment
  gen.outFile.writeLine "</object>"
  gen.outFile.writeLine ""

proc processTuple(gen: var AIDocGenerator; n: PNode; name: string) =
  var rhs = n[2]
  if rhs.kind == nkPragmaExpr and rhs.len > 0:
    rhs = rhs[0]
  let pragmas = attachedPragmas(n[0])
  let comment = getComment(n)

  gen.outFile.writeLine "<tuple name=\"" & name & "\">"
  if pragmas.len > 0:
    gen.outFile.writeLine "PRAGMAS:"
    for p in pragmas: gen.outFile.writeLine "  " & p
  gen.outFile.writeLine "FIELDS:"
  var idx = 0
  for part in rhs:
    if part.kind == nkIdentDefs and part.len >= 3:
      let typeStr = getTypeStr(part[part.len-2]).strip()
      let partComment = getComment(part)
      for j in 0 ..< part.len-2:
        if part[j].kind != nkIdent: continue
        var dummy = false
        let fname = getName(part[j], dummy)
        var line = "  " & fname
        if typeStr.len > 0: line.add ": " & typeStr
        if partComment.len > 0: line.add "               # " & partComment
        gen.outFile.writeLine line
        inc idx
    else:
      if part.kind notin {nkEmpty, nkIdent, nkSym}:
        let t = getTypeStr(part).strip()
        if t.len > 0:
          gen.outFile.writeLine "  item" & $idx & ": " & t
          inc idx
  if comment.len > 0:
    gen.outFile.writeLine "DESCRIPTION: " & comment
  gen.outFile.writeLine "</tuple>"
  gen.outFile.writeLine ""

proc processConcept(gen: var AIDocGenerator; n: PNode; name: string) =
  var rhs = n[2]
  if rhs.kind == nkPragmaExpr and rhs.len > 0:
    rhs = rhs[0]
  let pragmas = attachedPragmas(n[0])
  let comment = getComment(n)

  gen.outFile.writeLine "<concept name=\"" & name & "\">"
  var defStr = ""
  for i in 0..<rhs.len:
    let part = rhs[i]
    if part.kind == nkIdentDefs and part.len >= 2:
      defStr.add "concept "
      for j in 0 ..< part.len-2:
        if part[j].kind == nkIdent:
          defStr.add part[j].ident.s
          if j < part.len-3: defStr.add ", "
      defStr.add ": "
      defStr.add getTypeStr(part[part.len-2])
      break
  if defStr.len > 0:
    gen.outFile.writeLine "DEFINITION: " & defStr
  if pragmas.len > 0:
    gen.outFile.writeLine "PRAGMAS:"
    for p in pragmas: gen.outFile.writeLine "  " & p
  if comment.len > 0:
    gen.outFile.writeLine "DESCRIPTION: " & comment
  gen.outFile.writeLine "</concept>"
  gen.outFile.writeLine ""

proc routineTag(k: TNodeKind): string =
  case k
  of nkProcDef: "proc"
  of nkFuncDef: "func"
  of nkMethodDef: "method"
  of nkIteratorDef: "iterator"
  of nkTemplateDef: "template"
  of nkMacroDef: "macro"
  of nkConverterDef: "converter"
  else: "proc"

proc processRoutine(gen: var AIDocGenerator; n: PNode; name: string) =
  let tag = routineTag(n.kind)
  let comment = getComment(n)
  var pragmas: seq[string] = @[]
  for i in 0..<n.len:
    if n[i].kind == nkPragma:
      for p in getPragmaSeq(n[i]): pragmas.add p

  gen.outFile.writeLine "<" & tag & " name=\"" & name & "\">"

  var gParams: PNode = nil
  var fParams: PNode = nil
  for i in 0..<n.len:
    let ch = n[i]
    if ch.kind == nkGenericParams and gParams.isNil:
      gParams = ch
    elif ch.kind == nkFormalParams and fParams.isNil:
      fParams = ch

  if gParams != nil and gParams.len > 0:
    gen.outFile.writeLine "GENERIC_PARAMS:"
    for gp in gParams:
      if gp.kind != nkIdentDefs or gp.len < 3: continue
      let constraintNode = gp[gp.len-2]
      let constraintStr = getTypeStr(constraintNode).strip()
      for j in 0..<gp.len-2:
        if gp[j].kind != nkIdent: continue
        let gname = gp[j].ident.s
        if constraintStr.len > 0:
          gen.outFile.writeLine "  " & gname & ": " & constraintStr
        else:
          gen.outFile.writeLine "  " & gname

  if fParams != nil and fParams.len > 1:
    gen.outFile.writeLine "PARAMETERS:"
    for i in 1..<fParams.len:
      let group = fParams[i]
      if group.kind != nkIdentDefs or group.len < 3: continue
      let typeStr = getTypeStr(group[group.len-2]).strip()
      let defaultStr = getTypeStr(group[group.len-1]).strip()
      let paramComment = getComment(group)
      for j in 0..<group.len-2:
        if group[j].kind != nkIdent: continue
        var line = "  " & group[j].ident.s
        if typeStr.len > 0: line.add ": " & typeStr
        if defaultStr.len > 0: line.add " = " & defaultStr
        if paramComment.len > 0:
          let pad = max(0, 30 - line.len)
          if pad > 0: line.add repeat(' ', pad)
          line.add "# " & paramComment
        gen.outFile.writeLine line

  if pragmas.len > 0:
    gen.outFile.writeLine "PRAGMAS:"
    for p in pragmas:
      gen.outFile.writeLine "  " & p
  if comment.len > 0:
    gen.outFile.writeLine "DESCRIPTION: " & comment

  gen.outFile.writeLine "</" & tag & ">"
  gen.outFile.writeLine ""

# ----------------------------------------------------------------------------
# Tree traversal (centralized export decision)
# ----------------------------------------------------------------------------

proc processNode(gen: var AIDocGenerator; n: PNode) =
  case n.kind
  of nkNone..nkNilLit: discard
  of nkConstSection:
    for c in n:
      if c.kind == nkConstDef:
        var exported = false
        let name = getName(c[0], exported)
        if exported and name.len > 0:
          processConst(gen, c, name)
  of nkTypeSection:
    for t in n:
      if t.kind != nkTypeDef or t.len < 3: continue
      var exported = false
      let name = getName(t[0], exported)
      if not exported or name.len == 0: continue
      var impl = t[2]
      var bare = impl
      if bare.kind == nkPragmaExpr and bare.len > 0:
        bare = bare[0]
      case bare.kind
      of nkEnumTy: processEnum(gen, t, name)
      of nkObjectTy: processObject(gen, t, name)
      of nkTupleTy: processTuple(gen, t, name)
      of nkTypeClassTy: processConcept(gen, t, name)
      else: discard
  of nkWhenStmt:
    processNode(gen, n[0])
  of routineDefs:
    var exported = false
    let name = getName(n[0], exported)
    if exported:
      processRoutine(gen, n, name)
  else:
    for child in n:
      processNode(gen, child)

# ----------------------------------------------------------------------------
# Driver
# ----------------------------------------------------------------------------

proc generateAIDoc(gen: var AIDocGenerator; inputFile: string; outputDir: string) =
  let abs = expandFilename(inputFile)
  if gen.processedFiles.contains(abs): return
  gen.processedFiles.incl(abs)
  echo "Processing: ", inputFile
  let ast = parseFile(inputFile)
  let namePart = splitFile(inputFile).name
  let outPath = outputDir / (namePart & ".aidoc")
  if open(gen.outFile, outPath, fmWrite):
    processNode(gen, ast)
    close(gen.outFile)
    echo "Generated: ", outPath
  else:
    echo "Failed to create output file: ", outPath

proc showHelp() = echo Usage

proc main() =
  var inputFile = ""
  var outputDir = ""
  var p = initOptParser()
  while true:
    next(p)
    case p.kind
    of cmdEnd: break
    of cmdShortOption, cmdLongOption:
      case p.key
      of "o", "outputdir": outputDir = p.val
      of "h", "help":
        showHelp()
        quit QuitSuccess
      else:
        echo "Unknown option: ", p.key
        showHelp()
        quit QuitFailure
    of cmdArgument:
      if inputFile.len == 0:
        inputFile = p.key
      else:
        echo "Multiple input files not supported"
        quit QuitFailure
  if inputFile.len == 0:
    echo "No input file specified"
    showHelp()
    quit QuitFailure
  if not fileExists(inputFile):
    echo "Input file does not exist: ", inputFile
    quit QuitFailure
  if outputDir.len == 0:
    outputDir = splitFile(inputFile).dir
  createDir(outputDir)
  var gen = AIDocGenerator()
  generateAIDoc(gen, inputFile, outputDir)

when isMainModule:
  main()

entry_diff.py

#!/usr/bin/env python3
import sys
import re
import difflib

def parse_entries(content):
    """Parse content into entries based on XML tags"""
    entries = []
    lines = content.split('\n')
    i = 0
    
    while i < len(lines):
        line = lines[i].strip()
        if line.startswith('<') and not line.startswith('</'):
            # Found opening tag
            entry_lines = [lines[i]]
            tag_match = re.match(r'<(\w+)([^>]*)>', line)
            if tag_match:
                tag_name = tag_match.group(1)
                closing_tag = f'</{tag_name}>'
                
                # Collect all lines until closing tag
                i += 1
                while i < len(lines) and not lines[i].strip().startswith(closing_tag):
                    entry_lines.append(lines[i])
                    i += 1
                
                # Add closing tag if found
                if i < len(lines):
                    entry_lines.append(lines[i])
                
                entries.append('\n'.join(entry_lines))
        i += 1
    
    return entries

def main():
    if len(sys.argv) != 3:
        print("Usage: entry_diff.py <file1> <file2>")
        sys.exit(1)
    
    with open(sys.argv[1], 'r') as f1, open(sys.argv[2], 'r') as f2:
        content1 = f1.read()
        content2 = f2.read()
    
    entries1 = parse_entries(content1)
    entries2 = parse_entries(content2)
    
    # Create a simple diff by comparing entries
    print("=== ENTRY-LEVEL DIFF ===")
    print()
    
    # For simplicity, we'll just show which entries changed
    # In a real implementation, we'd do more sophisticated comparison
    
    # Convert to sets for comparison
    set1 = set(entries1)
    set2 = set(entries2)
    
    added = set2 - set1
    removed = set1 - set2
    unchanged = set1 & set2
    
    if removed:
        print("REMOVED ENTRIES:")
        for entry in removed:
            print("-" * 40)
            print(entry)
            print()
    
    if added:
        print("ADDED ENTRIES:")
        for entry in added:
            print("-" * 40)
            print(entry)
            print()
    
    # For modified entries, show detailed diff
    common_names = {}
    for entry in entries1:
        first_line = entry.split('\n')[0]
        common_names[first_line] = entry
    
    for entry in entries2:
        first_line = entry.split('\n')[0]
        if first_line in common_names and common_names[first_line] != entry:
            print("MODIFIED ENTRY:")
            print("-" * 40)
            diff = difflib.unified_diff(
                common_names[first_line].split('\n'),
                entry.split('\n'),
                fromfile='old',
                tofile='new',
                lineterm=''
            )
            for line in diff:
                print(line)
            print()

if __name__ == '__main__':
    main()