Skip to content

Instantly share code, notes, and snippets.

@isocroft

isocroft/decode.go

Forked from nl5887/decode.go
Created March 9, 2026 01:21
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save isocroft/0ed7efcc6d3e8cfdea60b14cbded644d to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save isocroft/0ed7efcc6d3e8cfdea60b14cbded644d to your computer and use it in GitHub Desktop.
Download ZIP
Golang implementation for RFC 1342: Non-ASCII Mail Headers
Raw
decode.go
package main
import (
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"mime/quotedprintable"
"regexp"
"strings"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/charmap"
"golang.org/x/text/encoding/unicode"
)
func decoder(encoding string) (*encoding.Decoder, error) {
if strings.ToUpper(encoding) == "UTF-8" {
return unicode.UTF8.NewDecoder(), nil
} else if strings.ToUpper(encoding) == "ISO-8859-1" {
return charmap.ISO8859_1.NewDecoder(), nil
} else {
return nil, fmt.Errorf("Unknown encoding")
}
}
func decodeHeader(str string) (string, error) {
re := regexp.MustCompile(`\=\?(?P<charset>.*?)\?(?P<encoding>.*)\?(?P<body>.*?)\?(.*?)\=`)
matches := re.FindAllStringSubmatch(str, -1)
if len(matches) == 0 {
return str, nil
}
for _, match := range matches {
var r io.Reader = strings.NewReader(match[3])
if match[2] == "Q" {
r = quotedprintable.NewReader(r)
} else if match[2] == "B" {
r = base64.NewDecoder(base64.StdEncoding, r)
}
if d, err := decoder(match[1]); err == nil {
r = d.Reader(r)
}
if val, err := ioutil.ReadAll(r); err == nil {
str = strings.Replace(str, match[0], string(val), -1)
} else if err != nil {
fmt.Println(err.Error())
continue
}
}
return str, nil
}
func main() {
fmt.Println(decodeHeader("=?UTF-8?Q?=F3=BE=AC=8D_?= ... Laten we ontmoeten! Ik woon in de buurt .."))
fmt.Println(decodeHeader("=?ISO-8859-1?Q?Keld_J=F8rn_Simonsen?= <keld@dkuug.dk>"))
}
@isocroft

isocroft commented May 7, 2026 •
edited
Loading

Copy link
Copy Markdown
Author 
package deptree

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"debugging"
)

type Node struct {
	Path         string
	Dependencies []*Node
}

// @INFO: Global root node for the dependency tree
var rootNode *Node

// @INFO: Keeps track of already analyzed files to avoid circular recursion
var visited = map[string]bool{}

// @INFO: Keeps track of currently visited node for the dependency tree (since DFS is used)
var currentNodeVisited *Node = nil

/*
 * readSourceFile:
 *
 * read the contents of a source file
 */
func readSourceFile(sourceFilePath string) (string, error) {
	content, err := os.ReadFile(sourceFilePath)
	if err != nil {
		return "", fmt.Errorf("failed to read source file %s: %w", sourceFilePath, err)
	}

	return string(content), nil
}

/*
 * resolveRelativeImport:
 *
 * converts relative imports into absolute paths.
 *
 * Examples:
 *	"./utils"     -> /project/utils.js
 *	"../config"   -> /project/config.py
 */
func resolveRelativeImport(currentFile string, importPath string, extension string) string {
	if filepath.Ext(importPath) == "" {
		importPath += extension
	}

	if filepath.IsAbs(importPath) {
		return filepath.Clean(importPath)
	}

	baseDir := filepath.Dir(currentFile)

	return filepath.Clean(filepath.Join(baseDir, importPath))
}

/*
 * analyze:
 *
 * scans and tracks each source file content and matches
 * import patterns based on file extension while building a
 * dependency tree.
*/
func analyze(sourceFileContent string, sourceFilePath string) (bool, error) {
	extension := filepath.Ext(sourceFilePath)

	/* @NOTE: 
	      Simplified regex approach for brevity; in production,
	      use tree-sitter or AST parsers.
	*/
	importPatterns := map[string]*regexp.Regexp{
		".js":  regexp.MustCompile(`import.*from\s+['"](.*)['"]`),
		".py":  regexp.MustCompile(`(?:import|from)\s+([a-zA-Z0-9_./-]+)`),
		".php": regexp.MustCompile(`(?:include|require)(?:_once)?\s*['"](.*)['"]`),
	}

	importRegex, exists := importPatterns[extension]
	if !exists {
		return false, fmt.Errorf("unsupported file extension: %s", extension)
	}
	
	/*
	   @HINT: 
	   Resolve the root source file path before assigning
          it to the current visited node.
	*/
	var absoluteSourceFilePath string
       var err error

        if filepath.IsAbs(sourceFilePath) {
                 absoluteSourceFilePath = sourceFilePath
                 //err = nil
        } else {
                 // @HINT: Used '=' not ':=' here to avoid shadowing
                 absoluteSourceFilePath, err = filepath.Abs(sourceFilePath)
        }
	
	if err != nil {
		return false, fmt.Errorf(
			"failed to resolve absolute path for %s: %w",
			sourceFilePath,
			err,
		)
	}
	
	

	if rootNode == nil {
		rootNode = &Node{
			Path:         filepath.Clean(absoluteSourceFilePath),
			Dependencies: make([]*Node, 0),
		}
		currentNodeVisited = rootNode
	}


	if visited[absoluteSourceFilePath] {
	  /*
		Prevent infinite recursion from cyclic dependencies
	  */
	       return true, nil
	}

	visited[absoluteSourceFilePath] = true

	/*
	   @INFO: Recursive walk logic starts here...
	*/

	// Find all import statements
	matches := importRegex.FindAllStringSubmatch(sourceFileContent, -1)

	for _, matchPair := range matches {
		if len(matchPair) < 2 {
			continue
		}
		
		if len(matchPair) > 2 {
		   matchPair = matchPair[:2]
		}

               matchedImportPathString = matchPair[1]
		importPath := strings.TrimSpace(matchedImportPathString)

		// @HINT: Ignore package imports (e.g. "react", "numpy")
		if !strings.HasPrefix(importPath, ".") {
			continue
		}

		// @HINT: Resolve dependency path
		resolvedPath := resolveRelativeImport(
			absoluteSourceFilePath,
			importPath,
			extension,
		)


		dependencyNode := &Node{
			Path:         resolvedPath,
			Dependencies: make([]*Node, 0),
		}

              if currentNodeVisited != nil {
		     // @HINT: Attach dependency to currently visited node and update the visited node
		     currentNodeVisited.Dependencies = append(
			   currentNodeVisited.Dependencies,
			   dependencyNode,
		     )
		     currentNodeVisited = dependencyNode
	     } else {
                    Assert(currentNodeVisited == nil, "invariant failed: `currentVisitedNode` must not be `nil`")
            }

		// Read dependency file contents
		dependencyContent, err := readSourceFile(resolvedPath)
		
		if err != nil {
			return false, err
		}

		// Recursively call `analyze`
		_, err = analyze(dependencyContent, resolvedPath)
		
		if err != nil {
			return false, err
		}
	}

	return true, nil
}

/*
  * GenerateAll:
  *
  * build out a complete dependency tree for all source files in
  * a given software project root folder.
  */
func GenerateAll (entryPointSourceFilePath string) (*Node, error) {
	var absoluteSourceFilePath string
       var err error

        if filepath.IsAbs(sourceFilePath) {
                 absoluteSourceFilePath = entryPointSourceFilePath
                 //err = nil
        } else {
                 // @HINT: Used '=' not ':=' here to avoid shadowing
                 absoluteSourceFilePath, err = filepath.Abs(entryPointSourceFilePath)
        }
        
	if err != nil {
		return struct{}, fmt.Errorf(
			"failed to resolve absolute path for %s: %w",
			entryPointSourceFilePath,
			err,
		)
	}
   
       sourceRootContent, err := readSourceFile(entryPointSourceFilePath)
       
       if err != nil {
          return struct{}, fmt.Errorf("Could not generate tree; reason: %w", err, err)
       }

       _, err := analyze(sourceRootContent, absoluteSourceFilePath);
       
       if err != nill {
          return struct{}, fmt.Errorf("Could not generate tree; reason: %w", err, err)
       }
       
       return rootNode, nil
}

@isocroft

isocroft commented May 7, 2026 •
edited
Loading

Copy link
Copy Markdown
Author

assert function in Go until go implements it as a language feature

package debugging

import (
  "log"
  "fmt"
)

func Assert (condition bool, message string) {
  if condition == false {
     log.Errorf(message);
     panic(fmt.Sprintf("assertion error: %s", message))
   }
}

@isocroft

isocroft commented May 7, 2026 •
edited
Loading

Copy link
Copy Markdown
Author 
package callgraph

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	sitter "github.com/smacker/go-tree-sitter"
	"github.com/smacker/go-tree-sitter/javascript"
	"github.com/smacker/go-tree-sitter/php"
	"github.com/smacker/go-tree-sitter/python"
)

/*
	Node represents a dependency tree node.

	-- This is assumed to come from your dependency tree package. --
*/
type Node struct {
	Path         string
	Dependencies []*Node
}

type FunctionNode struct {
	ID string
	// e.g. "main.py:doSomething"
	// e.g. "datastore:seeder.py:run_with_schema"
	// e.g. "events.emitters:broadcast.py:normalizeEgress"
	// e.g. "system:root.go:Liftoff"
	// e.g. "main.go:init"
	// e.g. "index.js:collectInput"
	// e.g. "root:index.php:array_push"

	Callees []string
	// e.g. ["print", "saveUser", "db.insert"]

	Signature string
	// e.g. lightweight non-cryptographic hash
	// derived from function name + params
}

/*
	Call map
*/
var callMap = make(map[string][]FunctionNode)








/*
	CallGraph represents a directed graph:

	Caller ---> Callee
*/
type CallGraph struct {

	/*
		Adjacency list:

		FunctionName -> Set of Callees
	*/
	graph map[string]map[string]struct{}

	/*
		Set of all discovered function nodes
	*/
	nodes map[string]struct{}
}

/*
	NewCallGraph creates and initializes
	a new call graph.
*/
func NewCallGraph() *CallGraph {
	return &CallGraph{
		graph: make(map[string]map[string]struct{}),
		nodes: make(map[string]struct{}),
	}
}

/*
	AddCall adds a directed edge:

		caller ---> callee
*/
func (cg *CallGraph) AddCall(
	caller string,
	callee string,
) {

	/*
		Initialize caller adjacency set
		if it does not exist.
	*/
	if _, exists := cg.graph[caller]; !exists {
		cg.graph[caller] = make(map[string]struct{})
	}

	/*
		Add callee into caller adjacency set
	*/
	cg.graph[caller][callee] = struct{}{}

	/*
		Register both nodes
	*/
	cg.nodes[caller] = struct{}{}
	cg.nodes[callee] = struct{}{}
}

/*
	GetSuccessorNodes returns all functions
	called by the given function.
*/
func (cg *CallGraph) GetSuccessorNodes(
	functionName string,
) []string {

	callees, exists := cg.graph[functionName]

	if !exists {
		return []string{}
	}

	results := make([]string, 0, len(callees))

	for callee := range callees {
		results = append(results, callee)
	}

	return results
}

/*
	HasNode checks whether a function exists
	in the graph.
*/
func (cg *CallGraph) HasNode(
	functionName string,
) bool {

	_, exists := cg.nodes[functionName]

	return exists
}

/*
	GetAllNodes returns all graph nodes.
*/
func (cg *CallGraph) GetAllNodes() []string {

	results := make([]string, 0, len(cg.nodes))

	for node := range cg.nodes {
		results = append(results, node)
	}

	return results
}

/*
	String implements fmt.Stringer.

	Equivalent to Python's __repr__.
*/
func (cg *CallGraph) String() string {
	return fmt.Sprintf("%v", cg.graph)
}





/*

==== PYTHON PORT ====

from collections import defaultdict

class CallGraph:
    def __init__(self):
        # Adjacency list: Function -> Set of Callees
        self.graph = defaultdict(set)
        self.nodes = set()

    def add_call(self, caller, callee):
        """Adds a directed edge from caller to callee."""
        self.graph[caller].add(callee)
        self.nodes.add(caller)
        self.nodes.add(callee)

    def get_successor_nodes(self, function_name):
        """
        API: Returns a list of functions called by the given function.
        """
        return list(self.graph.get(function_name, []))

    def __repr__(self):
        return str(dict(self.graph))
*/







/*
	GetLanguage resolves a tree-sitter language grammar
	from a source file extension.
*/
func getLanguage(extension string) (*sitter.Language, error) {
	switch extension {
	case ".js":
		return javascript.GetLanguage(), nil

	case ".py":
		return python.GetLanguage(), nil

	case ".php":
		return php.GetLanguage(), nil

	default:
		return nil, fmt.Errorf(
			"no matching source file extension: %s",
			extension,
		)
	}
}

/*
	readSourceFile reads source code contents.
*/
func readSourceFile(sourceFilePath string) ([]byte, error) {
	content, err := os.ReadFile(sourceFilePath)
	if err != nil {
		return nil, fmt.Errorf(
			"failed to read source file %s: %w",
			sourceFilePath,
			err,
		)
	}

	return content, nil
}

/*
	getNodeText extracts node text from source code.
*/
func getNodeText(node *sitter.Node, source []byte) string {
	if node == nil {
		return ""
	}

	return string(source[node.StartByte():node.EndByte()])
}

/*
	findChildrenByType recursively searches for child nodes
	of a given type.
*/
func findChildrenByType(
	node *sitter.Node,
	nodeType string,
	results *[]*sitter.Node,
) {
	if node == nil {
		return
	}

	if node.Type() == nodeType {
		*results = append(*results, node)
	}

	for i := 0; i < int(node.ChildCount()); i++ {
		findChildrenByType(
			node.Child(i),
			nodeType,
			results,
		)
	}
}

/*
	extractFunctionName extracts function identifiers
	from tree-sitter function definition nodes.
*/
func extractFunctionName(
	node *sitter.Node,
	source []byte,
) string {
	if node == nil {
		return ""
	}

	for i := 0; i < int(node.NamedChildCount()); i++ {
		child := node.NamedChild(i)

		if child == nil {
			continue
		}

		switch child.Type() {
		case "identifier", "name":
			return getNodeText(child, source)
		}
	}

	return "anonymous"
}

/*
	extractCallName extracts callee names from call_expression nodes.
*/
func extractCallName(
	node *sitter.Node,
	source []byte,
) string {
	if node == nil {
		return ""
	}

	for i := 0; i < int(node.NamedChildCount()); i++ {
		child := node.NamedChild(i)

		if child == nil {
			continue
		}

		switch child.Type() {
		case "identifier", "member_expression", "attribute":
			return getNodeText(child, source)
		}
	}

	return ""
}


/*
	extractModuleName extracts module/package namespaces
	based on the source language.

       Examples:
	      Python:
		      /path/to/pkg/utils/helpers.py
		      => pkg.utils:helpers.py
      
	      PHP:
		      namespace Vendor\Module;
		      => Vendor\Module:index.php
		      
	      Go:
                    package name
                    => name:start.go
*/
func extractModuleName(
	sourceFilePath string,
	sourceCode []byte,
	extension string,
) string {

	switch extension {

       /*
            Go module extraction
       */
       case ".go":
              /*
                     Matches:
                          package name;
		*/
		packageRegex := regexp.MustCompile(
			`(?m)^\s*package\s+([a-zA-Z0-9_]+)\s*`,
		)

		match := packageRegex.FindSubmatch(sourceCode)
		moduleName := "main"
		
		if len(match) > 1 {
			moduleName = string(match[1])
		}
		 
               return moduleName + ":" + filepath.Base(sourceFilePath)

	/*
		Python module extraction
	*/
	case ".py":

		normalizedPath := filepath.ToSlash(sourceFilePath)

		/*
			Remove base filename:
			pkg/utils/helpers.py
			=> pkg/utils
		*/
		withoutBase := strings.TrimSuffix(
			normalizedPath,
			filepath.Base(normalizedPath),
		)

		/*
			Convert path separators into dots:
			pkg/utils
			=> pkg.utils
		*/
		moduleName := strings.ReplaceAll(
			withoutBase,
			"/",
			".",
		)

		/*
			Optionally remove leading dots
		*/
		moduleName = strings.TrimPrefix(
			moduleName,
			".",
		)

		return moduleName + ":" + filepath.Base(sourceFilePath)

	/*
		PHP namespace extraction
	*/
	case ".php":

		/*
			Matches:
				namespace Vendor\Module;
		*/
		namespaceRegex := regexp.MustCompile(
			`(?m)^\s*namespace\s+([a-zA-Z0-9_\\]+)\s*;`,
		)

		match := namespaceRegex.FindSubmatch(sourceCode)
		nameSpace := "root"
		
		if len(match) > 1 {
			nameSpace = string(match[1])
		}
		
		moduleName := strings.ReplaceAll(
			nameSpace,
			"\\",
			".",
		)
		
		/*
			Optionally remove leading dots
		*/
		moduleName = strings.TrimPrefix(
			moduleName,
			".",
		)

		return moduleName + ":" + filepath.Base(sourceFilePath)

	default:
		return filepath.Base(sourceFilePath)
	}
}



/*
	extractFunctionParameters extracts the raw parameter
	list from a function definition node.
*/
func extractFunctionParameters(
	node *sitter.Node,
	source []byte,
) string {

	if node == nil {
		return ""
	}

	for i := 0; i < int(node.NamedChildCount()); i++ {

		child := node.NamedChild(i)

		if child == nil {
			continue
		}

		switch child.Type() {

		/*
			JavaScript:
				formal_parameters

			Python:
				parameters

			PHP:
				formal_parameters
		*/
		case "formal_parameters",
			"parameters":

			return getNodeText(
				child,
				source,
			)
		}
	}

	return ""
}


/*
	buildSignature creates a lightweight pseudo-signature.
*/
func buildSignature(functionName string, parameters string) string {
       signatureSeed := fmt.Sprintf(
		"%s(%s)_%x",
		functionName,
		strings.TrimSpace(parameters),
		len(parameters)+len(strings.TrimSpace(functionName))
	)

       /*
          @NOTE:
	   In production, use xxHash3 / SHA-1 / FNV-1a
      */
	return fmt.Sprintf(
		"sig_%s",
		signatureSeed,
	)
}

/*
	scanForFunctionCalls:

	scans each source file for all function calls and maps each
	into a function-node struct.
*/
func scanForFunctionCalls(
	sourceFilePath string,
) ([]FunctionNode, error) {

	/*
		@HINT
		Initialize Tree-sitter parser
	*/
	parser := sitter.NewParser()

	extension := filepath.Ext(sourceFilePath)

	language, err := getLanguage(extension)
	if err != nil {
		return nil, err
	}

	parser.SetLanguage(language)

	/*
		@HINT
		Parse file into a tree
	*/
	sourceCode, err := readSourceFile(sourceFilePath)
	if err != nil {
		return nil, err
	}

	tree, err := parser.ParseCtx(
		nil,
		nil,
		sourceCode,
	)

	if err != nil {
		return nil, fmt.Errorf(
			"failed to parse %s: %w",
			sourceFilePath,
			err,
		)
	}

	rootNode := tree.RootNode()

	/*
		@HINT:
		Query the tree for 'call_expression',
		'function_declaration' and 'function_definition'
	*/

	var functionNodes []*sitter.Node
	var callNodes []*sitter.Node

	findChildrenByType(
		rootNode,
		"function_definition",
		&functionNodes,
	)

	findChildrenByType(
		rootNode,
		"function_declaration",
		&functionNodes,
	)

	findChildrenByType(
		rootNode,
		"call_expression",
		&callNodes,
	)

	results := make([]FunctionNode, 0)

	/*
		Map function definitions
	*/
	for _, fnNode := range functionNodes {

		functionName := extractFunctionName(
			fnNode,
			sourceCode,
		)

		moduleName := extractModuleName(
	              sourceFilePath,
	              sourceCode,
	              extension,
              )
              
              functionID := fmt.Sprintf(
	              "%s:%s",
	              moduleName,
	              functionName,
              )
              
              parameters := extractFunctionParameters(
	              fnNode,
	              sourceCode,
              )

		functionEntry := FunctionNode{
			ID:        functionID,
			Callees:   make([]string, 0),
			Signature: buildSignature(moduleName + ":" + functionName, strings.TrimPrefix(parameters, functionName+
			"(")),
		}

		/*
			Find calls inside the function body
		*/
		var nestedCalls []*sitter.Node

		findChildrenByType(
			fnNode,
			"call_expression",
			&nestedCalls,
		)

		for _, callNode := range nestedCalls {
			callName := extractCallName(
				callNode,
				sourceCode,
			)

			if callName == "" {
				continue
			}

			functionEntry.Callees = append(
				functionEntry.Callees,
				callName,
			)
		}

		results = append(results, functionEntry)
	}

	/*
		@INFO: 
		Handle loose/global calls
		(not inside functions)
	*/
	if len(functionNodes) == 0 && len(callNodes) > 0 {
	
	       moduleName := extractModuleName(
	              sourceFilePath,
	              sourceCode,
	              extension,
              )
              
              parameters := extractFunctionParameters(
	              rootNode,
	              sourceCode,
              )

		globalNode := FunctionNode{
			ID: fmt.Sprintf(
				"%s:<global>",
				moduleName,
			),
			Callees:   make([]string, 0),
			Signature: buildSignature(moduleName + ":global_scope", parameters),
		}

		for _, callNode := range callNodes {
			callName := extractCallName(
				callNode,
				sourceCode,
			)

			if callName == "" {
				continue
			}

			globalNode.Callees = append(
				globalNode.Callees,
				callName,
			)
		}

		results = append(results, globalNode)
	}

	return results, nil
}

/*
	walkDependencyTree recursively traverses the dependency tree
	and generates function call mappings.
*/
func walkDependencyTree(
	node *Node,
	visited map[string]bool,
) error {

	if node == nil {
		return nil
	}

	absolutePath := node.Path

	if visited[absolutePath] {
		return nil
	}

	visited[absolutePath] = true

	functionCalls, err := scanForFunctionCalls(
		absolutePath,
	)

	if err != nil {
		return err
	}

	callMap[absolutePath] = functionCalls

	for _, dependency := range node.Dependencies {
		err := walkDependencyTree(
			dependency,
			visited,
		)

		if err != nil {
			return err
		}
	}

	return nil
}

/*
	GenerateAll:

	generate the static call map using the dependency tree.
*/
func GenerateAll(
	rootNode *Node,
) (map[string][]FunctionNode, error) {

	if rootNode == nil {
		return nil, fmt.Errorf(
			"root dependency node cannot be nil",
		)
	}

	visited := make(map[string]bool)

	err := walkDependencyTree(
		rootNode,
		visited,
	)

	if err != nil {
		return nil, err
	}

	return callMap, nil
}

@isocroft

isocroft commented May 24, 2026 •
edited
Loading

Copy link
Copy Markdown
Author 
package gitsifter

/*
	LIMITATIONS / CONCERNS:

	1. Aliasing:
		Handled using alias extraction maps.

	2. Dynamic Calls:
		Static analysis cannot fully resolve:

			PHP:
				$func()

			JS:
				eval()

			Python:
				getattr()

		These require runtime instrumentation.

	3. Performance:
		Files are parsed concurrently per commit
		using goroutines and worker pools.
*/

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"

	"github.com/go-git/go-billy/v5"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing/object"

	sitter "github.com/smacker/go-tree-sitter"
	"github.com/smacker/go-tree-sitter/javascript"
	"github.com/smacker/go-tree-sitter/php"
	"github.com/smacker/go-tree-sitter/python"
	"github.com/smacker/go-tree-sitter/golang"
)

/*
	To determine if an interface changed,
	you cannot just check whether the function
	was edited.

	You must:

	1. Extract parameter list using Tree-sitter
	2. Normalize whitespace/comments
	3. Hash parameter/type/order definition
	4. Compare against previous commit
*/

type InterfaceChange struct {
	FunctionName string
	CommitHash   string
	ChangeCount  int
}

/*
	FunctionSignature represents a normalized
	function interface.
*/
type FunctionSignature struct {
	Name      string
	Signature string
}

/*
	TrackChanges scans git history and tracks
	interface changes across commits.
*/
func TrackChanges(
	repoPath string,
) (map[string]int, error) {

	repo, err := git.PlainOpen(repoPath)
	if err != nil {
		return nil, err
	}

	iter, err := repo.Log(&git.LogOptions{})
	if err != nil {
		return nil, err
	}

       /* @INFO:
		Map:
			functionID -> total count for signature hash mismatch across commits
	*/
	stats := make(map[string]int)

	/* @INFO:
		Map:
			functionID -> previous signature hash
	*/
	previousSignatures := make(map[string]string)

	/* @HINT:
		Protect shared maps during concurrent parsing
	*/
	var mu sync.Mutex

	err = iter.ForEach(func(
		c *object.Commit,
	) error {

		/*
			@HINT:
			Get file tree at this commit
		*/
		tree, err := c.Tree()
		if err != nil {
			return err
		}

		/* @HINT:
			Collect candidate source files
		*/
		sourceFiles := make([]*object.File, 0)

		err = tree.Files().ForEach(func(
			f *object.File,
		) error {

			ext := filepath.Ext(f.Name)

			switch ext {
			case ".js", ".py", ".php", ".go":
				sourceFiles = append(
					sourceFiles,
					f,
				)
			}

			return nil
		})

		if err != nil {
			return err
		}

		/*
			@HINT:
			
			Parse files concurrently for performance.

			Big O(C x F) runtime complexity mitigation:
				- Parallelize within commit
		*/

		workerCount := runtime.NumCPU()

		fileChan := make(chan *object.File)
		errorChan := make(chan error, 1)

		var wg sync.WaitGroup

		for i := 0; i < workerCount; i++ {

			wg.Add(1)

			go func() {
				defer wg.Done()

				for file := range fileChan {

					functions, err := parseFunctionsFromFile(
						file,
					)

					if err != nil {
						errorChan <- err
						return
					}

					mu.Lock()

					/*
						@HINT:
						Compare function signatures
						against previous commit
					*/
					for _, fn := range functions {

						oldSignature,
							exists := previousSignatures[fn.Name]

						/*
							@HINT:
							
							If changed:
								increment count for signature hash mismatch across commits
						*/
						if exists &&
							oldSignature != fn.Signature {

							stats[fn.Name]++
						}

						previousSignatures[fn.Name] =
							fn.Signature
					}

					mu.Unlock()
				}
			}()
		}

		go func() {
			defer close(fileChan)

			for _, file := range sourceFiles {
				fileChan <- file
			}
		}()

		wg.Wait()

		select {
		case err := <-errorChan:
			return err
		default:
		}

		return nil
	})

	return stats, err
}

/*
	parseFunctionsFromFile parses all functions
	and extracts normalized interface hashes.
*/
func parseFunctionsFromFile(
	file *object.File,
) ([]FunctionSignature, error) {

	  reader, err := file.Reader()
         if err != nil {
	       return nil, err
         }

         defer reader.Close()

          /* @HINT:
	       Read file incrementally in chunks
	       instead of loading entire file at once.
         */
         const chunkSize = 32 * 1024   // @INFO: 32KB

         buffer := make([]byte, chunkSize)

        var sourceCode bytes.Buffer

        for {

	      bytesRead, readErr := reader.Read(buffer)

	      if bytesRead > 0 {

		     _, writeErr := sourceCode.Write(
			   buffer[:bytesRead],
		     )

		     if writeErr != nil {
			  return nil, writeErr
		     }
	      }

	      if readErr != nil {

		    /* @HINT:
			  EOF means we're done reading.
		    */
		    if readErr.Error() == "EOF" {
			  break
		    }

		    return nil, readErr
	     }
        }

	extension := filepath.Ext(file.Name)

	parser := sitter.NewParser()

	language, err := getLanguage(extension)
	if err != nil {
		return nil, err
	}

	parser.SetLanguage(language)

	tree, err := parser.ParseCtx(
		nil,
		nil,
		sourceCode.Bytes(),
	)

	if err != nil {
		return nil, err
	}

	root := tree.RootNode()

	functionNodes := make([]*sitter.Node, 0)

	findFunctionNodes(
		root,
		&functionNodes,
	)

	results := make([]FunctionSignature, 0)

	/*
		Alias resolution map:

		Python:
			import long_name as ln

		JS:
			import * as api from "./x";

		PHP:
			use Vendor\Service as Svc;

              Go:
                     import snp "https://github.com/corvis/snipper"
	*/
	aliases := extractAliases(
		sourceCode.Bytes(),
		extension,
	)

	for _, fnNode := range functionNodes {

		functionName := extractFunctionName(
			fnNode,
			sourceCode.Bytes(),
		)

		parameters := extractParameters(
			fnNode,
			sourceCode.Bytes(),
		)

               resultType := extractResultType(
                      fnNode,
			sourceCode.Bytes(),
               )

		/* @HINT:
			Normalize interface:
				remove comments/whitespace
		*/
		normalizedParameters :=
			normalizeInterface(parameters)

               /*   @HINT:
			Resolve aliases
		*/
		functionName =
			resolveAliasedName(
				functionName,
				aliases,
			)

		/*  @HINT:
			Hash normalized params, fuction name and result type
		*/
		signatureHash :=
			hashSignature(functionName + "|" + normalizedParameters + "|" + resultType)

		results = append(
			results,
			FunctionSignature{
				Name:      functionName,
				Signature: signatureHash,
			},
		)
	}

	return results, nil
}

/*
	getLanguage resolves Tree-sitter language.
*/
func getLanguage(
	extension string,
) (*sitter.Language, error) {

	switch extension {

	case ".js":
		return javascript.GetLanguage(), nil

	case ".py":
		return python.GetLanguage(), nil

	case ".php":
		return php.GetLanguage(), nil
		
       case ".go:
              return go.GetLanguage(), nil
              
	default:
		return nil,
			fmt.Errorf(
				"unsupported extension: %s",
				extension,
			)
	}
}

/*
	findFunctionNodes recursively finds
	function definitions.
*/
func findFunctionNodes(
	node *sitter.Node,
	results *[]*sitter.Node,
) {

	if node == nil {
		return
	}

	switch node.Type() {

	case "function_definition",
		"function_declaration",
		"method_definition":

		*results = append(
			*results,
			node,
		)
	}

	for i := 0; i < int(node.ChildCount()); i++ {

		findFunctionNodes(
			node.Child(i),
			results,
		)
	}
}

/*
	extractFunctionName extracts function identifier.
*/
func extractFunctionName(
	node *sitter.Node,
	source []byte,
) string {

	for i := 0; i < int(node.NamedChildCount()); i++ {

		child := node.NamedChild(i)

		if child == nil {
			continue
		}

		switch child.Type() {

		case "identifier",
			"name":

			return string(
				source[
					child.StartByte():
						child.EndByte()
				],
			)
		}
	}

	return "anonymous"
}

/*
	extractParameters extracts raw parameter text.
*/
func extractParameters(
	node *sitter.Node,
	source []byte,
) string {

	for i := 0; i < int(node.NamedChildCount()); i++ {

		child := node.NamedChild(i)

		if child == nil {
			continue
		}

		switch child.Type() {

		case "formal_parameters",
			"parameters":

			return string(
				source[
					child.StartByte():
						child.EndByte()
				],
			)
		}
	}

	return ""
}

/*
	normalizeInterface removes comments
	and unnecessary whitespace.
*/
func normalizeInterface(
	parameters string,
) string {

	/*
		Remove block comments
	*/
	blockComments :=
		regexp.MustCompile(`/\*[\s\S]*?\*/`)

	parameters =
		blockComments.ReplaceAllString(
			parameters,
			"",
		)

	/*
		Remove line comments
	*/
	lineComments :=
		regexp.MustCompile(`//.*|#.*`)

	parameters =
		lineComments.ReplaceAllString(
			parameters,
			"",
		)

	/*
		Collapse whitespace
	*/
	whitespace :=
		regexp.MustCompile(`\s+`)

	parameters =
		whitespace.ReplaceAllString(
			parameters,
			"",
		)

	return strings.TrimSpace(parameters)
}

/*
	hashSignature hashes normalized interface.
*/
func hashSignature(
	normalized string,
) string {

	hash := sha1.Sum(
		[]byte(normalized),
	)

	return hex.EncodeToString(hash[:])
}

/*
	extractAliases extracts import aliases.

	Examples:

	Python:
		import numpy as np

	JavaScript:
		import * as api from "./api"

	PHP:
		use Vendor\Service as Svc;

	Go:
		import svc "project/service"

	Go grouped:
		import (
			db "project/database"
			httpx "project/http"
		)
*/
func extractAliases(
	source []byte,
	extension string,
) map[string]string {

	results := make(map[string]string)

	content := string(source)

	switch extension {

	/*
		Python aliases
	*/
	case ".py":

		/*
			import numpy as np
		*/
		importAliasRegex := regexp.MustCompile(
			`import\s+([a-zA-Z0-9_.]+)\s+as\s+([a-zA-Z0-9_]+)`,
		)

		importMatches :=
			importAliasRegex.FindAllStringSubmatch(
				content,
				-1,
			)

		for _, match := range importMatches {

			if len(match) < 3 {
				continue
			}

			original := strings.TrimSpace(match[1])
			alias := strings.TrimSpace(match[2])

			results[alias] = original
		}

		/*
			from package import module as alias
		*/
		fromImportRegex := regexp.MustCompile(
			`from\s+([a-zA-Z0-9_.]+)\s+import\s+([a-zA-Z0-9_]+)\s+as\s+([a-zA-Z0-9_]+)`,
		)

		fromMatches :=
			fromImportRegex.FindAllStringSubmatch(
				content,
				-1,
			)

		for _, match := range fromMatches {

			if len(match) < 4 {
				continue
			}

			fullImport := fmt.Sprintf(
				"%s.%s",
				match[1],
				match[2],
			)

			alias := strings.TrimSpace(match[3])

			results[alias] = fullImport
		}

	/*
		JavaScript aliases
	*/
	case ".js":

		/*
			import * as api from "./api"
		*/
		namespaceImportRegex := regexp.MustCompile(
			`import\s+\*\s+as\s+([a-zA-Z0-9_]+)\s+from\s+['"]([^'"]+)['"]`,
		)

		namespaceMatches :=
			namespaceImportRegex.FindAllStringSubmatch(
				content,
				-1,
			)

		for _, match := range namespaceMatches {

			if len(match) < 3 {
				continue
			}

			alias := strings.TrimSpace(match[1])
			original := strings.TrimSpace(match[2])

			results[alias] = original
		}

		/*
			import { something as alias } from ...
		*/
		namedImportRegex := regexp.MustCompile(
			`import\s+\{([^}]+)\}\s+from`,
		)

		namedMatches :=
			namedImportRegex.FindAllStringSubmatch(
				content,
				-1,
			)

		for _, match := range namedMatches {

			if len(match) < 2 {
				continue
			}

			imports :=
				strings.Split(match[1], ",")

			for _, item := range imports {

				item = strings.TrimSpace(item)

				if strings.Contains(item, " as ") {

					parts :=
						strings.Split(item, " as ")

					if len(parts) != 2 {
						continue
					}

					original :=
						strings.TrimSpace(parts[0])

					alias :=
						strings.TrimSpace(parts[1])

					results[alias] = original
				}
			}
		}

	/*
		PHP aliases
	*/
	case ".php":

		/*
			use Vendor\Service as Svc;
		*/
		phpUseRegex := regexp.MustCompile(
			`use\s+([a-zA-Z0-9_\\]+)\s+as\s+([a-zA-Z0-9_]+)`,
		)

		phpMatches :=
			phpUseRegex.FindAllStringSubmatch(
				content,
				-1,
			)

		for _, match := range phpMatches {

			if len(match) < 3 {
				continue
			}

			original := strings.TrimSpace(match[1])
			alias := strings.TrimSpace(match[2])

			results[alias] = original
		}

	/*
		Go aliases
	*/
	case ".go":

		/*
			Single-line import aliases:

				import svc "project/service"
		*/
		goSingleImportRegex := regexp.MustCompile(
			`import\s+([a-zA-Z0-9_]+)\s+"([^"]+)"`,
		)

		singleMatches :=
			goSingleImportRegex.FindAllStringSubmatch(
				content,
				-1,
			)

		for _, match := range singleMatches {

			if len(match) < 3 {
				continue
			}

			alias := strings.TrimSpace(match[1])
			original := strings.TrimSpace(match[2])

			results[alias] = original
		}

		/*
			Grouped imports:

				import (
					db "project/database"
					httpx "project/http"
				)
		*/
		groupImportRegex := regexp.MustCompile(
			`import\s*\(([\s\S]*?)\)`,
		)

		groupMatches :=
			groupImportRegex.FindAllStringSubmatch(
				content,
				-1,
			)

		for _, group := range groupMatches {

			if len(group) < 2 {
				continue
			}

			lines :=
				strings.Split(group[1], "\n")

			for _, line := range lines {

				line = strings.TrimSpace(line)

				if line == "" {
					continue
				}

				/*
					db "project/database"
				*/
				parts :=
					regexp.MustCompile(
						`^([a-zA-Z0-9_\.]+)\s+"([^"]+)"`,
					).FindStringSubmatch(line)

				if len(parts) < 3 {
					continue
				}

				alias := strings.TrimSpace(parts[1])
				original := strings.TrimSpace(parts[2])

				results[alias] = original
			}
		}
	}

	return results
}


/*
	extractResultType extracts the return/result type
	from a function definition node for multiple languages.

	Supported:
		- Go
		- JavaScript / TypeScript
		- Python
		- PHP

	Returns:
		- explicit type if available
		- "void" if no return type exists
		- "unknown" if dynamically inferred
*/
func extractResultType(
	node *sitter.Node,
	source []byte,
) string {

	if node == nil {
		return "unknown"
	}

	/*
		Language-specific Tree-sitter nodes:

		Go:
			result
			parameter_list

		TypeScript:
			type_annotation

		Python:
			type

		PHP:
			primitive_type
			union_type
			named_type
	*/

	for i := 0; i < int(node.NamedChildCount()); i++ {

		child := node.NamedChild(i)

		if child == nil {
			continue
		}

		switch child.Type() {

		/*
			========================
			Go
			========================

			func Add(a int, b int) int
			func GetUser() (*User, error)

			Tree-sitter:
				result
		*/
		case "result":

			resultText := strings.TrimSpace(
				string(
					source[
						child.StartByte():
							child.EndByte()
					],
				),
			)

			if resultText == "" {
				return "void"
			}

			return resultText

		/*
			========================
			TypeScript / JavaScript
			========================

			function x(): string

			Tree-sitter:
				type_annotation
		*/
		case "type_annotation":

			typeText := strings.TrimSpace(
				string(
					source[
						child.StartByte():
							child.EndByte()
					],
				),
			)

			typeText =
				strings.TrimPrefix(
					typeText,
					":",
				)

			typeText =
				strings.TrimSpace(typeText)

			if typeText == "" {
				return "unknown"
			}

			return typeText

		/*
			========================
			Python
			========================

			def add(a: int) -> str:

			Tree-sitter:
				type
		*/
		case "type":

			typeText := strings.TrimSpace(
				string(
					source[
						child.StartByte():
							child.EndByte()
					],
				),
			)

			if typeText == "" {
				return "unknown"
			}

			return typeText

		/*
			========================
			PHP
			========================

			function x(): string
			function x(): User|nil

			Tree-sitter:
				primitive_type
				union_type
				named_type
		*/
		case "primitive_type",
			"named_type",
			"union_type":

			typeText := strings.TrimSpace(
				string(
					source[
						child.StartByte():
							child.EndByte()
					],
				),
			)

			if typeText == "" {
				return "unknown"
			}

			return typeText
		}
	}

	/*
		Fallback heuristics for dynamic languages
		where explicit types may not exist.
	*/

	functionBody := strings.TrimSpace(
		string(
			source[
				node.StartByte():
					node.EndByte()
			],
		),
	)

	/*
		Python / JS / PHP heuristic:
			check if function has return statement
	*/
	if strings.Contains(
		functionBody,
		"return ",
	) {

		/*
			Return exists but type is dynamic
		*/
		return "dynamic"
	}

	return "void"
}


/*
	resolveAliasedName resolves aliases back
	to original source names.
*/
func resolveAliasedName(
	functionName string,
	aliases map[string]string,
) string {

	for alias, original := range aliases {

		if strings.HasPrefix(
			functionName,
			alias+".",
		) {

			return strings.Replace(
				functionName,
				alias,
				original,
				1,
			)
		}
	}

	return functionName
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment