listcontents - list all the files along with their content, perfect to give to AI

listcontents.py

#!/usr/bin/env python3
import argparse
import os
import magic
import pathlib
import logging
from typing import List, Optional

# Set up logging
logging.basicConfig(level=logging.WARNING)
logger = logging.getLogger(__name__)

def is_binary_file(file_path: str) -> bool:
    """
    Check if a file is binary by scanning its content for null bytes.

    Args:
        file_path (str): Path to the file to check.

    Returns:
        bool: True if file is binary, False otherwise.
    """
    try:
        # Check if the file can be opened and read
        with open(file_path, 'rb') as f:
            # Read the first 1024 bytes for analysis
            chunk = f.read(1024)
            # A file is binary if it contains a null byte
            return b'\x00' in chunk
    except Exception as e:
        logger.warning(f"Error checking if file is binary ({file_path}): {str(e)}")
        return True  # Treat as binary if an error occurs

def is_excluded(path: str, exclude_patterns: Optional[List[str]]) -> bool:
    """
    Check if a path should be excluded based on exclude patterns.
    
    Args:
        path (str): Path to check
        exclude_patterns (List[str]): List of patterns to exclude
        
    Returns:
        bool: True if path should be excluded, False otherwise
    """
    if not exclude_patterns:
        return False
        
    # Normalize path to use forward slashes for consistent matching
    norm_path = path.replace(os.sep, '/')
    
    for pattern in exclude_patterns:
        # Normalize pattern too
        norm_pattern = pattern.replace(os.sep, '/')
        
        # Ensure pattern ends with slash if it's meant to be a directory
        if not norm_pattern.endswith('/'):
            norm_pattern_dir = norm_pattern + '/'
        else:
            norm_pattern_dir = norm_pattern
            
        # Check exact match, directory match, or if path starts with pattern for directories
        if (norm_path == norm_pattern or 
            norm_path.startswith(norm_pattern_dir) or
            ('/' + norm_pattern) in norm_path):
            return True
            
    return False

def should_process_file(file_path: str, extensions: Optional[List[str]], exclude_patterns: Optional[List[str]]) -> bool:
    """
    Determine if a file should be processed based on its extension and exclusion patterns.
    
    Args:
        file_path (str): Path to the file
        extensions (List[str]): List of allowed extensions (None means all)
        exclude_patterns (List[str]): List of patterns to exclude
        
    Returns:
        bool: True if file should be processed, False otherwise
    """
    try:
        # Check if file matches any exclude patterns
        if is_excluded(file_path, exclude_patterns):
            return False
        
        # If no extensions specified, process all files
        if not extensions:
            return True
        
        # Check if file extension matches any in the allowed list
        file_ext = os.path.splitext(file_path)[1].lower()
        return file_ext in extensions
    except Exception as e:
        logger.warning(f"Error checking file processing criteria ({file_path}): {str(e)}")
        return False

def process_file(file_path: str, base_dir: str) -> None:
    """
    Process a single file and print its contents.
    
    Args:
        file_path (str): Path to the file to process
        base_dir (str): Base directory for creating relative paths
    """
    try:
        # Check if file exists and is accessible
        if not os.path.exists(file_path):
            print(f"// {file_path}")
            print("<File not found>")
            print()
            return
            
        if not os.access(file_path, os.R_OK):
            print(f"// {file_path}")
            print("<Permission denied>")
            print()
            return
            
        # Get relative path
        try:
            rel_path = os.path.relpath(file_path, base_dir)
        except ValueError:
            # Handle case where file_path and base_dir are on different drives
            rel_path = file_path
            
        print(f"// {rel_path}")
        
        # Handle binary files
        if is_binary_file(file_path):
            print("<Binary file>")
            print()
            return
        
        # Print file contents
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                print(f.read())
            print()
        except UnicodeDecodeError:
            print("<File contains invalid Unicode characters>")
            print()
        except PermissionError:
            print("<Permission denied>")
            print()
        except OSError as e:
            print(f"<Error reading file: {str(e)}>")
            print()
    except Exception as e:
        print(f"// {file_path}")
        print(f"<Error processing file: {str(e)}>")
        print()

def safe_walk(top, exclude_patterns=None, **kwargs):
    """
    A safe version of os.walk that handles permission errors and respects exclude patterns.
    """
    try:
        for root, dirs, files in os.walk(top, **kwargs):
            # Check if current directory should be excluded
            if is_excluded(root, exclude_patterns):
                dirs[:] = []  # Don't process subdirectories
                continue
                
            # Remove directories we should exclude or can't access
            dirs[:] = [d for d in dirs if os.access(os.path.join(root, d), os.R_OK) and 
                      not is_excluded(os.path.join(root, d), exclude_patterns)]
            
            # Filter files that we can access
            files = [f for f in files if os.access(os.path.join(root, f), os.R_OK)]
            
            yield root, dirs, files
    except Exception as e:
        logger.warning(f"Error walking directory {top}: {str(e)}")
        yield top, [], []

def main():
    parser = argparse.ArgumentParser(description='Print contents of files in directory')
    parser.add_argument('--dir', default=os.getcwd(),
                      help='Starting directory (default: current directory)')
    parser.add_argument('--extensions', nargs='+',
                      help='List of file extensions to include (e.g., .py .txt)')
    parser.add_argument('--max-depth', '-md', type=int,
                      help='Maximum directory depth to traverse')
    parser.add_argument('--exclude', '-e', nargs='+', default=[],
                      help='Patterns to exclude (e.g., node_modules/ vendor/)')
    parser.add_argument('--skip-binary', action='store_true',
                      help='Skip binary files')
    parser.add_argument('--follow-links', action='store_true',
                      help='Follow symbolic links')
    parser.add_argument('--include-hidden', '-ih', action='store_true',
                      help='Include hidden files and directories (default: False)')
    parser.add_argument('--verbose', '-v', action='store_true',
                      help='Enable verbose logging')
    
    args = parser.parse_args()
    
    if args.verbose:
        logger.setLevel(logging.INFO)
    
    # Convert extensions to lowercase and ensure they start with dot
    extensions = None
    if args.extensions:
        extensions = [ext.lower() if ext.startswith('.') else f'.{ext.lower()}'
                     for ext in args.extensions]
    
    try:
        # Walk through directory
        for root, dirs, files in safe_walk(args.dir, exclude_patterns=args.exclude, followlinks=args.follow_links):
            try:
                # Skip hidden directories unless include_hidden is True
                if not args.include_hidden:
                    dirs[:] = [d for d in dirs if not d.startswith('.')]
                
                # Check depth
                if args.max_depth is not None:
                    try:
                        current_depth = root[len(args.dir):].count(os.sep)
                        if current_depth >= args.max_depth:
                            dirs[:] = []  # Don't go deeper
                            continue
                    except Exception as e:
                        logger.warning(f"Error checking directory depth: {str(e)}")
                        continue
                
                # Process files
                for file in sorted(files):
                    try:
                        if not args.include_hidden and file.startswith('.'):  # Skip hidden files unless include_hidden is True
                            continue
                            
                        file_path = os.path.join(root, file)
                        
                        if not is_excluded(file_path, args.exclude) and should_process_file(file_path, extensions, None):
                            if args.skip_binary and is_binary_file(file_path):
                                continue
                            process_file(file_path, args.dir)
                    except Exception as e:
                        logger.warning(f"Error processing file {file}: {str(e)}")
                        continue
                        
            except Exception as e:
                logger.warning(f"Error processing directory {root}: {str(e)}")
                continue
                
    except Exception as e:
        logger.error(f"Fatal error: {str(e)}")
        return 1
        
    return 0

if __name__ == '__main__':
    exit(main())