find-file-count.py

import os
import csv
import glob


# Script to parse all csv files in folder and output the number of files in which each value in target column exists. And it can count files in subfolders and make columns for them in output.
# Simply put this where the csv files are and fire away.


# Some config

# Use script file's directory as target directory
pyfiledir = os.path.dirname(os.path.realpath(__file__))

output_path = pyfiledir + os.sep + 'output.csv'

target_col = 'To'

# The column in the output to use for sorting
sortby_col = 'Count'

# First n number of rows of a file, 0 for no limit
lines_limit = 100


# The input
files = glob.glob(pyfiledir+'/**/*.csv', recursive=True)

sub_folders = next(os.walk(pyfiledir))[1][::-1]
# print(sub_folders)

results = {}

print('\nProcess is starting...', '\n')
print(f'Lines Limit: {lines_limit}', '\n')

line_count = 0
file_count = 0

for filepath in files:
    if(filepath == output_path):
        continue
    file_count += 1
    current_file_line_count = 0
    with open(filepath, mode='r') as csv_file:
        csv_reader = csv.DictReader(csv_file)
        for row in csv_reader:
            if(target_col not in row):
                continue
            if(lines_limit != 0 and current_file_line_count >= lines_limit):
                break
            current_file_line_count += 1
            line_count += 1
            key = row[target_col]
            output_col = 'Count'
            if(key not in results):
                results[key] = {output_col: []}
            for col in sub_folders:
                if(col not in results[key]):
                    results[key][col] = []
                if((os.sep+col+os.sep) in filepath):
                    output_col = col
            if(filepath in results[key][output_col]):
                continue
            results[key][output_col].append(filepath)


print(f'Total number of lines processed: {line_count}', '\n')
print(f'Total number of files processed: {file_count}', '\n')
resultAsList = []

for key in results:
    data = {'String': key, 'Count': len(results[key]['Count'])}
    for col in results[key].keys():
        data[col] = len(results[key][col])
    resultAsList.append(data)


def sortFunc(e):
    return e[sortby_col]


resultAsList.sort(reverse=True, key=sortFunc)


# Write output
with open(output_path, mode='w', newline='') as csv_file:
    fieldnames = ['String', 'Count'] + sub_folders
    writer = csv.DictWriter(csv_file, fieldnames=fieldnames)

    writer.writeheader()

    for row in resultAsList:
        writer.writerow(row)


print('File written: ' + output_path, '\n')
print('Completed!', '\n')