Last active
July 14, 2026 09:16
-
-
Save gitgotgitgotit/9d0ef51270af4fcdecde7917af06e7a3 to your computer and use it in GitHub Desktop.
Python3 pack statsgen
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| # StatsGen - Password Statistical Analysis tool | |
| # | |
| # This tool is part of PACK (Password Analysis and Cracking Kit) | |
| # | |
| # VERSION 0.2.0 | |
| # | |
| # Original Copyright (C) 2013 Peter Kacherginsky | |
| # Python 3 port / fixes / new options, 2026 | |
| # | |
| # Please see the attached LICENSE file for additional licensing information. | |
| # | |
| # CHANGELOG vs 0.0.3 (original Python 2): | |
| # 0.0.4: | |
| # - Ported to Python 3 | |
| # - Fixed ZeroDivisionError on empty input / all-filtered-out input | |
| # - --hiderare applied consistently across all sections | |
| # - Output file handle properly closed; file-read errors handled cleanly | |
| # - Reads no longer crash on invalid/non-UTF8 bytes (surrogateescape) | |
| # - stdin support via "-" | |
| # - Removed dead code / unused imports | |
| # 0.2.0: | |
| # - Migrated optparse -> argparse (better help, error messages, no more | |
| # manual "wrong arg count" handling) | |
| # - Character classification is unicode-aware by default (accented / | |
| # non-ASCII letters now count as lower/upper instead of "special"); | |
| # --ascii-only restores the old strict-ASCII behavior | |
| # - Added median and stddev to the length summary | |
| # - -o/--output now exports the FULL stat set (length, charset, | |
| # simplemask, advancedmask, complexity) as CSV or JSON, not just | |
| # advanced masks. Format is auto-detected from the file extension or | |
| # set explicitly with --output-format. Exported data is always | |
| # complete and unfiltered by --hiderare/--top (those only affect what | |
| # prints to the console). | |
| # - Added --progress / --progress-interval for large files | |
| import sys | |
| import re | |
| import csv | |
| import json | |
| import time | |
| import string | |
| import argparse | |
| import operator | |
| VERSION = "0.2.0" | |
| VALID_CHARSETS = { | |
| 'numeric', 'loweralpha', 'upperalpha', 'special', 'mixedalpha', | |
| 'loweralphanum', 'upperalphanum', 'loweralphaspecial', 'upperalphaspecial', | |
| 'specialnum', 'mixedalphaspecial', 'upperalphaspecialnum', | |
| 'loweralphaspecialnum', 'mixedalphanum', 'all', | |
| } | |
| SIMPLEMASK_RE = re.compile(r'^(string|digit|special){1,3}$') | |
| def percentage(count, total): | |
| """Integer floor percentage; returns 0 if total is 0 instead of raising.""" | |
| if not total: | |
| return 0 | |
| return count * 100 // total | |
| def weighted_length_stats(length_counts, total): | |
| """Compute min/max/mean/median/stddev over a {length: count} distribution | |
| without materializing every individual length (works fine at any scale).""" | |
| if not total: | |
| return None | |
| mean = sum(l * c for l, c in length_counts.items()) / total | |
| variance = sum(c * (l - mean) ** 2 for l, c in length_counts.items()) / total | |
| stddev = variance ** 0.5 | |
| sorted_lengths = sorted(length_counts.items()) | |
| half = total / 2.0 | |
| cum = 0 | |
| median = sorted_lengths[0][0] | |
| for l, c in sorted_lengths: | |
| cum += c | |
| if cum >= half: | |
| median = l | |
| break | |
| return { | |
| 'min': sorted_lengths[0][0], | |
| 'max': sorted_lengths[-1][0], | |
| 'mean': mean, | |
| 'median': median, | |
| 'stddev': stddev, | |
| } | |
| class StatsGen: | |
| def __init__(self): | |
| # Filters | |
| self.minlength = None | |
| self.maxlength = None | |
| self.simplemasks = None | |
| self.charsets = None | |
| # Display (console only -- never affects exported file data) | |
| self.hiderare = False | |
| self.top = None | |
| # Behavior | |
| self.encoding = 'utf-8' | |
| self.ascii_only = False | |
| self.progress = False | |
| self.progress_interval = 100000 | |
| # Stats dictionaries | |
| self.stats_length = dict() | |
| self.stats_simplemasks = dict() | |
| self.stats_advancedmasks = dict() | |
| self.stats_charactersets = dict() | |
| self.filter_counter = 0 | |
| self.total_counter = 0 | |
| self.mindigit = None | |
| self.minupper = None | |
| self.minlower = None | |
| self.minspecial = None | |
| self.maxdigit = None | |
| self.maxupper = None | |
| self.maxlower = None | |
| self.maxspecial = None | |
| def _classify(self, letter): | |
| """Return one of 'digit', 'lower', 'upper', 'special' for a character.""" | |
| if self.ascii_only: | |
| if letter in string.digits: | |
| return 'digit' | |
| elif letter in string.ascii_lowercase: | |
| return 'lower' | |
| elif letter in string.ascii_uppercase: | |
| return 'upper' | |
| else: | |
| return 'special' | |
| else: | |
| # Unicode-aware: accented/non-Latin letters land in lower/upper | |
| # instead of being dumped into 'special'. | |
| if letter.isdecimal(): | |
| return 'digit' | |
| elif letter.islower(): | |
| return 'lower' | |
| elif letter.isupper(): | |
| return 'upper' | |
| else: | |
| return 'special' | |
| def analyze_password(self, password): | |
| pass_length = len(password) | |
| digit = 0 | |
| lower = 0 | |
| upper = 0 | |
| special = 0 | |
| simplemask = list() | |
| advancedmask_parts = [] | |
| for letter in password: | |
| kind = self._classify(letter) | |
| if kind == 'digit': | |
| digit += 1 | |
| advancedmask_parts.append("?d") | |
| group = 'digit' | |
| elif kind == 'lower': | |
| lower += 1 | |
| advancedmask_parts.append("?l") | |
| group = 'string' | |
| elif kind == 'upper': | |
| upper += 1 | |
| advancedmask_parts.append("?u") | |
| group = 'string' | |
| else: | |
| special += 1 | |
| advancedmask_parts.append("?s") | |
| group = 'special' | |
| if not simplemask or simplemask[-1] != group: | |
| simplemask.append(group) | |
| advancedmask_string = ''.join(advancedmask_parts) | |
| simplemask_string = ''.join(simplemask) if len(simplemask) <= 3 else 'othermask' | |
| policy = (digit, lower, upper, special) | |
| if digit and not lower and not upper and not special: | |
| charset = 'numeric' | |
| elif not digit and lower and not upper and not special: | |
| charset = 'loweralpha' | |
| elif not digit and not lower and upper and not special: | |
| charset = 'upperalpha' | |
| elif not digit and not lower and not upper and special: | |
| charset = 'special' | |
| elif not digit and lower and upper and not special: | |
| charset = 'mixedalpha' | |
| elif digit and lower and not upper and not special: | |
| charset = 'loweralphanum' | |
| elif digit and not lower and upper and not special: | |
| charset = 'upperalphanum' | |
| elif not digit and lower and not upper and special: | |
| charset = 'loweralphaspecial' | |
| elif not digit and not lower and upper and special: | |
| charset = 'upperalphaspecial' | |
| elif digit and not lower and not upper and special: | |
| charset = 'specialnum' | |
| elif not digit and lower and upper and special: | |
| charset = 'mixedalphaspecial' | |
| elif digit and not lower and upper and special: | |
| charset = 'upperalphaspecialnum' | |
| elif digit and lower and not upper and special: | |
| charset = 'loweralphaspecialnum' | |
| elif digit and lower and upper and not special: | |
| charset = 'mixedalphanum' | |
| else: | |
| charset = 'all' | |
| return (pass_length, charset, simplemask_string, advancedmask_string, policy) | |
| def generate_stats(self, filename): | |
| """ Generate password statistics. """ | |
| if filename == '-': | |
| f = sys.stdin | |
| else: | |
| f = open(filename, 'r', encoding=self.encoding, errors='surrogateescape') | |
| start_time = time.time() | |
| try: | |
| for password in f: | |
| password = password.rstrip('\r\n') | |
| if len(password) == 0: | |
| continue | |
| self.total_counter += 1 | |
| if self.progress and self.total_counter % self.progress_interval == 0: | |
| elapsed = time.time() - start_time | |
| rate = self.total_counter / elapsed if elapsed > 0 else 0 | |
| sys.stderr.write("\r[*] Processed %d passwords (%.0f/sec)..." % (self.total_counter, rate)) | |
| sys.stderr.flush() | |
| (pass_length, characterset, simplemask, advancedmask, policy) = self.analyze_password(password) | |
| (digit, lower, upper, special) = policy | |
| if (self.charsets is None or characterset in self.charsets) and \ | |
| (self.simplemasks is None or simplemask in self.simplemasks) and \ | |
| (self.maxlength is None or pass_length <= self.maxlength) and \ | |
| (self.minlength is None or pass_length >= self.minlength): | |
| self.filter_counter += 1 | |
| if self.mindigit is None or digit < self.mindigit: self.mindigit = digit | |
| if self.maxdigit is None or digit > self.maxdigit: self.maxdigit = digit | |
| if self.minupper is None or upper < self.minupper: self.minupper = upper | |
| if self.maxupper is None or upper > self.maxupper: self.maxupper = upper | |
| if self.minlower is None or lower < self.minlower: self.minlower = lower | |
| if self.maxlower is None or lower > self.maxlower: self.maxlower = lower | |
| if self.minspecial is None or special < self.minspecial: self.minspecial = special | |
| if self.maxspecial is None or special > self.maxspecial: self.maxspecial = special | |
| self.stats_length[pass_length] = self.stats_length.get(pass_length, 0) + 1 | |
| self.stats_charactersets[characterset] = self.stats_charactersets.get(characterset, 0) + 1 | |
| self.stats_simplemasks[simplemask] = self.stats_simplemasks.get(simplemask, 0) + 1 | |
| self.stats_advancedmasks[advancedmask] = self.stats_advancedmasks.get(advancedmask, 0) + 1 | |
| finally: | |
| if filename != '-': | |
| f.close() | |
| if self.progress: | |
| elapsed = time.time() - start_time | |
| sys.stderr.write("\r[*] Processed %d passwords in %.1fs%s\n" % | |
| (self.total_counter, elapsed, " " * 20)) | |
| sys.stderr.flush() | |
| def build_summary(self): | |
| """Full, unfiltered stats as a plain dict -- used for both file export | |
| and as the source of truth for console rendering.""" | |
| if self.total_counter == 0 or self.filter_counter == 0: | |
| return None | |
| length_stats = weighted_length_stats(self.stats_length, self.filter_counter) | |
| return { | |
| 'total_passwords': self.total_counter, | |
| 'analyzed_passwords': self.filter_counter, | |
| 'coverage_percent': percentage(self.filter_counter, self.total_counter), | |
| 'length': { | |
| 'min': length_stats['min'], | |
| 'max': length_stats['max'], | |
| 'avg': round(length_stats['mean'], 2), | |
| 'median': length_stats['median'], | |
| 'stddev': round(length_stats['stddev'], 2), | |
| 'distribution': dict(self.stats_length), | |
| }, | |
| 'charset': dict(self.stats_charactersets), | |
| 'simplemask': dict(self.stats_simplemasks), | |
| 'advancedmask': dict(self.stats_advancedmasks), | |
| 'complexity': { | |
| 'digit': {'min': self.mindigit, 'max': self.maxdigit}, | |
| 'lower': {'min': self.minlower, 'max': self.maxlower}, | |
| 'upper': {'min': self.minupper, 'max': self.maxupper}, | |
| 'special': {'min': self.minspecial, 'max': self.maxspecial}, | |
| }, | |
| } | |
| def _rows_for_console(self, dist, total): | |
| """Sort a distribution dict by frequency and apply --hiderare/--top. | |
| Console-only; exported files always get the full dist via build_summary().""" | |
| items = sorted(dist.items(), key=operator.itemgetter(1), reverse=True) | |
| if self.hiderare: | |
| items = [(k, c) for (k, c) in items if percentage(c, total) >= 1] | |
| if self.top: | |
| items = items[:self.top] | |
| return items | |
| def print_console(self): | |
| """ Print password statistics to the console. """ | |
| if self.total_counter == 0: | |
| print("[!] No passwords were read from the input -- nothing to analyze.") | |
| return | |
| print("[+] Analyzing %d%% (%d/%d) of passwords" % | |
| (percentage(self.filter_counter, self.total_counter), self.filter_counter, self.total_counter)) | |
| print(" NOTE: Statistics below are relative to the number of analyzed passwords, not the total") | |
| if self.filter_counter == 0: | |
| print("[!] No passwords matched the given filters.") | |
| return | |
| summary = self.build_summary() | |
| L = summary['length'] | |
| print("\n[*] Length: min(%d) avg(%.2f) median(%d) stddev(%.2f) max(%d)" % | |
| (L['min'], L['avg'], L['median'], L['stddev'], L['max'])) | |
| for (length, count) in self._rows_for_console(self.stats_length, self.filter_counter): | |
| print("[+] %25d: %02d%% (%d)" % (length, percentage(count, self.filter_counter), count)) | |
| print("\n[*] Character-set:") | |
| for (charset, count) in self._rows_for_console(self.stats_charactersets, self.filter_counter): | |
| print("[+] %25s: %02d%% (%d)" % (charset, percentage(count, self.filter_counter), count)) | |
| print("\n[*] Password complexity:") | |
| print("[+] digit: min(%s) max(%s)" % (self.mindigit, self.maxdigit)) | |
| print("[+] lower: min(%s) max(%s)" % (self.minlower, self.maxlower)) | |
| print("[+] upper: min(%s) max(%s)" % (self.minupper, self.maxupper)) | |
| print("[+] special: min(%s) max(%s)" % (self.minspecial, self.maxspecial)) | |
| print("\n[*] Simple Masks:") | |
| for (mask, count) in self._rows_for_console(self.stats_simplemasks, self.filter_counter): | |
| print("[+] %25s: %02d%% (%d)" % (mask, percentage(count, self.filter_counter), count)) | |
| print("\n[*] Advanced Masks:") | |
| for (mask, count) in self._rows_for_console(self.stats_advancedmasks, self.filter_counter): | |
| print("[+] %25s: %02d%% (%d)" % (mask, percentage(count, self.filter_counter), count)) | |
| def export(self, fh, fmt): | |
| """Write the FULL, unfiltered stat set to fh as 'csv' or 'json'.""" | |
| summary = self.build_summary() | |
| if summary is None: | |
| sys.stderr.write("[!] Nothing to export (no analyzed passwords).\n") | |
| return | |
| if fmt == 'json': | |
| json.dump(summary, fh, indent=2, sort_keys=True) | |
| fh.write("\n") | |
| return | |
| # CSV: flat (section, key, value) table -- covers both distributions | |
| # and scalar summary stats in one importable file. | |
| writer = csv.writer(fh) | |
| writer.writerow(['section', 'key', 'value']) | |
| writer.writerow(['summary', 'total_passwords', summary['total_passwords']]) | |
| writer.writerow(['summary', 'analyzed_passwords', summary['analyzed_passwords']]) | |
| writer.writerow(['summary', 'coverage_percent', summary['coverage_percent']]) | |
| for stat in ('min', 'max', 'avg', 'median', 'stddev'): | |
| writer.writerow(['length', stat, summary['length'][stat]]) | |
| for length, count in sorted(summary['length']['distribution'].items()): | |
| writer.writerow(['length_distribution', length, count]) | |
| for charset, count in sorted(summary['charset'].items(), key=operator.itemgetter(1), reverse=True): | |
| writer.writerow(['charset', charset, count]) | |
| for mask, count in sorted(summary['simplemask'].items(), key=operator.itemgetter(1), reverse=True): | |
| writer.writerow(['simplemask', mask, count]) | |
| for mask, count in sorted(summary['advancedmask'].items(), key=operator.itemgetter(1), reverse=True): | |
| writer.writerow(['advancedmask', mask, count]) | |
| for field in ('digit', 'lower', 'upper', 'special'): | |
| writer.writerow(['complexity', '%s_min' % field, summary['complexity'][field]['min']]) | |
| writer.writerow(['complexity', '%s_max' % field, summary['complexity'][field]['max']]) | |
| HEADER = r""" _ | |
| StatsGen {ver} | | | |
| _ __ __ _ ___| | / | |
| | '_ \ / _` |/ __| |/ / | |
| | |_) | (_| | (__| < | |
| | .__/ \__,_|\___|_|\_\ | |
| | | | |
| |_| iphelix@thesprawl.org | |
| """ | |
| def build_arg_parser(): | |
| parser = argparse.ArgumentParser( | |
| prog="statsgen.py", | |
| description="StatsGen - Password Statistical Analysis tool (part of PACK)", | |
| ) | |
| parser.add_argument('passwords_file', metavar='passwords.txt', | |
| help="Password file to analyze, or '-' to read from stdin") | |
| parser.add_argument('--version', action='version', version='%(prog)s ' + VERSION) | |
| filters = parser.add_argument_group('Password Filters') | |
| filters.add_argument('--minlength', type=int, metavar='N', help="Minimum password length") | |
| filters.add_argument('--maxlength', type=int, metavar='N', help="Maximum password length") | |
| filters.add_argument('--charset', dest='charsets', metavar='loweralpha,numeric', | |
| help="Password charset filter (comma separated)") | |
| filters.add_argument('--simplemask', dest='simplemasks', metavar='stringdigit,allspecial', | |
| help="Password mask filter (comma separated)") | |
| display = parser.add_argument_group('Display Options') | |
| display.add_argument('--hiderare', action='store_true', default=False, | |
| help="Hide console stats covering less than 1%% of the sample " | |
| "(applies to all sections; exported files are unaffected)") | |
| display.add_argument('--top', type=int, metavar='N', | |
| help="Only show the top N entries per console section " | |
| "(exported files are unaffected)") | |
| display.add_argument('-q', '--quiet', action='store_true', default=False, | |
| help="Don't show the banner") | |
| behavior = parser.add_argument_group('Input/Output Options') | |
| behavior.add_argument('-o', '--output', dest='output_file', metavar='stats.json', | |
| help="Export the full stat set (length, charset, simplemask, " | |
| "advancedmask, complexity) to a file") | |
| behavior.add_argument('--output-format', choices=['csv', 'json'], | |
| help="Export format. Default: inferred from -o's extension, " | |
| "falling back to csv") | |
| behavior.add_argument('--encoding', default='utf-8', | |
| help="Input file encoding (default: utf-8). Invalid bytes are " | |
| "preserved rather than crashing the run.") | |
| behavior.add_argument('--ascii-only', action='store_true', default=False, | |
| help="Classify characters using strict ASCII rules only " | |
| "(legacy 0.0.3 behavior). Default is unicode-aware.") | |
| behavior.add_argument('--progress', action='store_true', default=False, | |
| help="Print progress to stderr while processing") | |
| behavior.add_argument('--progress-interval', type=int, default=100000, metavar='N', | |
| help="Passwords between progress updates (default: 100000)") | |
| return parser | |
| def main(): | |
| parser = build_arg_parser() | |
| args = parser.parse_args() | |
| if not args.quiet: | |
| print(HEADER.format(ver=VERSION)) | |
| print("[*] Analyzing passwords in [%s]" % args.passwords_file) | |
| statsgen = StatsGen() | |
| statsgen.minlength = args.minlength | |
| statsgen.maxlength = args.maxlength | |
| statsgen.encoding = args.encoding | |
| statsgen.ascii_only = args.ascii_only | |
| statsgen.hiderare = args.hiderare | |
| statsgen.top = args.top | |
| statsgen.progress = args.progress | |
| statsgen.progress_interval = args.progress_interval | |
| if args.charsets is not None: | |
| statsgen.charsets = [x.strip() for x in args.charsets.split(',')] | |
| unknown = set(statsgen.charsets) - VALID_CHARSETS | |
| if unknown: | |
| sys.stderr.write("[!] Warning: unrecognized --charset value(s): %s\n" % ", ".join(sorted(unknown))) | |
| sys.stderr.write(" Valid values: %s\n" % ", ".join(sorted(VALID_CHARSETS))) | |
| if args.simplemasks is not None: | |
| statsgen.simplemasks = [x.strip() for x in args.simplemasks.split(',')] | |
| unknown = [m for m in statsgen.simplemasks if m != 'othermask' and not SIMPLEMASK_RE.match(m)] | |
| if unknown: | |
| sys.stderr.write("[!] Warning: unrecognized --simplemask value(s): %s\n" % ", ".join(unknown)) | |
| sys.stderr.write(" Expected 1-3 of 'string'/'digit'/'special' concatenated, or 'othermask'\n") | |
| output_fmt = args.output_format | |
| if output_fmt is None and args.output_file: | |
| output_fmt = 'json' if args.output_file.lower().endswith('.json') else 'csv' | |
| try: | |
| statsgen.generate_stats(args.passwords_file) | |
| except OSError as e: | |
| sys.stderr.write("[!] Could not read [%s]: %s\n" % (args.passwords_file, e)) | |
| sys.exit(1) | |
| statsgen.print_console() | |
| if args.output_file: | |
| try: | |
| with open(args.output_file, 'w', newline='' if output_fmt == 'csv' else None) as fh: | |
| statsgen.export(fh, output_fmt) | |
| print("\n[*] Exported full stats (%s) to [%s]" % (output_fmt, args.output_file)) | |
| except OSError as e: | |
| sys.stderr.write("[!] Could not write [%s]: %s\n" % (args.output_file, e)) | |
| sys.exit(1) | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment