leak-filter-bridge

#!/usr/bin/env node

const fs = require('fs');
const path = require('path');
const child_process = require('child_process');


const defaults = {
    outputPath: 'terms',
    sourceRepoPath: '.',
    descriptionSlugRegex: /(?:Uncovered a |Identified a |Detected a )(.*)(?:,.*)/,
}

const usage = `
usage: leak-filter-bridge <leaks-report-path> [--output-path <output-dir>] [--source-repo-path <repo-dir>] [--description-slug-regex <slug-capture-regex>] [--group-by-rule] [--multiline] [--help|-h] 
`

const longUsage = `${usage} 

creates git-filter-repo replacement terms files from a gitleaks report.


arguments
=========

leaks-report-path
  the location of the gitleaks report file.


options
=======

--output-path
  a location for the generated replacement-terms files.
  defaults to "${defaults.outputPath}".

--source-repo-path
  the location of the git repository used for generating the gitleaks report file.
  used for extracting the original encoded secrets from the source-code.
  only required if the report contains decoded leaks (due to using gitleaks' --max-decode-depth).
  defaults to "${defaults.sourceRepoPath}".

--description-slug-regex
  the regex used to capture an excerpt from gitleaks report items' description.
  the description slug is used for naming the generated replacement-terms files. 
  defaults to ${defaults.descriptionSlugRegex}.
  

flags
=====

--multiline
  represent line-breaks in secrets as escaped line-breaks in terms files.
  see https://www.mankier.com/1/git-filter-repo#Examples-Content_based_filtering 

--group-by-rule
  generate term files per each gitleaks rule, instead of a single file with all (unique) terms.

--help|-h
  show this help menu and quit.

`

const retrieveSecret = (reportItem) => {
    const {
        opts: {
            ['--source-repo-path']: sourceRepoPath = defaults.sourceRepoPath,
        },
    } = parseArgs()
    const sourceRepoDir = path.resolve(sourceRepoPath)

    if (reportItem.Tags.some(tag => tag.startsWith('decoded:'))) {
        return extractRepoTerm(sourceRepoDir, reportItem)
    }
    return reportItem.Secret
}

const getSecrets = (leaksReport, ruleId) => (
    Array.from(new Set(
        leaksReport
            .filter(reportItem => (
                !ruleId || ruleId === reportItem.RuleID
            ))
            .map(reportItem => retrieveSecret(reportItem))
    ))
        .sort()
)

const createRuleMap = (leaksReport) => (
    leaksReport
        .reduce((accum, reportItem) => ({
            ...accum,
            [reportItem.RuleID]: descriptionAsSlug(reportItem),
        }), {})
)

const createSecretsMap = (leaksReport) => {
    const ruleSlugById = createRuleMap(leaksReport)

    return Object.entries(ruleSlugById).reduce((accum, [ruleId, ruleSlug]) => ({
        ...accum,
        [`${ruleId}--${ruleSlug}`]: getSecrets(leaksReport, ruleId),
    }), {})
}

const createSecretsCollection = (leaksReport) => (
    [].concat(...Object.values(getSecrets(leaksReport)))
)

const createTermFiles = () => {
    const {
        args: [ leaksReportPath ],
        opts: {
            ['--output-path']: outputPath = defaults.outputPath,
        },
        flags,
    } = parseArgs()

    const leaksReport = require(path.resolve(leaksReportPath));
    const outputDir = path.resolve(outputPath)
    const ruleGrouping = flags.includes('--group-by-rule');

    ensureDir(outputDir)

    if (ruleGrouping) {
        const secretsByRule = createSecretsMap(leaksReport)
        console.log(`found ${sumValues(secretsByRule)} terms overall`)
        Object.entries(secretsByRule).forEach(([rule, secrets]) => {
            console.log(`writing ${secrets.length} terms to file "${rule}.txt"`)
            writeFile(
                path.resolve(outputDir, `${rule}.txt`),
                asTerms(secrets),
            )
        })
    } else {
        const allSecrets = createSecretsCollection(leaksReport)
        console.log(`found ${allSecrets.length} unique terms overall`)
        console.log('writing all terms to file "all.txt"')
        writeFile(
            path.resolve(outputDir, `all.txt`),
            asTerms(allSecrets),
        )
    }
}

const asTerms = (secrets) => {
    const { flags } = parseArgs()
    const multilineTerms = flags.includes('--multiline');

    const escapeMultilineSecret = secret => (
        secret.includes('\n') ? `regex:(?m)${(escapeString(secret))}` : secret
    )

    return (multilineTerms ? secrets.map(escapeMultilineSecret) : secrets)
        .join('\n')
}

const descriptionAsSlug = (leaksReportItem) => {
    const {
        opts: {
            ['--description-slug-regex']: descriptionSlugRegex = defaults.descriptionSlugRegex,
        },
    } = parseArgs()

    return String(
        leaksReportItem.Description
            .split(new RegExp(descriptionSlugRegex))
            .filter(Boolean)[0]
    )
        .replaceAll(' ', '-')
        .toLowerCase()
}

const escapeString = (str) => (
    // stringify() escapes any backslashed sequences,
    // replace() gets rid of the wrapping quotes by-product 
    JSON.stringify(str).replace(/^"|"$/g, '')
)

const sumValues = (obj) => (
    Object.values(obj).reduce((accum, item) => (accum += item.length), 0)
)

const writeFile = (path, contents) => {
    fs.writeFile(
        path,
        contents,
        (err) => err && console.error(err),
    )
}

const ensureDir = (path) => {
    if (!fs.existsSync(path)) {
        fs.mkdirSync(path, { recursive: true });
    }
}

const extractRepoTerm = (repoPath, {
    Commit: commitHash,
    File: filePath,
    StartLine: startLine,
    StartColumn: startColumn,
    EndLine: endLine,
    EndColumn: endColumn,
}) => {
    const fileContent = child_process.execSync(
        `git -C ${repoPath} show ${commitHash}:${filePath}`
    )?.toString()

    const liStart = startLine - 1
    const liEnd = endLine
    const colStart = startColumn - 2
    const colEnd = endColumn - 1

    let lines = fileContent.split('\n')

    // scope lines
    lines = lines.slice(liStart, liEnd)

    // adjust column boundries
    lines[0] = lines[0].slice(colStart)
    lines[lines.length - 1] = lines[lines.length - 1].slice(0, lines.length > 1 ? colEnd : colEnd - colStart)

    return lines.join('')
}

const parseArgs = (args = process.argv.slice(2)) => (
    args.reduce((accum, arg, i, arr) => {
        if (arg.startsWith('-')) {
            const nextArg = arr[i + 1];
            if (nextArg?.length && !nextArg.startsWith('-')) {
                accum.opts[arg] = nextArg
            } else if (!nextArg?.length || nextArg.startsWith('-')) {
                accum.flags.push(arg)
            }
        } else if (!arr[i - 1]?.startsWith('-')) {
            accum.args.push(arg)
        }
        return accum
    }, { args: [], opts: {}, flags: [] })
)

const main = () => {
    const {
        args: [ leaksReportPath ],
        flags,
    } = parseArgs()

    const showHelp = flags.includes('--help') || flags.includes('-h')
    if (showHelp) {
        console.log(longUsage)
        return process.exit(0)
    }

    if (!leaksReportPath) {
        console.log(usage)
        return process.exit(1)
    }

    createTermFiles()
}


main()