Gitscan aggregate to TAB separated stream on stdout

gitscanaggregate.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"
)

func main() {
	folders, err := ioutil.ReadDir("./")
	if err != nil {
		log.Fatal(err)
	}

	allTeams := uint32(0)
	totalLeaks := uint32(0)
	totalUnique := uint32(0)

	for _, teamName := range folders {
		if teamName.IsDir() {
			allTeams += 1
			totalTeamLeaks := uint32(0)
			totalTeamUnique := uint32(0)
			files, listerr := ioutil.ReadDir("./" + teamName.Name())
			if listerr != nil {
				log.Fatal(err)
			}

			for _, reportFile := range files {

				filename := teamName.Name() + "/" + reportFile.Name()
				filePath := "./" + filename

				content, err := ioutil.ReadFile(filePath)
				if err != nil {
					log.Fatal(err)
				}

				strcontent := string(content)

				if len(strcontent) > 0 {
					switch filepath.Ext(reportFile.Name()) {
					case ".txt":
						unique := make(map[string]uint32)
						lines := strings.Split(strcontent, "\n")
						for _, line := range lines {
							parts := strings.Split(line, ":")
							line = strings.Join(parts[1:], ":")
							unique[line]++
						}
						// because a few of the secrets are made with custom grep, they store to different files
						// lets unwrap and simply remember the type of secret that is leaked
						source := ""
						if strings.Contains(reportFile.Name(), "artifactory") {
							source = "artifactory"
						} else {
							source = "client_secret"
						}
						repoName := strings.ReplaceAll(reportFile.Name(), "_artifactory.txt", "")
						repoName = strings.ReplaceAll(repoName, "_client_secret.txt", "")
						repoName = strings.ReplaceAll(repoName, ".gitty", "")
						for k, v := range unique {
							if len(k) > 0 {
								fmt.Printf("%s\t%s\t%s\t%s\t%s\t%d\n", teamName.Name(), repoName, source, reportFile.Name(), k, v)
								totalTeamLeaks += v
								totalTeamUnique += 1
							}
						}
						break
					case ".json":
						var jsonData []map[string]interface{}
						json.Unmarshal([]byte(strcontent), &jsonData)
						unique := make(map[string]uint32)
						for item := range jsonData {
							line := fmt.Sprintf("%s\t%s\t%s\t%s\t%s", teamName.Name(), jsonData[item]["repo"], jsonData[item]["rule"], reportFile.Name(), jsonData[item]["offender"])
							unique[line]++
						}
						for k, v := range unique {
							if len(k) > 0 {
								fmt.Printf("%s\t%d\n", k, v)
								totalTeamLeaks += v
								totalTeamUnique += 1
							}
						}
						break
					case ".gitty":
						unique := make(map[string]uint32)
						lines := strings.Split(strcontent, "\n")
						for _, line := range lines {
							if line != "No matches." {
								unique[line]++
							}
						}

						repoName := strings.ReplaceAll(reportFile.Name(), "_artifactory.txt", "")
						repoName = strings.ReplaceAll(repoName, "_client_secret.txt", "")
						repoName = strings.ReplaceAll(repoName, ".gitty", "")
						for k, v := range unique {
							if len(k) > 0 {
								fmt.Printf("%s\t%s\t%s\t%s\t%s\t%d\n", teamName.Name(), repoName, "gitty", reportFile.Name(), k, v)
								totalTeamLeaks += v
								totalTeamUnique += 1
							}
						}
						break
					}
				}
			}
			//fmt.Printf("TEAM: %s\tTOTAL: %d\tUNIQUE: %d\n", teamName.Name(), totalTeamLeaks, totalTeamUnique)
			totalLeaks += totalTeamLeaks
			totalUnique += totalTeamUnique
		}
	}

	//fmt.Printf("ALL %d TEAMS\tTOTAL: %d\tUNIQUE: %d\n", allTeams, totalLeaks, totalUnique)
}