Go Quick Wordlist Process

go-quick-wordlist-process.go

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"sync"
)

// go run file.go -f wordlist.txt -t 50
// ************************** MAIN RUNNER **************************** //

func run(i int, str string) {
	fmt.Printf("i - %d, str - %s\n", i, str)
	// data, err := getURLStr(str, "")
}

func main() {

	var file string
	var threads int

	flag.StringVar(&file, "f", "", "(required) wordlist file")
	flag.IntVar(&threads, "t", 20, "threads")
	flag.Parse()

	if !fileExists(file) {
		flag.Usage()
		return
	}

	arr, count := fileWordList(file)

	fmt.Printf("File 		- %s\n", absPath(file))
	fmt.Printf("Total Word	- %s\n", colGreen(count))
	fmt.Printf("Thread 		- %s\n", colRed(threads))

	var wg sync.WaitGroup
	var c = make(chan int, threads)

	for i, str := range arr {
		wg.Add(1)
		go func(i int, str string) {
			c <- i
			run(i, str)
			<-c
			wg.Done()
		}(i, str)
	}

	wg.Wait()
	close(c)
}

// ************************** helpers **************************** //
func fileExists(filename string) bool {

	info, err := os.Stat(filename)

	if os.IsNotExist(err) {
		return false
	}

	return !info.IsDir()
}

func fileWordList(path string) ([]string, int) {

	var lines []string
	var count = 0

	file, err := os.Open(path)

	if err != nil {
		return nil, 0
	}

	scanner := bufio.NewScanner(file)

	for scanner.Scan() {
		count++
		lines = append(lines, scanner.Text())
	}

	file.Close()

	return lines, count
}

var (
	userAgentCrawler = "Crawler"
	colBlack         = xColor("\033[1;30m%s\033[0m")
	colRed           = xColor("\033[1;31m%s\033[0m")
	colGreen         = xColor("\033[1;32m%s\033[0m")
	colYellow        = xColor("\033[1;33m%s\033[0m")
	colPurple        = xColor("\033[1;34m%s\033[0m")
	colMagenta       = xColor("\033[1;35m%s\033[0m")
	colTeal          = xColor("\033[1;36m%s\033[0m")
	colWhite         = xColor("\033[1;37m%s\033[0m")
)

// Color get a new color from ansi code
func xColor(colorString string) func(...interface{}) string {
	sprint := func(args ...interface{}) string {
		return fmt.Sprintf(colorString, fmt.Sprint(args...))
	}
	return sprint
}

func absPath(path string) string {
	path, err := filepath.Abs(path)
	if err != nil {
		fmt.Printf("Error getting absPath of %s\n", colRed(path))
	}
	return path
}

// StrToFile write string to a file
func strToFile(outFilepath, str string) error {
	f, err := os.Create(outFilepath)
	if err != nil {
		return err
	}
	_, err = f.WriteString(str)
	f.Close()
	return err
}

// BytesToFile write byte to a file
func bytesToFile(outFilepath string, bytes []byte) error {
	f, err := os.Create(outFilepath)
	if err != nil {
		return err
	}
	_, err = f.Write(bytes)
	f.Close()
	return err
}

// URLContentMust return []bytes
// panics if failed
func getURLContentMust(urlStr string, userAgent string) []byte {

	htmlBytes, err := getURLContent(urlStr, userAgent)

	if err != nil {
		panic("[URLContentMust] Error getting data - " + err.Error())
	}

	return htmlBytes
}

func isURLValid(toTest string) bool {

	_, err := url.ParseRequestURI(toTest)

	if err != nil {
		return false
	}

	u, err := url.Parse(toTest)

	if err != nil || u.Scheme == "" || u.Host == "" {
		return false
	}

	return true
}

// getURLStr return string of a url
// panics if failed
func getURLStr(urlStr string, userAgent string) (string, error) {
	htmlBytes, err := getURLContent(urlStr, userAgent)
	return string(htmlBytes), err
}

// getURLStrMust return string of a url
// panics if failed
func getURLStrMust(urlStr string, userAgent string) string {

	htmlBytes, err := getURLContent(urlStr, userAgent)

	if err != nil {
		panic("[URLStrMust] Error getting data - " + err.Error())
	}

	return string(htmlBytes)
}

func getURLContent(urlStr string, userAgent string) ([]byte, error) {

	// Make request
	response, err := getURLResponse(urlStr, userAgent)

	if err != nil {
		return nil, err
	}

	htmlBytes, err := ioutil.ReadAll(response.Body)

	if err != nil {
		return nil, err
	}

	response.Body.Close()

	return htmlBytes, nil
}

// getURLResponse get full response of a url
// make sure to call `defer response.Body.Close()` in your caller function
func getURLResponse(urlStr string, userAgent string) (*http.Response, error) {

	if len(userAgent) == 0 {
		userAgent = userAgentCrawler
	}
	// Create HTTP client with timeout
	client := &http.Client{}

	// Create and modify HTTP request before sending
	request, err := http.NewRequest("GET", urlStr, nil)

	if err != nil {
		return &http.Response{}, err
	}

	// set user agent
	request.Header.Set("User-Agent", userAgent)

	// Make request
	response, err := client.Do(request)

	if err != nil {
		return &http.Response{}, err
	}

	// defer response.Body.Close()
	client.CloseIdleConnections()

	return response, nil
}

// getArrStrUnique returns array with unique values of array from a array of string
func getArrStrUnique(array []string) []string {
	var uniques []string
	for _, v := range array {
		if !getArrStrContains(uniques, v) {
			uniques = append(uniques, v)
		}
	}
	return uniques
}

// getArrStrContains check whether a string contains in a string array
func getArrStrContains(array []string, value string) bool {
	var contains = false
	for _, v := range array {
		if v == value {
			contains = true
			break
		}
	}
	return contains
}

// getArrIntContains check whether a interger contains in a interger array
func getArrIntContains(array []int, value int) bool {
	var contains = false
	for _, a := range array {
		if a == value {
			contains = true
			break
		}
	}
	return contains
}

// getNonCreatedFileName Parameter: "out", "txt", 1
// output: out_1.txt
func getNonCreatedFileName(baseName string, ext string, i int) string {
	if !fileExists(baseName + ext) {
		return baseName + ext
	} else if !fileExists(baseName + "_" + strconv.Itoa(i) + ext) {
		return baseName + "_" + strconv.Itoa(i) + ext
	}
	return getNonCreatedFileName(baseName, ext, i+1)
}

func dirCreateIfNotExists(path string) error {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return os.Mkdir(path, 0755)
	}
	return nil
}

// getFileBytes get []byte of a file
func getFileBytes(filePath string) ([]byte, error) {

	file, err := os.Open(filePath)

	if err != nil {
		return nil, err
	}

	b, err := ioutil.ReadAll(file)

	if err != nil {
		return nil, err
	}

	file.Close()

	return b, nil
}

// getFileBytesMust get []byte of a file
// panics if failed
func getFileBytesMust(filePath string) []byte {
	bytes, err := getFileBytes(filePath)
	if err != nil {
		panic("Error in FileBytesMust, filePath: " + filePath + ", err: " + err.Error())
	}
	return bytes
}

// getFileStr get string content of a file
func getFileStr(path string) (string, error) {
	bytes, err := getFileBytes(path)
	return string(bytes), err
}

// getFileStrMust get string of a file
// panics if failed
func getFileStrMust(filePath string) string {
	bytes, err := getFileBytes(filePath)
	if err != nil {
		panic("[panic] in FileStrMust, filePath: " + filePath + ", err: " + err.Error())
	}
	return string(bytes)
}

func fileRemoveIfExists(path string) error {
	if fileExists(path) {
		return os.Remove(path)
	}
	return nil
}