Preprocess a set of documents, one on each line of the input file, for LDA inference.

preprocess.go

// For each line of the input file, remove nonalphanumeric characters, 
// lowercase all letters, remove stopwords, and write the result to the output
// file. I used the answer here as a template for reading/writing files: 
// http://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file/9739903#9739903

package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
)

func main() {
	if len(os.Args) < 4 {
		fmt.Println("Too few arguments. Usage: preprocess IN_FILE STOPWORD_FILE OUT_FILE")
		return
	}

	// open input file
	infile, err := os.Open(os.Args[1])
	if err != nil {
		panic(err)
	}
	// close infile on exit and check for its returned error
	defer func() {
		if infile.Close() != nil {
			panic(err)
		}
	}()
	// make a read buffer
	reader := bufio.NewReader(infile)

	// build stopword set
	stopwordfile, err := os.Open(os.Args[2])
	if err != nil {
		panic(err)
	}
	stopwordreader := bufio.NewReader(stopwordfile)
	stopwords := make(map[string]bool)
	for {
		line, err := stopwordreader.ReadString('\n')
		if err != nil && err != io.EOF {
			panic(err)
		}
		word := strings.TrimSpace(line)
		stopwords[word] = true
		if err == io.EOF {
			break
		}
	}

	// open output file
	outfile, err := os.Create(os.Args[3])
	if err != nil {
		panic(err)
	}
	// close outfile on exit and check for its returned error
	defer func() {
		if outfile.Close() != nil {
			panic(err)
		}
	}()
	// make a write buffer
	writer := bufio.NewWriter(outfile)

	// remove nonalphanumeric characters, lowercase,
	// and remove stopwords for each line
	for {
		line, r_err := reader.ReadString('\n')
		if r_err != nil && r_err != io.EOF {
			panic(err)
		}

		nonalphanumeric, err := regexp.Compile(`\W`)
		if err != nil {
			panic(err)
		}

		alphanumeric := nonalphanumeric.ReplaceAllString(line, " ")
		lowercase := strings.ToLower(alphanumeric)
		tokens := strings.Fields(lowercase)

		filtered := []string{}
		for _, word := range tokens {
			if !stopwords[word] {
				filtered = append(filtered, word)
			}
		}

		if len(filtered) > 0 {
			csv := strings.Join(filtered, ",")

			// write a line
			if _, err := writer.WriteString(csv + "\n"); err != nil {
				panic(err)
			}
		}

		if r_err == io.EOF {
			break
		}
	}

	if err = writer.Flush(); err != nil {
		panic(err)
	}
}