logfile consolidation (sqlite)

logconsolidatordb.go

package main

import (
	"bufio"
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const (
	timefmt       = "15:04:05.000"
	dbname        = "./data.tmp"
	filesignature = "*.log"
)


func trace(s string) (string, time.Time) {
	log.Println("START:", s)
	return s, time.Now()
}

func un(s string, startTime time.Time) {
	endTime := time.Now()
	log.Println("  END:", s, "ElapsedTime in seconds:", endTime.Sub(startTime))
}

func main() {

	defer un(trace("SQLITE-GOLANG"))
	
	defer os.Remove(dbname)
	db, err := sql.Open("sqlite3", dbname)
	//db, err := sql.Open("sqlite3", "file::memory:?cache=shared")

	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sqlStmt := `
	create table log (time timestamp not null, counter integer, filename text, line text);
	delete from log;
	`
	_, err = db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
		return
	}

	//init transaction
	tx, err := db.Begin()
	if err != nil {
		log.Fatal(err)
	}
	stmt, err := tx.Prepare("INSERT INTO log(time, counter, filename, line) VALUES (?, ?, ?, ?)")
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()

	//file read
	files, _ := filepath.Glob(filesignature)
	counter := 0
	for _, filename := range files {

		inFile, _ := os.Open(filename)
		defer inFile.Close()

		scanner := bufio.NewScanner(inFile)
		scanner.Split(bufio.ScanLines)

		for scanner.Scan() {
			line := scanner.Text()
			key := line[1:13]
			timeKey := strings.Replace(key, ",", ".", 1)
			timeParsed, err := time.Parse(timefmt, timeKey)
			if err != nil {
				fmt.Println(err)
				return
			}

			_, err = stmt.Exec(timeParsed, counter, filename, line)
			counter++
			if err != nil {
				log.Fatal(err)
			}
		}
	}
	tx.Commit()

	//output
	rows, err := db.Query("SELECT filename || line  line FROM log ORDER BY time, counter")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		var line string
		rows.Scan(&line)
		fmt.Println(line)
	}

}