main.go

package main

import (
	"bufio"
	"encoding/binary"
	"fmt"
	"io"
	"os"
)

const BufferSize = 4096

func iterBufferedStringCol(posFilename, dataFilename string) error {
	posFile, err := os.Open(posFilename)
	if err != nil {
		return err
	}
	defer posFile.Close()

	dataFile, err := os.Open(dataFilename)
	if err != nil {
		return err
	}
	defer dataFile.Close()

	posReader := bufio.NewReaderSize(posFile, BufferSize)
	dataReader := bufio.NewReaderSize(dataFile, BufferSize)

	var pos, prevPos uint32
	var count int

	data := make([]byte, BufferSize)

	for {
		err := binary.Read(posReader, binary.LittleEndian, &pos)
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		size := int(pos - prevPos)

		if size > len(data) {
			data = make([]byte, size)
		}
		_, err = io.ReadFull(dataReader, data[:size])
		if err != nil {
			return err
		}
		prevPos = pos
		count++
	}
	fmt.Printf("Count: %d\n", count)
	return nil
}

func main() {
	err := iterBufferedStringCol(
		"./data/col-raw.pos",
		"./data/col-raw.data",
	)
	if err != nil {
		panic(err)
	}
}