build_column_shard.go

package main

import (
	"fmt"
	"github.com/golang/protobuf/proto"
	pb "github.com/taskgraph/taskgraph/example/bwmf/proto"
	"io/ioutil"
	"os"
	"strconv"
)

const nRow = 2000
const nColumn = 1000
const nTask = 4

func main() {
	in, _ := os.Open(os.Args[1])
	mat := make([][]float32, nRow)
	for i := 0; i < nRow; i++ {
		mat[i] = make([]float32, nColumn)
		for j := 0; j < nColumn; j++ {
			var z float32
			fmt.Fscanf(in, "%f", &z)
			mat[i][j] = z
		}
	}

	for iter := 0; iter < nTask; iter++ {
		v := &pb.SparseMatrixShard{Row: make([]*pb.SparseMatrixShard_SparseRow, nRow)}
		for i := 0; i < nRow; i++ {
			m := nColumn / nTask
			v.Row[i] = &pb.SparseMatrixShard_SparseRow{At: make(map[int32]float32)}
			for j := 0; j < m; j++ {
				z := mat[i][iter*m + j]
				if z == 0 {
					continue
				}
				v.Row[i].At[int32(j)] = z
			}
		}

		buf, err := proto.Marshal(v)
		if err != nil {
			fmt.Println("marshaling error: ", err)
		}
		ioutil.WriteFile("20ng_column_shard.dat."+strconv.Itoa(iter), buf, 0644)
	}
}

build_row_shard.go

package main

import (
	"fmt"
	"os"
	//	"testing"
	"github.com/golang/protobuf/proto"
	pb "github.com/taskgraph/taskgraph/example/bwmf/proto"
	"io/ioutil"
	"strconv"
)

const nRow = 2000
const nColumn = 1000
const nTask = 4

func main() {
	in, _ := os.Open(os.Args[1])

	for iter := 0; iter < nTask; iter++ {
		n := nRow/nTask
		v := &pb.SparseMatrixShard{Row: make([]*pb.SparseMatrixShard_SparseRow, n)}
		for i := 0; i < n; i++ {
			v.Row[i] = &pb.SparseMatrixShard_SparseRow{At: make(map[int32]float32)}
			for j := 0; j < nColumn; j++ {
				var z float32
				fmt.Fscanf(in, "%f", &z)
				if z == 0 {
					continue
				}
				v.Row[i].At[int32(j)] = z
			}
		}

		buf, err := proto.Marshal(v)
		if err != nil {
			fmt.Println("marshaling error: ", err)
		}
		ioutil.WriteFile("20ng_row_shard.dat."+strconv.Itoa(iter), buf, 0644)
	}
}