build_column_shard.go

package main

import (
	"fmt"
	"github.com/golang/protobuf/proto"
	pb "github.com/taskgraph/taskgraph/example/bwmf/proto"
	"io/ioutil"
	"os"
	"strconv"
)

const nRow = 2000
const nColumn = 1000
const nTask = 4

func main() {
	in, _ := os.Open(os.Args[1])
	mat := make([][]float32, nRow)
	for i := 0; i < nRow; i++ {
		mat[i] = make([]float32, nColumn)
		for j := 0; j < nColumn; j++ {
			var z float32
			fmt.Fscanf(in, "%f", &z)
			mat[i][j] = z
		}
	}

	for iter := 0; iter < nTask; iter++ {
		v := &pb.SparseMatrixShard{Row: make([]*pb.SparseMatrixShard_SparseRow, nRow)}
		for i := 0; i < nRow; i++ {
			m := nColumn / nTask
			fmt.Println(m)
			v.Row[i] = &pb.SparseMatrixShard_SparseRow{At: make(map[int32]float32)}
			for j := 0; j < m; j++ {
				z := mat[i][iter*m + j]
				if z == 0 {
					continue
				}
				v.Row[i].At[int32(j)] = z
			}
		}

		buf, err := proto.Marshal(v)
		if err != nil {
			fmt.Println("marshaling error: ", err)
		}
		ioutil.WriteFile("20ng_column_shard.dat."+strconv.Itoa(iter), buf, 0644)
	}
}

build_column_transpose_shard.go

package main

import (
	"fmt"
	"github.com/golang/protobuf/proto"
	pb "github.com/taskgraph/taskgraph/example/bwmf/proto"
	"io/ioutil"
	"os"
	"strconv"
)

const nRow = 2000
const nColumn = 1000
const nTask = 4

func main() {
	in, _ := os.Open(os.Args[1])
	mat := make([][]float32, nRow)
	for i := 0; i < nRow; i++ {
		mat[i] = make([]float32, nColumn)
		for j := 0; j < nColumn; j++ {
			var z float32
			fmt.Fscanf(in, "%f", &z)
			mat[i][j] = z
		}
	}

	for iter := 0; iter < nTask; iter++ {
		m := nColumn / nTask
		v := &pb.SparseMatrixShard{Row: make([]*pb.SparseMatrixShard_SparseRow, m)}
		for c := 0; c < m; c++ {
			v.Row[c] = &pb.SparseMatrixShard_SparseRow{At: make(map[int32]float32)}
			for r := 0; r < nRow; r++ {
				z := mat[r][iter*m+c]
				if z == 0 {
					continue
				}
				v.Row[c].At[int32(r)] = z
			}
		}

		buf, err := proto.Marshal(v)
		if err != nil {
			fmt.Println("marshaling error: ", err)
		}
		ioutil.WriteFile("20ng_column_shard.t.dat."+strconv.Itoa(iter), buf, 0644)
	}
}

build_row_shard.go

package main

import (
	"fmt"
	"os"
	//	"testing"
	"github.com/golang/protobuf/proto"
	pb "github.com/taskgraph/taskgraph/example/bwmf/proto"
	"io/ioutil"
	"strconv"
)

const nRow = 2000
const nColumn = 1000
const nTask = 4

func main() {
	in, _ := os.Open(os.Args[1])

	for iter := 0; iter < nTask; iter++ {
		n := nRow/nTask
		v := &pb.SparseMatrixShard{Row: make([]*pb.SparseMatrixShard_SparseRow, n)}
		for i := 0; i < n; i++ {
			v.Row[i] = &pb.SparseMatrixShard_SparseRow{At: make(map[int32]float32)}
			for j := 0; j < nColumn; j++ {
				var z float32
				fmt.Fscanf(in, "%f", &z)
				if z == 0 {
					continue
				}
				v.Row[i].At[int32(j)] = z
			}
		}

		buf, err := proto.Marshal(v)
		if err != nil {
			fmt.Println("marshaling error: ", err)
		}
		ioutil.WriteFile("20ng_row_shard.dat."+strconv.Itoa(iter), buf, 0644)
	}
}

combine_column_shard.go

package main

import (
	"fmt"
	"github.com/golang/protobuf/proto"
	pb "github.com/taskgraph/taskgraph/example/bwmf/proto"
	"io/ioutil"
	"os"
	"strconv"
)

const nColumn = 1000
const nTask = 1
const nTopic = 10

func main() {
	for t := 0; t < nTask; t++ {
		in, _ := os.Open(os.Args[1] + "." + strconv.Itoa(t))

		buf, _ := ioutil.ReadAll(in)

		message := &pb.DenseMatrixShard{}
		unmarshErr := proto.Unmarshal(buf, message)
		if unmarshErr != nil {
			panic(unmarshErr)
		}
		for r := 0; r < nColumn / nTask; r++ {
			for c := 0; c < nTopic; c++ {
				fmt.Printf("%f ", message.Row[r].At[int32(c)])
			}
			fmt.Println()
		}
	}
}