Example VOTable simple protobuf

votable.capnp

@0xb6b38e0f8e6996ae;

using Java = import "java.capnp";
$Java.package("org.capnproto.examples");
$Java.outerClassname("VOTable");

enum MetaType {
    info @0;
    field @1;
    param @2;
}

struct Meta {
    type @0 :MetaType;
    description @1 :Text;
    id @2 :Text;
    name @3 :Text;
    datatype @4 :Text;
    value @5 :Text;
    text @6 :Text;
}

enum RefType {
    field @0;
    param @1;
}

struct Ref {
    type @0 :RefType;
    ref @1 :Text;
    ucd @2 :Text;
    utype @3 :Text;
}

struct Group {
    description @0 :Text;
    elements @1 :List(Meta);
    refs @2 :List(Ref);
}

struct Metadata {
    elements @0 :List(Meta);
    groups @1 :List(Group);
}

struct Column {
    text @0 :Text;
#    2: i64 int64,
#    3: binary bin;
}

struct Row {
    columns @0 :List(Column);
}

struct Table {
    description @0 :Text;
    metadata @1 :Metadata;
    rows @2: List(Row);
    coda @3: List(Meta);
}

struct SimpleVOTable {
    description @0 :Text;
    metadata @1 :Metadata;
    table @2 :Table;
    coda @3 :List(Meta);
}

struct Response {
    metadata @0 :Metadata;
    result @1 :SimpleVOTable;
    coda @2 :List(Meta);
}

votable.proto

package ivoa;

option java_package = "org.lsst.ivoa.votable";
option java_outer_classname = "VOTableProtos";


message Meta { 
    enum Type {
        INFO = 1;
        FIELD = 2;
        PARAM = 3;
    }
    optional Type type = 1 [json_name = "$type"];      // Param, Field, or Info
    optional string description = 2;
    optional string id = 3;
    required string name = 4;
    optional string datatype = 5;
    optional string value = 6;
    optional string text = 7;
}

message Ref {
    enum Type {
        FIELD = 1;
        PARAM = 2;
    }
    optional Type type = 1 [json_name = "$type"];      // Param, Field, or Info
    required string ref = 2;
    optional string ucd = 3;
    optional string utype = 4;
}

message Group {
    optional string description = 1;
    repeated Meta elements = 2;
    repeated Ref refs = 3;
}

message Metadata {
    repeated Meta elements = 1;
    repeated Group groups = 2;
}

message Column {
    optional string value = 1; // Could be union type
    optional bool isnull = 2;
}

message Row {
    repeated Column column = 1 [json_name = "columns"];
}

message Table {
    optional string description = 1;
    optional Metadata metadata = 2;
    repeated Row data = 3 [json_name = "data"];
    repeated Meta coda = 4;
}

message SimpleVOTable {
    optional string description = 1;
    optional Metadata metadata = 2;
    required Table table = 3;
    repeated Meta coda = 4;
}

message Response {
    optional Metadata metadata = 1;
    required SimpleVOTable result = 2;
    repeated Meta coda = 3;
}

message ErrorResponse {
    required string error = 1;
    optional string message = 2;
    repeated Meta coda = 3;
}

votable.thrift

namespace java org.lsst.ivoa.votable.thrift


enum MetaType {
    INFO = 1,
    FIELD = 2,
    PARAM = 3
}

struct Meta {
    1: optional MetaType type,
    2: optional string description,
    3: optional string id,
    4: required string name,
    5: optional string datatype,
    6: optional string value,
    7: optional string text
}

enum RefType {
    FIELD = 1,
    PARAM = 2
}

struct Ref {
    1: optional RefType type,
    2: required string ref,
    3: optional string ucd,
    4: optional string utype
}

struct Group {
    1: optional string description,
    2: list<Meta> elements,
    3: list<Ref> refs
}

struct Metadata {
    1: optional list<Meta> elements = 1;
    2: optional list<Group> groups = 2;
}

union Column {
    1: string text,
    2: i64 int64,
    3: binary bin
}

struct Row {
    1: list<Column> columns
}

struct Table {
    1: optional string description,
    2: optional Metadata metadata,
    3: list<Row> rows,
    4: list<Meta> coda
}

struct SimpleVOTable {
    1: optional string description,
    2: optional Metadata metadata,
    3: required Table table,
    4: list<Meta> coda
}

struct Response {
    1: optional Metadata metadata,
    2: required SimpleVOTable result,
    3: list<Meta> coda
}

VOTableTest.java

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package org.lsst.ivoa.votable;

import com.google.protobuf.InvalidProtocolBufferException;
import java.io.UnsupportedEncodingException;
import org.junit.Test;
import org.lsst.ivoa.votable.VOTableProtos.*;
import com.google.protobuf.util.JsonFormat;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;

/**
 *
 * @author bvan
 */
public class VOTableTest {
    
    public VOTableTest(){
    }
    
    

    @Test
    public void testSomeMethod() throws UnsupportedEncodingException, InvalidProtocolBufferException, IOException{
        Table.Builder tableBuilder = Table.newBuilder();
        String[][] rows = {
            {"hello", "world"},
            {"hi", "mom"}
        };
        
        Metadata.Builder md = Metadata.newBuilder();
        
        
        md.addElements(Meta.newBuilder()
                .setType(Meta.Type.FIELD)
                .setName("1")
                .setDatatype("text")
                .build()
        );
        md.addElements(Meta.newBuilder()
                .setType(Meta.Type.FIELD)
                .setName("2")
                .setDatatype("text")
                .build()
        );
        
        for(String[] row: rows){
            Row.Builder rowBuilder = Row.newBuilder();
            for(String c: row){
                rowBuilder.addColumn(
                        //Column.newBuilder().setValue(ByteString.copyFrom(c, "UTF-8")).build()
                        Column.newBuilder().setValue(c).build()
                );
            }
            tableBuilder.addData(rowBuilder.build());
            Files.newOutputStream(Paths.get("./row.txt")).write(rowBuilder.build().toByteArray());
        }
        tableBuilder.setMetadata(md);
        SimpleVOTable.Builder votBuilder = SimpleVOTable.newBuilder();
        
        votBuilder.setTable(tableBuilder.build());
        
        votBuilder.build();
        
        Response.Builder resp = Response.newBuilder();
        resp.setResult(votBuilder);
        
        
        //votBuilder.build().toByteArray();
        System.out.println(JsonFormat.printer().print(resp.build()));
        
        System.out.println("JSON size:" + JsonFormat.printer().print(resp).getBytes().length);
        System.out.println("PB3 size:" + resp.build().toByteArray().length);
        

        StringBuilder hex = new StringBuilder();
        StringBuilder ascii = new StringBuilder();
        int i = 0;
        for(byte b: resp.build().toByteArray()){
            hex.append(String.format("%02x", b));
            byte[] bytes = new byte[]{b};
            switch (b){
                case 0x0a:
                    ascii.append("\\n");
                    break;
                case 0x0c:
                    ascii.append("\\f");
                    break;
                case 0x0d:
                    ascii.append("\\r");
                    break;
                case 0x09:
                    ascii.append("\\t");
                    break;
                default:
                    ascii.append(" ");
                    ascii.append(new String(bytes, "UTF-8"));
                
            }
        }
        System.out.println(hex.toString());
        System.out.println(ascii.toString());
        Files.newOutputStream(Paths.get("./dat.txt")).write(resp.build().toByteArray());
        Files.newOutputStream(Paths.get("./table.txt")).write(tableBuilder.build().toByteArray());
        
        //System.out.println(votBuilder.toString());
    }
    
}

VOTableThriftTest.java

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package org.lsst.ivoa.votable;



import java.io.UnsupportedEncodingException;
import org.apache.thrift.TException;
import org.apache.thrift.TSerializer;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TSimpleJSONProtocol;
import org.junit.Test;
import org.lsst.ivoa.votable.thrift.Column;
import org.lsst.ivoa.votable.thrift.Field;
import org.lsst.ivoa.votable.thrift.Meta;
import org.lsst.ivoa.votable.thrift.Metadata;
import org.lsst.ivoa.votable.thrift.Response;
import org.lsst.ivoa.votable.thrift.Row;
import org.lsst.ivoa.votable.thrift.SimpleVOTable;
import org.lsst.ivoa.votable.thrift.Table;

/**
 *
 * @author bvan
 */
public class VOTableThriftTest {
    
    public VOTableThriftTest(){
    }
    
    

    @Test
    public void testSomeMethod() throws UnsupportedEncodingException, TException {
        
        String[][] rows = {
            {"hello", "world"},
            {"hi", "mom"}
        };
        
        Metadata md = new Metadata();
        md.addToElements(new Meta().setName("1").setDatatype("text"));
        md.addToElements(new Meta().setName("2").setDatatype("text"));
        
        Table tableBuilder = new Table();
        tableBuilder.setMetadata(md);
        
        for(String[] row: rows){
            Row rowData = new Row();
            for(String c: row){
                Column col = new Column();
                col.setText(c);
                rowData.addToColumns(col);
            }
            tableBuilder.addToRows(rowData);
        }
        
        SimpleVOTable vot = new SimpleVOTable().setTable(tableBuilder);
                
        Response resp = new Response();
        resp.setResult(vot);
        
        TSerializer serializer = new TSerializer(new TSimpleJSONProtocol.Factory());        
        TSerializer bserializer = new TSerializer(new TBinaryProtocol.Factory());
        
        String json = serializer.toString(resp);
        byte[] bytes = bserializer.serialize(resp);
        System.out.println(json);

        System.out.println("JSON size:" + json.getBytes().length);
        System.out.println("thrift size:" + bytes.length);
    }

}

Running org.lsst.ivoa.votable.VOTableTest
{
  "result": {
    "data": {
      "rowSchema": {
        "field": [{
          "name": "1",
          "datatype": "string"
        }, {
          "name": "2",
          "datatype": "string"
        }]
      },
      "row": [{
        "column": [{
          "value": "hello"
        }, {
          "value": "world"
        }]
      }, {
        "column": [{
          "value": "hi"
        }, {
          "value": "mom"
        }]
      }]
    }
  }
}
JSON size:452
PB3 size:67

Modified:

{
  "result": {
    "data": {
      "fields": [{
        "name": "1",
        "datatype": "string"
      }, {
        "name": "2",
        "datatype": "string"
      }],
      "rows": [{
        "columns": [{
          "value": "hello"
        }, {
          "value": "world"
        }]
      }, {
        "columns": [{
          "value": "hi"
        }, {
          "value": "mom"
        }]
      }]
    }
  }
}
JSON size:413 (Actual is ~200)
PB3 size:65

Thrift equivalent output (json)

{
  "result": {
    "data": {
      "fields": [{
          "datatype": "string",
          "name": "1"
        }, {
          "datatype": "string",
          "name": "2"
        }],
      "rows": [{
          "columns": [{
              "utf8": "hello"
            }, {
              "utf8": "world"
            }]
        }, {
          "columns": [{
              "utf8": "hi"
            }, {
              "utf8": "mom"
            }]
        }]
    }
  }
}