NIX DataFrame Proof of Concept

nix-dataframe.cpp

//NIX DataFrame test
#include <nix.hpp>
#include <cstdlib>
#include <cstring>
#include <iostream>

#include "backend/hdf5/h5x/H5DataType.hpp"
#include "backend/hdf5/h5x/H5Group.hpp"
#include "backend/hdf5/h5x/H5DataSet.hpp"

namespace h5x = nix::hdf5::h5x;
namespace h5 = nix::hdf5;

//API
class DataFrame {
public:
  struct Header {
    std::string   name;
    std::string   unit;
    nix::DataType dtype;
  };

  static void create(const std::vector<Header> &cols);


  static std::vector<nix::Variant> readRow(nix::ndsize_t row);
  static void writeRow(h5::DataSet &ds, const std::vector<nix::Variant> &vals, nix::ndssize_t row = -1);
};

void DataFrame::writeRow(h5::DataSet &ds, const std::vector<nix::Variant> &vals, nix::ndssize_t row) {

  std::vector<size_t> offset(vals.size());
  std::vector<h5x::DataType> dtypes(vals.size());

  size_t s = sizeof(nix::Variant) * vals.size();
  std::cout << s << std::endl;
  char *base = (char *) std::malloc(s);

  s = 0;
  for (size_t i = 0; i < vals.size(); i++) {
    h5x::DataType ft = h5::data_type_to_h5_memtype(vals[i].type());
    char *ptr = base + s;

    dtypes[i] = ft;
    offset[i] = s;
    s += ft.size();

    switch(vals[i].type()) {
    case nix::DataType::Int32: {
      int32_t v;
      vals[i].get(v);
      std::memcpy(ptr, &v, sizeof(v));
    }
      break;

    case nix::DataType::String: {
      const char *str = vals[i].get<const char *>();
      std::memcpy(ptr, &str, sizeof(char **));
    }
      break;

    case nix::DataType::Double: {
      double v;
      vals[i].get(v);
      std::memcpy(ptr, &v, sizeof(v));
    }

    default:
      std::cerr << "Unsupported DatType" << std::endl;
    }
  }

  h5x::DataType ct = h5x::DataType::makeCompound(s);

  h5x::DataType dfile = ds.dataType();
  for (size_t i = 0; i < vals.size(); i++) {
    std::string name = dfile.member_name(i);
    ct.insert(name, offset[i], dtypes[i]);
  }

  std::cout << ct.isVariableString() << std::endl;
  
  ds.write(base, ct, nix::NDSize({1, 1}));
}

static h5::DataSet createDataFrame(h5::H5Group &group, const std::string &name, const std::vector<DataFrame::Header> &cols) {

  size_t s = 0;
  std::vector<size_t> offset(cols.size());
  std::vector<h5x::DataType> dtypes(cols.size());

  for (size_t i = 0; i < cols.size(); i++) {
    h5x::DataType ft = h5::data_type_to_h5_filetype(cols[i].dtype);
    dtypes[i] = ft;
    offset[i] = s;
    s += ft.size();
  }

  h5x::DataType ct = h5x::DataType::makeCompound(s);

  for (size_t i = 0; i < cols.size(); i++) {
    ct.insert(cols[i].name, offset[i], dtypes[i]);
  }

  h5::DataSet ds = group.createData(name, ct, nix::NDSize({1, 1}));
  return ds;
}

int main(int argc, char **argv) {

  std::vector<nix::Variant> vals = {nix::Variant(10),
                                    nix::Variant("test"),
                                    nix::Variant(42.0)};

  std::vector<nix::DataType> dtypes = {nix::DataType::Int32,
                                       nix::DataType::String,
                                       nix::DataType::Double};

  h5::H5Group fd = H5Fcreate("test_dataframe.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);

  std::vector<DataFrame::Header> hdr = {
    {"int32", "V", nix::DataType::Int32},
    {"string", "", nix::DataType::String},
    {"double", "mV", nix::DataType::Double}};

  h5::DataSet ds = createDataFrame(fd, "dataframe", hdr);

  DataFrame::writeRow(ds, vals);

  ds.close();
  fd.close();
  return 0;
}