Skip to content

Instantly share code, notes, and snippets.

@nihui
Created March 6, 2018 06:23
Show Gist options
  • Save nihui/435c455e98e9e826220796109464a991 to your computer and use it in GitHub Desktop.
Save nihui/435c455e98e9e826220796109464a991 to your computer and use it in GitHub Desktop.
simple rec read write wrapper
#ifndef MYREC_H
#define MYREC_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string>
#include <vector>
#include <dmlc/io.h>
#include <dmlc/recordio.h>
struct image_recordio_header
{
uint32_t label_count;
float label0;
uint64_t image_id[2];
};
static void buffer_append(std::string& buf, const void* data, int size)
{
int c = buf.size();
buf.resize(c + size);
memcpy((void*)(buf.data() + c), data, size);
}
static void buffer_pick(std::string& buf, void* data, int size)
{
memcpy(data, buf.data(), size);
int c = buf.size();
buf = std::string(buf.data() + size, c - size);
}
class REC
{
public:
REC()
{
s = 0;
writer = 0;
reader = 0;
}
~REC()
{
close();
}
int open(const char* path, bool write)
{
if (write)
{
s = dmlc::Stream::Create(path, "wb");
writer = new dmlc::RecordIOWriter(s);
}
else
{
s = dmlc::Stream::Create(path, "rb");
reader = new dmlc::RecordIOReader(s);
}
return 0;
}
void close()
{
delete s;
delete writer;
delete reader;
s = 0;
writer = 0;
reader = 0;
}
void write(uint64_t id, const std::string& imagebuf, const std::vector<float>& labels)
{
std::string buf;
struct image_recordio_header h = { (unsigned int)labels.size(), labels[0], { id, 0 } };
buffer_append(buf, &h, sizeof(h));
buffer_append(buf, &labels[0], labels.size() * sizeof(float));
buffer_append(buf, imagebuf.data(), imagebuf.size());
writer->WriteRecord(buf.data(), buf.size());
}
bool read(uint64_t& id, std::string& imagebuf, std::vector<float>& labels)
{
std::string buf;
bool fail = reader->NextRecord(&buf);
if (!fail)
return fail;
struct image_recordio_header h;
buffer_pick(buf, &h, sizeof(h));
id = h.image_id[0];
labels.resize(h.label_count);
buffer_pick(buf, &labels[0], labels.size() * sizeof(float));
imagebuf = buf;
return true;
}
private:
dmlc::Stream* s;
dmlc::RecordIOWriter* writer;
dmlc::RecordIOReader* reader;
};
#endif // MYREC_H
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment