Created
March 6, 2018 06:23
-
-
Save nihui/435c455e98e9e826220796109464a991 to your computer and use it in GitHub Desktop.
simple rec read write wrapper
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #ifndef MYREC_H | |
| #define MYREC_H | |
| #include <stdio.h> | |
| #include <stdlib.h> | |
| #include <string.h> | |
| #include <string> | |
| #include <vector> | |
| #include <dmlc/io.h> | |
| #include <dmlc/recordio.h> | |
| struct image_recordio_header | |
| { | |
| uint32_t label_count; | |
| float label0; | |
| uint64_t image_id[2]; | |
| }; | |
| static void buffer_append(std::string& buf, const void* data, int size) | |
| { | |
| int c = buf.size(); | |
| buf.resize(c + size); | |
| memcpy((void*)(buf.data() + c), data, size); | |
| } | |
| static void buffer_pick(std::string& buf, void* data, int size) | |
| { | |
| memcpy(data, buf.data(), size); | |
| int c = buf.size(); | |
| buf = std::string(buf.data() + size, c - size); | |
| } | |
| class REC | |
| { | |
| public: | |
| REC() | |
| { | |
| s = 0; | |
| writer = 0; | |
| reader = 0; | |
| } | |
| ~REC() | |
| { | |
| close(); | |
| } | |
| int open(const char* path, bool write) | |
| { | |
| if (write) | |
| { | |
| s = dmlc::Stream::Create(path, "wb"); | |
| writer = new dmlc::RecordIOWriter(s); | |
| } | |
| else | |
| { | |
| s = dmlc::Stream::Create(path, "rb"); | |
| reader = new dmlc::RecordIOReader(s); | |
| } | |
| return 0; | |
| } | |
| void close() | |
| { | |
| delete s; | |
| delete writer; | |
| delete reader; | |
| s = 0; | |
| writer = 0; | |
| reader = 0; | |
| } | |
| void write(uint64_t id, const std::string& imagebuf, const std::vector<float>& labels) | |
| { | |
| std::string buf; | |
| struct image_recordio_header h = { (unsigned int)labels.size(), labels[0], { id, 0 } }; | |
| buffer_append(buf, &h, sizeof(h)); | |
| buffer_append(buf, &labels[0], labels.size() * sizeof(float)); | |
| buffer_append(buf, imagebuf.data(), imagebuf.size()); | |
| writer->WriteRecord(buf.data(), buf.size()); | |
| } | |
| bool read(uint64_t& id, std::string& imagebuf, std::vector<float>& labels) | |
| { | |
| std::string buf; | |
| bool fail = reader->NextRecord(&buf); | |
| if (!fail) | |
| return fail; | |
| struct image_recordio_header h; | |
| buffer_pick(buf, &h, sizeof(h)); | |
| id = h.image_id[0]; | |
| labels.resize(h.label_count); | |
| buffer_pick(buf, &labels[0], labels.size() * sizeof(float)); | |
| imagebuf = buf; | |
| return true; | |
| } | |
| private: | |
| dmlc::Stream* s; | |
| dmlc::RecordIOWriter* writer; | |
| dmlc::RecordIOReader* reader; | |
| }; | |
| #endif // MYREC_H |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment