Skip to content

Instantly share code, notes, and snippets.

@Redchards
Created September 23, 2015 14:53
Show Gist options
  • Save Redchards/1c825fa48f7bfc013b32 to your computer and use it in GitHub Desktop.
Save Redchards/1c825fa48f7bfc013b32 to your computer and use it in GitHub Desktop.
Very simple asynchronous file reader and writer, along with a utility file class.
#include <AsyncFileReader.hxx>
AsyncFileReader::~AsyncFileReader()
{
if(th_.joinable())
{
th_.join();
}
}
auto AsyncFileReader::read(size_t numByte) -> jobQueue<ByteStream>
{
// Do the sysconf all incure some latency ? To benchmark, maybe it's only an accessor-like function.
// If it is, then RVO will do its job.
size_t pageSize = sysconf(_SC_PAGESIZE);
size_t chunkNumber = numByte / pageSize;
jobQueue<ByteStream> futures;
std::queue<std::thread> threadQueue;
for(size_t i = 0; i <= chunkNumber; ++i)
{
promises.push_back(std::move(promiseType()));
futures.push(promises.back().get_future());
}
// Someone told me I should use something like "std::bind", because this would cause a copy.
// So I assume that this person never heard of move semantic in C++.
th_ = std::thread{[&, pageSize, chunkNumber, numByte](){
for(size_t i = 0; i <= chunkNumber; ++i)
{
promises[i].set_value(this->file_->read((i != chunkNumber ? pageSize : numByte - (pageSize * (chunkNumber)))));
}
}};
return futures;
}
#ifndef UZIP_ASYNC_FILE_READER
#define UZIP_ASYNC_FILE_READER
#include <ByteStream.hxx>
#include <CommonTypes.hxx>
#include <File.hxx>
#include <future>
#include <queue>
#include <string>
#include <vector>
class AsyncFileReader
{
using promiseType = std::promise<ByteStream>;
public:
AsyncFileReader(File* file) : file_(file),
buffer_{ new char[sysconf(_SC_PAGESIZE)] },
th_{}
{};
~AsyncFileReader();
template<class T>
using jobQueue = std::queue<std::future<T>>;
jobQueue<ByteStream> read(size_t numBytes);
private:
File* file_;
std::unique_ptr<char[]> buffer_;
std::thread th_;
std::vector<promiseType> promises;
};
#endif // UZIP_ASYNC_FILE_READER
#include <AsyncFileWriter.hxx>
AsyncFileWriter::~AsyncFileWriter()
{
if(th_.joinable())
{
th_.join();
}
}
void AsyncFileWriter::writeBuffer(const char* buffer, size_t byteNum)
{
size_t pageSize = sysconf(_SC_PAGESIZE);
size_t chunkNumber = byteNum/pageSize;
th_ = std::thread([=](){
try
{
for(size_t i = 0; i <= chunkNumber; ++i)
{
file_->writeBuffer(buffer + (i*pageSize), (i != chunkNumber ? pageSize : byteNum - (i*pageSize)));
}
}
// TODO : Modify this !
catch(IOException e)
{
std::cout << e.what() << std::endl;
}
});
}
void AsyncFileWriter::write(const std::string& str)
{
writeBuffer(str.data(), str.length());
}
#ifndef UZIP_ASYNC_FILE_WRITER
#define UZIP_ASYNC_FILE_WRITER
#include <ByteStream.hxx>
#include <CommonTypes.hxx>
#include <File.hxx>
#include <thread>
class AsyncFileWriter
{
public:
AsyncFileWriter(File* file) : file_(file),
th_{}
{};
~AsyncFileWriter();
void writeBuffer(const char* buffer, size_t byteNum);
void write(const std::string& buf);
private:
File* file_;
std::thread th_;
};
#endif // UZIP_ASYNC_FILE_WRITER
#ifndef UZIP_BYTESTREAM
#define UZIP_BYTESTREAM
#include <memory>
class ByteStream
{
public:
ByteStream(char* c) : ptr_(c) {}
ByteStream(std::unique_ptr<char[]>&& ptr) : ptr_(std::move(ptr)) {}
operator char*() const{ return ptr_.get(); }
private:
std::unique_ptr<char[]> ptr_;
};
#endif // UZIP_BYTESTREAM
#ifndef COMMONTYPES
#define COMMONTYPES
/************************************************************************/
// Internet Software Consortium (ISC) License
// Version 1, December 2015
//
// Copyright (C) 2015 Loic URIEN <[email protected]>
//
// Permission to use, copy, modify, and/or distribute this software
// for any purpose with or without fee is hereby granted,
// provided that the above copyright notice and this permission notice
// appear in all copies unless the author says otherwise.
//
// THE SOFTWARE IS PROVIDED "AS IS"
// AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
// INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
// DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
// RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
/************************************************************************/
#include <limits>
#include <cstddef>
#include <cstdint>
typedef uint64_t allocation_size_type;
typedef ptrdiff_t ptrdiff;
typedef int8_t int8;
typedef int16_t int16;
typedef int32_t int32;
typedef int64_t int64;
typedef uint8_t uint8;
typedef uint16_t uint16;
typedef uint32_t uint32;
typedef uint64_t uint64;
typedef size_t max_type;
#endif // COMMONTYPES
#ifndef UZIP_EXCEPTIONS
#define UZIP_EXCEPTIONS
#include <CommonTypes.hxx>
#include <Platform.hxx>
#include <exception>
#include <sstream>
#include <string>
#include <climits>
#include <sys/stat.h>
// TODO move this in a header (CommonTypes for example) to be more versatile.
enum class IOAction : uint16 {
OPEN = 0,
CLOSE,
READ,
WRITE,
CREATE
};
// Returning msg_.data() to avoid copying the string with c_str.
// But this is valid only because the exception object will be destroyed just after the exception handling process,
// and the string never modified after the object creation.
class BasicException : std::exception
{
public:
BasicException(std::string const& msg) : msg_(msg) {};
virtual const char* what() const noexcept
{
return msg_.data();
}
protected:
std::string msg_;
};
class CStatException : public BasicException
{
public:
CStatException(int errCode, std::string const& filePath) : BasicException("Error during the call of 'stat()' function : ")
{
std::ostringstream oss;
switch(errCode)
{
case EACCES:
msg_ += "access permission is denied";
break;
case EIO:
msg_ += "an error occured while reading the filesystem";
break;
case ELOOP:
msg_ += "symbolic link loop detected";
break;
case ENAMETOOLONG:
msg_ += "the name of the path is too long (max length is ";
msg_ += (oss << PATH_MAX, oss).str();
msg_ += ") or a component is too long (max length of component is ";
msg_ += (oss << NAME_MAX, oss).str();
msg_ += ")";
break;
case ENOENT: case ENOTDIR:
msg_ += "the given path (" + filePath + ") resulted in a 'no such file or directory' error";
break;
case EOVERFLOW:
msg_ += "the given buf structured was too small for the file";
break;
default:
msg_ += "unknown error occured";
}
}
};
class IOException : public BasicException
{
public:
IOException(std::string const& pathName, const IOAction action) : BasicException(pathName + " : I/O error during file ")
{
switch(action)
{
case IOAction::OPEN:
msg_ += "openning.";
break;
case IOAction::CLOSE:
msg_ += "closing.";
break;
case IOAction::READ:
msg_ += "reading.";
break;
case IOAction::WRITE:
msg_ += "writing.";
break;
case IOAction::CREATE:
msg_ += "creation.";
default:
msg_ += "unknown I/O action.";
}
}
};
#endif // UZIP_EXCEPTIONS
#include <File.hxx>
File::File(const std::string& pathName) : fileInfos_(initPath(pathName)),
filePtr_(nullptr)
{
init();
}
File::File(std::string&& pathName): fileInfos_(initPath(std::move(pathName))),
filePtr_(nullptr)
{
init();
}
File::File(const File& other) : fileInfos_(other.fileInfos_.getFilePath()),
filePtr_(nullptr)
{
init();
}
File::File(File&& other) : fileInfos_(std::move(other.fileInfos_.getFilePath()))
{
std::swap(filePtr_, other.filePtr_);
other.filePtr_ = nullptr;
init();
}
File::~File()
{
std::fclose(filePtr_);
}
bool File::operator==(const File& rhs) const noexcept
{
return ((filePtr_ == rhs.filePtr_) && (fileInfos_ == rhs.fileInfos_));
}
File::operator bool()
{
return fileInfos_.exists();
}
FileInfos File::getFileInfos() const noexcept
{
return fileInfos_;
}
size_t File::getCursorPosition() const noexcept
{
return std::ftell(filePtr_);
}
void File::setCursorPosition(size_t pos) const noexcept
{
std::fseek(filePtr_, pos, SEEK_SET);
}
void File::goToEnd() noexcept
{
std::fseek(filePtr_, fileInfos_.getFileSize(), SEEK_SET);
}
char File::readByte() const
{
char tmp = std::fgetc(filePtr_);
if(tmp == EOF)
{
checkForReadError();
}
return tmp;
}
ByteStream File::read()
{
return read(fileInfos_.getFileSize());
}
ByteStream File::read(size_t byteNum) const
{
// Set this to debug, to see if we attempt to read furhter than the EOF
/*if(bytes > fileInfos_.getFileSize() - ftell(filePtr_))
{
throw IOException(fileInfos_.getFilePath(), IOAction::READ);
}*/
// WARNING : The fread function will read byteNum + 1 bytes. So we need to substract.
// This is due by the fact that it will first read the file into *(buf + 0), then *(buf + 1), etc etc ...
// until it get *(buf + n). If n is byteNum, and buf is sizeof byteNum, then we will do a buffer overrun.
// So yhea, we need to substract. In fact, this is more a reminder for myself than a true warning.
std::unique_ptr<char[]> buf{ new char[byteNum] };
size_t result = std::fread(buf.get(), 1, byteNum - 1, filePtr_);
if(result < byteNum)
{
checkForReadError();
}
return { std::move(buf) };
}
void File::writeByte(char byte) const
{
char tmp = std::fputc(byte, filePtr_);
if(tmp == EOF)
{
checkForWriteError();
}
}
void File::writeBuffer(const char* buf, size_t byteNum) const
{
size_t result = std::fwrite(buf, 1, byteNum, filePtr_);
if(result < byteNum)
{
checkForWriteError();
}
}
void File::write(const std::string& buf) const
{
writeBuffer(buf.c_str(), buf.length());
}
void File::write(std::string&& buf) const
{
writeBuffer(std::move(buf.data()), buf.length());
}
void File::rewind()
{
std::fseek(filePtr_, 0, SEEK_SET);
}
File File::create(const std::string& filePath)
{
std::fclose(std::fopen(filePath.data(), "w"));
File newFile(filePath);
newFile.fileInfos_.updateFileInfos();
newFile.init();
return newFile;
}
void File::init()
{
filePtr_ = std::fopen((fileInfos_.exists() ? fileInfos_.getFilePath().c_str() : ""), "r+");
if(filePtr_ == nullptr)
{
throw IOException(fileInfos_.getFilePath(), IOAction::OPEN);
}
}
void File::checkForReadError() const
{
if((ferror(filePtr_) != 0))
{
throw IOException(fileInfos_.getFilePath(), IOAction::READ);
}
}
void File::checkForWriteError() const
{
if((ferror(filePtr_) != 0))
{
throw IOException(fileInfos_.getFilePath(), IOAction::WRITE);
}
}
#ifndef UZIP_FILE
#define UZIP_FILE
#include <ByteStream.hxx>
#include <FileInfos.hxx>
#include <cstring>
#include <fstream>
// NOTE : Should I reimplement it with stl instead of c lib ?
// NOTE : Add append function
// NOTE : Add a "write" function supporting std::string
// TODO : Add exceptions for cursor manipulation if realy needed
class File
{
public:
File(const std::string& pathName);
File(std::string&& pathName);
File(const File& other);
File(File&& other);
~File();
bool operator==(const File&) const noexcept;
operator bool();
FileInfos getFileInfos() const noexcept;
size_t getCursorPosition() const noexcept;
void setCursorPosition(size_t pos) const noexcept;
void goToEnd() noexcept;
char readByte() const;
ByteStream read();
ByteStream read(size_t byteNum) const;
void writeByte(char byte) const;
void writeBuffer(const char* buf, size_t byteNum) const;
void write(const std::string& buf) const;
void write(std::string&& buf) const;
void rewind();
static File create(const std::string& filePath);
private:
void init();
template<class T>
std::string initPath(T&& pathName) const;
void checkForReadError() const;
void checkForWriteError() const;
FileInfos fileInfos_;
FILE* filePtr_;
};
template<class T>
std::string File::initPath(T&& pathName) const
{
if(pathName[0] == FileInfos::pathDelimiter)
{
return std::move(std::string(pathName));
}
std::string f(std::forward<T>(pathName));
auto last = f.find_last_of(FileInfos::pathDelimiter);
char* tmp{ get_current_dir_name() };
std::string retval = "";
if(last != std::string::npos)
{
retval = std::string(tmp + f.substr(last, f.length() - 1));
free(tmp);
return retval;
}
retval = std::string(tmp) + FileInfos::pathDelimiter + std::string(std::move(f));
free(tmp);
return retval;
}
#endif // UZIP_FILE
#include <FileInfos.hxx>
char FileInfos::pathDelimiter = '/';
bool FileInfos::operator==(const FileInfos& rhs) const noexcept
{
return (filePath_ == rhs.filePath_);
}
const std::string& FileInfos::getFilePath() const noexcept
{
return filePath_;
}
const std::string FileInfos::getFileName() const noexcept
{
auto last = filePath_.find_last_of(pathDelimiter);
return filePath_.substr(last, filePath_.length() - 1);
}
uint64 FileInfos::getFileSize() noexcept
{
updateFileInfos();
return fileStat_.st_size;
}
bool FileInfos::exists() noexcept
{
updateFileInfos();
return exists_;
}
std::unique_ptr<tm> FileInfos::getLastAccessTime() noexcept
{
updateFileInfos();
return std::unique_ptr<tm>{ gmtime(&(fileStat_.st_atime)) };
}
std::unique_ptr<tm> FileInfos::getLastModificationTime() noexcept
{
updateFileInfos();
return std::unique_ptr<tm>{ gmtime(&(fileStat_.st_mtime)) };
}
void FileInfos::updateFileInfos()
{
auto err = stat(filePath_.data(), &fileStat_);
if(err != 0)
{
if(err != ENOENT && err != ENOTDIR)
{
exists_ = false;
}
else
{
throw CStatException(err, filePath_);
}
}
}
#ifndef UZIP_FILE_INFOS
#define UZIP_FILE_INFOS
#include <CommonTypes.hxx>
#include <Exceptions.hxx>
#include <Platform.hxx>
#include <climits>
#include <ctime>
#include <unistd.h>
#include <sys/stat.h>
#include <memory>
#include <string>
#include <iostream>
class FileInfos
{
public:
template<class T>
FileInfos(T&& filePath) :
filePath_(filePath[0] == '/' ? std::forward<T>(filePath) : ""),
exists_(true),
fileStat_{}
{
updateFileInfos();
}
bool operator==(const FileInfos& rhs) const noexcept;
const std::string& getFilePath() const noexcept;
const std::string getFileName() const noexcept;
uint64 getFileSize() noexcept;
bool exists() noexcept;
std::unique_ptr<tm> getLastAccessTime() noexcept;
std::unique_ptr<tm> getLastModificationTime() noexcept;
void updateFileInfos();
static char pathDelimiter;
private:
const std::string filePath_;
bool exists_;
struct stat fileStat_;
// To change on other systems
};
#endif // UZIP_FILE_INFOS
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment