Flattening Iterators

fileline.cpp

#include "fileline.h"

FileLine::FileLine() {}

std::istream &operator >>(std::istream &istr, FileLine &line)
{
//    size_t count;
//    std::string title;
//    std::vector<std::string> list;
    line.list.clear();
    line.title = std::string();
    line.count = 0;
    istr >> line.count;
    if (line.count > 0) {
        istr >> line.title;
        for (int i = 0; i < (line.count - 1); ++i ) {
            std::string inp ;
            istr >> inp;
            line.list.push_back(inp);
        }
    }
    return istr;
}

std::ostream &operator <<(std::ostream &ostr, const FileLine &line)
{
    ostr << "count: " << line.count << " title: " << line.title << " list: [";
    for (auto l : line.list) {
        ostr << l << " ";
    }
    ostr << "]\n";
    return ostr;
}

fileline.h

#ifndef FILELINE_H
#define FILELINE_H

#include <iostream>
#include <string>
#include <vector>

class FileLine
{


public:
    FileLine();

    size_t count;
    std::string title;
    std::vector<std::string> list;

};


std::istream& operator >> ( std::istream& istr , FileLine& line);
std::ostream& operator << ( std::ostream& ostr , const FileLine& line);

#endif // FILELINE_H

Flattening Iterators

Flattening Iterators
====================

Summary
-------

This is an interator template that fits a pattern whereby you need to 
return a list of objects streamed out of a sorted list of files.

For instance, the Bitcoin blockchain comes in block files (blk00000.dat).

We want to hide the individual block files from the consumer of our iterator.

The outer iterator is just a ```std::vector<std::string>```. The inner iterator is a ```std::istream_iterator<TYPE>```, 
where ```TYPE``` can be used to stream in objects from the file.

flattening_iterator.h

#ifndef FLATITER_H
#define FLATITER_H

#include <vector>
#include <string>
#include <functional>
#include <fstream>
#include <iostream>
#include <cassert>
#include <memory>

template< typename OUTERCOL , typename INNERCOL  >
struct flattening_iterator
{

  typename OUTERCOL::iterator outer_iterator_;
  typename OUTERCOL::iterator outer_iterator_end_;
  std::ifstream current_;
  INNERCOL current_inner_;
  INNERCOL current_inner_end_;
  bool is_end_ = false;
  size_t index = 0;

  bool prepare_file(typename OUTERCOL::iterator it) {
      if (it == outer_iterator_end_) return false ;

      current_ = std::ifstream( *it);
      current_.seekg(0,std::ios_base::end);
      size_t file_size = current_.tellg();
      current_.seekg(0,std::ios_base::beg);

      if (file_size > 0)
        current_inner_ = INNERCOL( current_ );
      else
        current_inner_ = INNERCOL();

      current_inner_end_ = INNERCOL();

      bool empty =  (current_inner_ == current_inner_end_) ;
      return !empty;
  }

  void find_next_non_empty_outer(  )
  {
      while ( outer_iterator_ != outer_iterator_end_) {
          bool contains_data = prepare_file( outer_iterator_ );
          outer_iterator_++;
          if (contains_data) {
              return;
          }
      }
      if (outer_iterator_ == outer_iterator_end_) is_end_ = true;
  }

  flattening_iterator( const flattening_iterator& it )
    : current_inner_(it.current_inner_),
      current_inner_end_( it.current_inner_end_),
      outer_iterator_(it.outer_iterator_),
      outer_iterator_end_(it.outer_iterator_end_),
      index(it.index)
  {}


  static flattening_iterator end( OUTERCOL& outer )
  {
    return flattening_iterator( outer.end() , outer.end());
  }

  flattening_iterator( typename OUTERCOL::iterator start , typename OUTERCOL::iterator end )
    :  outer_iterator_( start ) , outer_iterator_end_( end)
  {
    find_next_non_empty_outer();
  }

  flattening_iterator(  OUTERCOL& outer )
    : flattening_iterator( outer.begin() , outer.end() )
  {}

  bool operator == ( const flattening_iterator< OUTERCOL , INNERCOL >&  it) {

    if (is_end_ && it.is_end_) return true;

    return (outer_iterator_ == it.outer_iterator_)
            && (current_inner_ == it.current_inner_);
  }

  bool operator != ( const flattening_iterator<OUTERCOL , INNERCOL >&  it) {
      return ! (*this == it);
  }

  flattening_iterator<OUTERCOL , INNERCOL >& operator++() {
    current_inner_ ++;
    index++;
    if (current_inner_ == current_inner_end_) {
        find_next_non_empty_outer();
    }
    return *this;
  }

  flattening_iterator< OUTERCOL , INNERCOL > operator++(int) {
      flattening_iterator<OUTERCOL , INNERCOL > tmp(*this);
      ++*this;
      return tmp;
  }

  typename INNERCOL::value_type operator*() const {
    return * current_inner_;
  }
};

#endif // FLATITER_H

test_main.cp

#include <iostream>
#include <iterator>
#include <fstream>
#include <vector>
#include <string>
#include <blockchain.h>
#include <initializer_list>
#include <cassert>
#include <sstream>

#include "fileline.h"
#define CURDIR TESTFILESDIR

typedef std::istream_iterator<FileLine> bc_file_it_t;
typedef flattening_iterator< std::vector<std::string> , bc_file_it_t > bc_it_t;

#define TEST(x) std::cout << "[TEST  ] " << x << "\n[RESULT] ";
#define PASS std::cout << "PASS\n\n";

template< typename T >
size_t std_distance( T start , T end ) {
    size_t sz = 0;
    for (auto x = start ; x != end ; x++) {
        ++sz;
    }
    return sz;
}


int main()
{

    {
        TEST("Test0.1 - test stream it with empty file");
        bc_file_it_t it( CURDIR "empty.txt" );
        assert( it.begin() == it.end() );
        PASS
    }

    {
        TEST("Test0.2 - test stream it with non-empty empty file");
        bc_file_it_t it( CURDIR "file_1.txt" );
        assert( it.begin() != it.end() );
        size_t count = std_distance( it.begin() , it.end() );
        assert ( count == 5);
        PASS
    }


    {
        TEST("Test1 - single empty file")
        std::vector<std::string> ss {  CURDIR "empty.txt"  };
        bc_it_t bc( ss  );
        bc_it_t end = bc_it_t::end( ss  );
        assert( bc == end);
        PASS
    }



    {
        TEST("Test2 - two empty files")
        std::vector<std::string> ss {  CURDIR "empty2.txt", CURDIR "empty.txt"  };
        bc_it_t bc( ss  );
        bc_it_t end = bc_it_t::end( ss  );
        assert( bc == end);
        PASS
    }

    {
        TEST("Test3 - no files")
        std::vector<std::string> ss {  };
        bc_it_t bc( ss  );
        bc_it_t end = bc_it_t::end( ss  );
        assert( bc == end);
        PASS
    }

    {
        TEST("Test4 - empty file folled by non empty file")
        std::vector<std::string> ss { CURDIR "empty.txt" , CURDIR "file_1.txt" };
        bc_it_t bc( ss  );
        bc_it_t end = bc_it_t::end( ss  );
        assert( bc != end);
        size_t count = std_distance(bc,end);
        assert( count == 5);
        PASS
    }


    {
        TEST("Test5 - non empty file followed by empty file")
        std::vector<std::string> ss {  CURDIR "file_1.txt", CURDIR "empty.txt" };
        bc_it_t bc( ss  );
        bc_it_t end = bc_it_t::end( ss  );
        assert( bc != end);
        size_t count = std_distance(bc,end);
        assert( count == 5);
        PASS
    }



    {
        TEST("Test6 - single non-empty file")
        std::vector<std::string> ss {  CURDIR "file_1.txt" };
        bc_it_t bc( ss  );
        bc_it_t end = bc_it_t::end( ss  );
        assert( bc != end);
        size_t count = std_distance(bc,end);
        assert( count == 5);
        PASS
    }



    {
        TEST("Test7 - two non-empty file")
        std::vector<std::string> ss {  CURDIR "file_1.txt" , CURDIR "file_2.txt"   };
        bc_it_t bc( ss  );
        bc_it_t end = bc_it_t::end( ss  );
        assert( bc != end);
        size_t count = std_distance(bc,end);
        assert( count == 9);
        PASS
    }

    {
        TEST("Test8 - three non-empty file")
        std::vector<std::string> ss {  CURDIR "file_1.txt" , CURDIR "file_2.txt" , CURDIR "file_3.txt" };
        bc_it_t bc( ss  );
        bc_it_t end = bc_it_t::end( ss  );
        assert( bc != end);
        size_t count = std_distance(bc,end);
        assert( count == 13);
        PASS
    }

    {
        TEST("Test9 - non empty followed by empty folled by nonempty")
        std::vector<std::string> ss {  CURDIR "file_1.txt" , CURDIR "empty.txt" , CURDIR "file_2.txt" };
        bc_it_t bc( ss  );
        bc_it_t end = bc_it_t::end( ss  );
        assert( bc != end);
        size_t count = std_distance(bc,end);
        assert( count == 9);
        assert( bc != end);
        PASS
    }


    {
        std::string results[] {
            "count: 3 title: FILE1.1 list: [BB CC ]\n",
            "count: 2 title: FILE1.2 list: [BB ]\n",
            "count: 5 title: FILE1.3 list: [BB CC DD EE ]\n",
            "count: 1 title: FILE1.4 list: []\n",
            "count: 0 title:  list: []\n",
            "count: 3 title: FILE2.1 list: [BB CC ]\n",
            "count: 2 title: FILE2.2 list: [BB ]\n",
            "count: 5 title: FILE2.3 list: [BB CC DD EE ]\n",
            "count: 1 title: FILE2 list: []\n"
        };

        TEST("Test10 - same as test 9 but looking at output")
        std::vector<std::string> ss {  CURDIR "file_1.txt" , CURDIR "empty.txt" , CURDIR "file_2.txt" };
        bc_it_t bc( ss  );
        bc_it_t end = bc_it_t::end( ss  );
        assert( bc != end);
        for (auto fl = bc ; fl != end ; ++fl  ) {
            std::ostringstream ostrstrm;
            ostrstrm << *fl;
            assert(  ostrstrm.str() == results[fl.index] );
        }
        PASS
    }

    std::cout << "******** ALL TESTS PASSED ***************\n";
    return 0;
}

TESTFILES.md

empty.txt

empty2.txt

file_1.txt

03 FILE1.1 BB CC
02 FILE1.2 BB
05 FILE1.3 BB CC DD EE
1 FILE1.4
0

file_2.txt

03 FILE2.1 BB CC
02 FILE2.2 BB
05 FILE2.3 BB CC DD EE
1 FILE2

file_3.txt

04 FILE3.1 BB CC AA
02 FILE3.2 BB
05 FILE3.3 BB CC DD EE
1 FILE3.4