test_cpp_string_tokenize-00.cpp

gistfile1.cpp

#include <iostream>
#include <iterator>
#include <set>
#include <sstream>
#include <string>

namespace {

const std::string text("lonely lovely baboon sat on his ass an he was a very lovely but lonely too most likely the loveliest baboon though his ass wasn't lovely really");

        template<typename T>
        std::ostream& operator<<(std::ostream& out, const std::set<T>& s) {
                std::copy(s.begin(),
                          s.end(),
                          std::ostream_iterator<T>(out, " "));

                return out;
        }

}

int main(int argc, char* argv[]) {

        std::cout << "<input>" << text << "</input>" << std::endl;

        std::istringstream in(text);

        std::set<std::string> uniq;

        for(std::string s;
            std::getline(in, s, ' ');
            uniq.insert(s));

        std::cout << "<output>" << uniq << "</output>" << std::endl;
}