Created
November 12, 2016 19:54
-
-
Save blippy/2c66cbc4c491e171ad41b3301d447f2a to your computer and use it in GitHub Desktop.
Deleted tokenising code
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| strings multi_split(const string& line, char c) | |
| { | |
| strings result; | |
| std::size_t prev = 0, pos; | |
| while((pos = line.find_first_of(c, prev)) != std::string::npos) | |
| { | |
| if(pos>prev) | |
| result.push_back(line.substr(prev, pos-prev)); | |
| prev = pos+1; | |
| } | |
| if (prev<line.length()) | |
| result.push_back(line.substr(prev, std::string::npos)); | |
| return result; | |
| } | |
| // TODO definitely reusable! | |
| string char_to_string(char c) | |
| { | |
| char cstr[] = {c}; | |
| return cstr; | |
| } | |
| strings tokenise_2(const string& line) | |
| { | |
| constexpr char FS = 0x1C; | |
| bool found_quote = false; | |
| char* line_cs = (char *)line.c_str(); // potentially tricky, I guess | |
| for(int i=0; i< line.size(); ++i){ | |
| char c = line[i]; | |
| if(c == '"') { | |
| found_quote = true ; | |
| line_cs[i] = FS; | |
| } | |
| if(found_quote) continue; | |
| if(c == ' ' || c == '\t') line_cs[i] = FS; | |
| } | |
| return multi_split(line, FS); | |
| } | |
| void check_tokeniser_2(const string& msg, const string& input, const strings& expected_result) | |
| { | |
| strings strs = tokenise_2(input); | |
| if(false) { // for debugging purposes | |
| for(auto s:strs) cout << "[" << s << "] "; | |
| cout << endl; | |
| } | |
| bool ok = strs == expected_result; | |
| check(ok, msg); | |
| } | |
| void check_tokeniser_2_all() | |
| { | |
| check_tokeniser_2("toke2-01", " how now brown cow", | |
| {"how", "now", "brown", "cow"}); | |
| check_tokeniser_2("toke2-02", "how now \"brown cow\"", | |
| {"how", "now", "brown cow"}); | |
| string cow = "how now\t \"brown cow\""; | |
| Timer time; | |
| time.start(); | |
| for(int i =0; i<1000; ++i) parse::tokenise_line(cow); | |
| time.stop(); | |
| cout << "Elapsed: " << time.nanos() << endl; | |
| time.start(); | |
| for(int i =0; i<1000; ++i) tokenise_2(cow); | |
| time.stop(); | |
| cout << "Elapsed: " << time.nanos() << endl; | |
| } |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
It turns out that this code is slower than my original code. So I should just stick to my original code. I am surprised there is such a bottleneck. find_first_of() perhaps??