Created
November 30, 2023 14:05
-
-
Save niksteff/fb860463af47219af885277d700548e2 to your computer and use it in GitHub Desktop.
This is some go example code of two readers reading words/bytes from input data before passing a limited size buffer on to a server.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
func GetExampleData() []byte { | |
return []byte(`"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."`) | |
} | |
var ErrWordTooLong = errors.New("word too long") | |
var ErrBufferEmpty = errors.New("buffer empty") | |
var ErrEndOfWord = errors.New("end of word") | |
func TestWordBuffer(t *testing.T) { | |
b := bytes.NewBuffer(GetExampleData()) | |
r := bufio.NewReaderSize(b, b.Len()) | |
maxSize := 10 | |
for { | |
w, err := readWord(maxSize, r) | |
if err != nil { | |
if err == io.EOF { | |
t.Logf("%s", string(*w)) | |
break | |
} | |
if err == ErrEndOfWord { | |
t.Logf("%s", string(*w)) | |
continue | |
} | |
if err == ErrWordTooLong { | |
t.Error(err.Error()) | |
continue | |
} | |
t.Error(err) | |
continue | |
} | |
if w != nil { | |
t.Logf("%s", string(*w)) | |
} | |
} | |
} | |
func readWord(maxSize int, r *bufio.Reader) (*[]byte, error) { | |
size := 0 | |
word := []byte{} | |
for { | |
if size > maxSize { | |
return nil, ErrWordTooLong | |
} | |
b, err := r.ReadByte() | |
if err != nil { | |
if err == io.EOF { | |
return &word, io.EOF | |
} | |
return nil, err | |
} | |
if unicode.IsSpace(rune(b)) { | |
// end of word | |
if len(word) == 0 { | |
// jump over whitespace | |
continue | |
} | |
return &word, ErrEndOfWord | |
} | |
word = append(word, b) | |
size++ | |
} | |
} | |
func TestPassOnBytes(t *testing.T) { | |
// build a server | |
mux := http.NewServeMux() | |
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { | |
w.WriteHeader(http.StatusAccepted) | |
defer r.Body.Close() | |
b, err := io.ReadAll(r.Body) | |
if err != nil { | |
t.Errorf("error reading body: %s", err.Error()) | |
w.WriteHeader(http.StatusBadRequest) | |
_, err := w.Write([]byte(err.Error())) | |
if err != nil { | |
t.Errorf("error writing response: %s", err.Error()) | |
return | |
} | |
return | |
} | |
t.Logf("server received: %s", string(b)) | |
_, err = w.Write([]byte(`ok`)) | |
if err != nil { | |
t.Errorf("error writing response: %s", err.Error()) | |
return | |
} | |
}) | |
srv := httptest.NewServer(mux) | |
defer srv.Close() | |
b := bytes.NewBuffer(GetExampleData()) | |
client := http.DefaultClient | |
client.Timeout = 5 * time.Second | |
for w := range scanBytes(2, bufio.NewScanner(b)) { | |
func() { | |
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) | |
defer cancel() | |
r := bytes.NewReader(w) | |
req, err := http.NewRequestWithContext(ctx, http.MethodPost, srv.URL, r) | |
if err != nil { | |
t.Errorf("error creating request: %s", err.Error()) | |
return | |
} | |
res, err := client.Do(req) | |
if err != nil { | |
t.Errorf("error sending request: %s", err.Error()) | |
return | |
} | |
if res.StatusCode != http.StatusAccepted { | |
t.Errorf("unexpected status code: %d", res.StatusCode) | |
return | |
} | |
}() | |
time.Sleep(250 * time.Millisecond) | |
} | |
} | |
func scanBytes(amount int, scan *bufio.Scanner) <-chan []byte { | |
out := make(chan []byte) | |
go func() { | |
defer close(out) | |
scan.Split(bufio.ScanBytes) | |
batch := make([]byte, 0, amount) | |
for scan.Scan() { | |
word := scan.Bytes() | |
batch = append(batch, word...) | |
if len(batch) == amount { | |
out <- batch | |
batch = make([]byte, 0, amount) | |
} | |
} | |
out <- batch | |
}() | |
return out | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment