Skip to content

Instantly share code, notes, and snippets.

@mertyildiran

mertyildiran/README.md

Last active October 9, 2021 11:09
Download ZIP

Revisions

  1. mertyildiran revised this gist Oct 9, 2021. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,6 +1,6 @@
    ## Reproducing malformed HTTP packets

    This is a slightly edited version of [reassemblydump of GoPacket](https://github.com/google/gopacket/blob/3eaba08943250fd212520e5cff00ed808b8fc60a/examples/reassemblydump/main.go).
    This is a slightly edited version of [reassemblydump example of GoPacket](https://github.com/google/gopacket/blob/3eaba08943250fd212520e5cff00ed808b8fc60a/examples/reassemblydump/main.go).

    Setup [Sock Shop](https://microservices-demo.github.io/) and its load tests in Google Kubernetes Engine.

  2. mertyildiran revised this gist Oct 9, 2021. 1 changed file with 2 additions and 0 deletions.
    2 changes: 2 additions & 0 deletions README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,7 @@
    ## Reproducing malformed HTTP packets

    This is a slightly edited version of [reassemblydump of GoPacket](https://github.com/google/gopacket/blob/3eaba08943250fd212520e5cff00ed808b8fc60a/examples/reassemblydump/main.go).

    Setup [Sock Shop](https://microservices-demo.github.io/) and its load tests in Google Kubernetes Engine.

    Add and enter into sniffer pod:
  3. mertyildiran revised this gist Oct 9, 2021. 1 changed file with 14 additions and 0 deletions.
    14 changes: 14 additions & 0 deletions stdout4
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,14 @@
    Processed 879000 packets (306545482 bytes) in 2m36.2046171s (errors: 0, errTypes:0)
    Processed 880000 packets (306979871 bytes) in 2m36.412358992s (errors: 0, errTypes:0)
    Processed 881000 packets (307191407 bytes) in 2m36.527988927s (errors: 0, errTypes:0)
    Processed 882000 packets (307828000 bytes) in 2m36.633326568s (errors: 0, errTypes:0)
    Processed 883000 packets (308354200 bytes) in 2m36.8194696s (errors: 0, errTypes:0)
    Processed 884000 packets (308678065 bytes) in 2m37.001787402s (errors: 0, errTypes:0)
    Processed 885000 packets (309058639 bytes) in 2m37.11179756s (errors: 0, errTypes:0)
    Processed 886000 packets (309390268 bytes) in 2m37.480168344s (errors: 0, errTypes:0)
    Processed 887000 packets (309730478 bytes) in 2m37.703792308s (errors: 0, errTypes:0)
    Processed 888000 packets (310076998 bytes) in 2m37.813966134s (errors: 0, errTypes:0)
    Processed 889000 packets (310455502 bytes) in 2m38.007170931s (errors: 0, errTypes:0)
    Processed 890000 packets (310738704 bytes) in 2m38.29305549s (errors: 0, errTypes:0)
    HTTP/10.32.4.5->10.40.2.163 39806->80 Response error: malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/items" (malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/items",malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/items")
    HTTP/10.32.4.5->10.32.1.16 39806->80 Response error: malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/items" (malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/items",malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/items")
  4. mertyildiran revised this gist Oct 9, 2021. 1 changed file with 41 additions and 1 deletion.
    42 changes: 41 additions & 1 deletion stdout3
    View file
    Original file line number Diff line number Diff line change
    @@ -22,4 +22,44 @@ Processed 63183000 packets (22025247559 bytes) in 3h29m14.709286237s (errors: 0,
    Processed 63184000 packets (22025663034 bytes) in 3h29m14.903195091s (errors: 0, errTypes:0)
    Processed 63185000 packets (22025944898 bytes) in 3h29m14.963401643s (errors: 0, errTypes:0)
    HTTP/10.32.4.5->10.40.5.168 39414->80 Response error: malformed HTTP status code "/orders" (malformed HTTP status code "/orders",malformed HTTP status code "/orders")
    HTTP/10.32.4.5->10.32.3.16 39414->80 Response error: malformed HTTP status code "/orders" (malformed HTTP status code "/orders",malformed HTTP status code "/orders")
    HTTP/10.32.4.5->10.32.3.16 39414->80 Response error: malformed HTTP status code "/orders" (malformed HTTP status code "/orders",malformed HTTP status code "/orders")
    root@gke-mertyildiran-test-2-default-pool-88ccd2a7-g09s:/tmp# du -sh *
    22G pcapgo679303473
    96M trace00
    96M trace01
    96M trace02
    96M trace03
    96M trace04
    96M trace05
    96M trace06
    96M trace07
    96M trace08
    96M trace09
    96M trace10
    96M trace11
    96M trace12
    96M trace13
    96M trace14
    96M trace15
    96M trace16
    96M trace17
    96M trace18
    96M trace19
    96M trace20
    96M trace21
    96M trace22
    96M trace23
    96M trace24
    96M trace25
    96M trace26
    96M trace27
    96M trace28
    96M trace29
    1.7M trace30
    13M trace31
    96M trace32
    55M trace33
    96M trace34
    96M trace35
    26M trace36
    36M trace37
  5. mertyildiran revised this gist Oct 9, 2021. 1 changed file with 25 additions and 0 deletions.
    25 changes: 25 additions & 0 deletions stdout3
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,25 @@
    Processed 63163000 packets (22018608433 bytes) in 3h29m9.936404707s (errors: 0, errTypes:0)
    Processed 63164000 packets (22019076472 bytes) in 3h29m10.144506291s (errors: 0, errTypes:0)
    Processed 63165000 packets (22019344946 bytes) in 3h29m10.349658069s (errors: 0, errTypes:0)
    Processed 63166000 packets (22019679324 bytes) in 3h29m10.530290859s (errors: 0, errTypes:0)
    Processed 63167000 packets (22019993110 bytes) in 3h29m10.738000317s (errors: 0, errTypes:0)
    Processed 63168000 packets (22020306188 bytes) in 3h29m10.845088292s (errors: 0, errTypes:0)
    Processed 63169000 packets (22020624357 bytes) in 3h29m11.041834446s (errors: 0, errTypes:0)
    Processed 63170000 packets (22020962380 bytes) in 3h29m11.146049921s (errors: 0, errTypes:0)
    Processed 63171000 packets (22021347377 bytes) in 3h29m11.269324782s (errors: 0, errTypes:0)
    Processed 63172000 packets (22021591774 bytes) in 3h29m11.436771009s (errors: 0, errTypes:0)
    Processed 63173000 packets (22021971323 bytes) in 3h29m11.538732892s (errors: 0, errTypes:0)
    Processed 63174000 packets (22022219541 bytes) in 3h29m11.756398072s (errors: 0, errTypes:0)
    Processed 63175000 packets (22022624535 bytes) in 3h29m11.852018808s (errors: 0, errTypes:0)
    Processed 63176000 packets (22022942354 bytes) in 3h29m12.191820411s (errors: 0, errTypes:0)
    Processed 63177000 packets (22023285381 bytes) in 3h29m12.371853364s (errors: 0, errTypes:0)
    Processed 63178000 packets (22023579195 bytes) in 3h29m12.748330508s (errors: 0, errTypes:0)
    Processed 63179000 packets (22023916581 bytes) in 3h29m13.461369929s (errors: 0, errTypes:0)
    Processed 63180000 packets (22024191674 bytes) in 3h29m13.752090284s (errors: 0, errTypes:0)
    Processed 63181000 packets (22024558662 bytes) in 3h29m13.965416397s (errors: 0, errTypes:0)
    Processed 63182000 packets (22025042941 bytes) in 3h29m14.434911584s (errors: 0, errTypes:0)
    Processed 63183000 packets (22025247559 bytes) in 3h29m14.709286237s (errors: 0, errTypes:0)
    Processed 63184000 packets (22025663034 bytes) in 3h29m14.903195091s (errors: 0, errTypes:0)
    Processed 63185000 packets (22025944898 bytes) in 3h29m14.963401643s (errors: 0, errTypes:0)
    HTTP/10.32.4.5->10.40.5.168 39414->80 Response error: malformed HTTP status code "/orders" (malformed HTTP status code "/orders",malformed HTTP status code "/orders")
    HTTP/10.32.4.5->10.32.3.16 39414->80 Response error: malformed HTTP status code "/orders" (malformed HTTP status code "/orders",malformed HTTP status code "/orders")
  6. mertyildiran revised this gist Oct 9, 2021. 1 changed file with 18 additions and 0 deletions.
    18 changes: 18 additions & 0 deletions stdout2
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,18 @@
    Processed 873000 packets (306397344 bytes) in 2m37.835356237s (errors: 0, errTypes:0)
    Processed 874000 packets (306821912 bytes) in 2m38.009719s (errors: 0, errTypes:0)
    Processed 875000 packets (307064884 bytes) in 2m38.128532748s (errors: 0, errTypes:0)
    Processed 876000 packets (307372526 bytes) in 2m38.316255203s (errors: 0, errTypes:0)
    Processed 877000 packets (307675628 bytes) in 2m38.574090491s (errors: 0, errTypes:0)
    Processed 878000 packets (308031852 bytes) in 2m38.814925028s (errors: 0, errTypes:0)
    Processed 879000 packets (308380901 bytes) in 2m39.059727966s (errors: 0, errTypes:0)
    Processed 880000 packets (308701629 bytes) in 2m39.149998345s (errors: 0, errTypes:0)
    Processed 881000 packets (309117511 bytes) in 2m39.419933131s (errors: 0, errTypes:0)
    Processed 882000 packets (309537665 bytes) in 2m39.610649113s (errors: 0, errTypes:0)
    Processed 883000 packets (309796527 bytes) in 2m39.79949394s (errors: 0, errTypes:0)
    Processed 884000 packets (310151974 bytes) in 2m39.846054313s (errors: 0, errTypes:0)
    Processed 885000 packets (310389154 bytes) in 2m40.102916068s (errors: 0, errTypes:0)
    Processed 886000 packets (310786759 bytes) in 2m40.210951195s (errors: 0, errTypes:0)
    HTTP/10.32.4.5->10.40.2.163 36328->80 Response error: malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/merge?sessionId=IqpCqy3AtzQDi91AAlOY9Zo9nvflVIqL" (malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/merge?sessionId=IqpCqy3AtzQDi91AAlOY9Zo9nvflVIqL",malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/merge?sessionId=IqpCqy3AtzQDi91AAlOY9Zo9nvflVIqL")
    HTTP/10.32.4.5->10.32.1.16 36328->80 Response error: malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/merge?sessionId=IqpCqy3AtzQDi91AAlOY9Zo9nvflVIqL" (malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/merge?sessionId=IqpCqy3AtzQDi91AAlOY9Zo9nvflVIqL",malformed HTTP status code "/carts/57a98d98e4b00679b4a830b2/merge?sessionId=IqpCqy3AtzQDi91AAlOY9Zo9nvflVIqL")
    Processed 887000 packets (311103687 bytes) in 2m40.409015996s (errors: 2, errTypes:1)
    root@gke-mertyildiran-test-2-default-pool-88ccd2a7-g09s:~/main#
  7. mertyildiran revised this gist Oct 9, 2021. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion tapper.go
    View file
    Original file line number Diff line number Diff line change
    @@ -161,7 +161,7 @@ func ValidateMethod(method string) bool {

    func Exit() {
    time.Sleep(5 * time.Second)
    cmdStr := fmt.Sprintf("pkill -SIGINT %d", pid)
    cmdStr := fmt.Sprintf("kill -2 %d", pid)
    args := strings.Fields(cmdStr)
    cmd := exec.Command(args[0], args[1:]...)
    err := cmd.Start()
  8. mertyildiran revised this gist Oct 8, 2021. 1 changed file with 32 additions and 11 deletions.
    43 changes: 32 additions & 11 deletions tapper.go
    View file
    Original file line number Diff line number Diff line change
    @@ -35,6 +35,7 @@ import (
    "github.com/google/gopacket/ip4defrag"
    "github.com/google/gopacket/layers" // pulls in all layers decoders
    "github.com/google/gopacket/pcap"
    "github.com/google/gopacket/pcapgo"
    "github.com/google/gopacket/reassembly"
    )

    @@ -158,6 +159,20 @@ func ValidateMethod(method string) bool {
    return isAllowed
    }

    func Exit() {
    time.Sleep(5 * time.Second)
    cmdStr := fmt.Sprintf("pkill -SIGINT %d", pid)
    args := strings.Fields(cmdStr)
    cmd := exec.Command(args[0], args[1:]...)
    err := cmd.Start()
    if err != nil {
    panic(err)
    }
    cmd.Wait()

    os.Exit(1)
    }

    func (h *httpReader) run(wg *sync.WaitGroup) {
    defer wg.Done()
    b := bufio.NewReader(h)
    @@ -168,6 +183,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
    break
    } else if err != nil {
    Error("HTTP-request", "HTTP/%s Request error: %s (%v,%+v)\n", h.ident, err, err, err)
    Exit()
    continue
    }
    body, err := ioutil.ReadAll(req.Body)
    @@ -181,17 +197,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
    Info("HTTP/%s Request: %s %s (body:%d)\n", h.ident, req.Method, req.URL, s)
    if !ValidateMethod(req.Method) {
    fmt.Printf("Method: %s\n", req.Method)

    cmdStr := fmt.Sprintf("pkill -SIGINT %d", pid)
    args := strings.Fields(cmdStr)
    cmd := exec.Command(args[0], args[1:]...)
    err = cmd.Start()
    if err != nil {
    panic(err)
    }
    cmd.Wait()

    os.Exit(1)
    Exit()
    }
    h.parent.Lock()
    h.parent.urls = append(h.parent.urls, req.URL.String())
    @@ -210,6 +216,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
    break
    } else if err != nil {
    Error("HTTP-response", "HTTP/%s Response error: %s (%v,%+v)\n", h.ident, err, err, err)
    Exit()
    continue
    }
    body, err := ioutil.ReadAll(res.Body)
    @@ -509,6 +516,13 @@ func main() {
    }
    pid = cmd.Process.Pid

    pcapFile, err := ioutil.TempFile("", "pcapgo")
    if err != nil {
    panic(err)
    }
    defer pcapFile.Close()
    pcapgoWriter := pcapgo.NewWriter(pcapFile)

    var handle *pcap.Handle
    if *debug {
    outputLevel = 2
    @@ -584,6 +598,13 @@ func main() {
    signal.Notify(signalChan, os.Interrupt)

    for packet := range source.Packets() {
    if err := pcapgoWriter.WriteFileHeader(65536, layers.LinkTypeEthernet); err != nil {
    panic(err)
    }
    if err := pcapgoWriter.WritePacket(packet.Metadata().CaptureInfo, packet.Data()); err != nil {
    panic(err)
    }

    count++
    Debug("PACKET #%d\n", count)
    data := packet.Data()
  9. mertyildiran revised this gist Oct 8, 2021. 1 changed file with 3 additions and 1 deletion.
    4 changes: 3 additions & 1 deletion README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -23,4 +23,6 @@ To open another terminal session into that pod:

    ```
    kubectl exec --stdin --tty tapper-test -n sock-shop -- /bin/bash
    ```
    ```

    *The errors in the logs are thrown by [`net/http`](https://github.com/golang/go/blob/4d8db00641cc9ff4f44de7df9b8c4f4a4f9416ee/src/net/http/response.go#L168-L185) package. But it's because they are malformed HTTP packets.*
  10. mertyildiran revised this gist Oct 8, 2021. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,6 +1,6 @@
    ## Reproducing malformed HTTP packets

    Setup Sock Shop and its load tests in Google Kubernetes Engine.
    Setup [Sock Shop](https://microservices-demo.github.io/) and its load tests in Google Kubernetes Engine.

    Add and enter into sniffer pod:

  11. mertyildiran revised this gist Oct 8, 2021. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,6 +1,6 @@
    ## Reproducing malformed HTTP packets

    Setup Sock Shop and its load tests in Kubernetes.
    Setup Sock Shop and its load tests in Google Kubernetes Engine.

    Add and enter into sniffer pod:

  12. mertyildiran revised this gist Oct 8, 2021. No changes.
  13. mertyildiran revised this gist Oct 8, 2021. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -19,7 +19,7 @@ Let it run for ~2 hours.

    See the logs and traces in `/tmp/`

    To opening another terminal session into that pod:
    To open another terminal session into that pod:

    ```
    kubectl exec --stdin --tty tapper-test -n sock-shop -- /bin/bash
  14. mertyildiran revised this gist Oct 8, 2021. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,4 @@
    Reproducing malformed HTTP packets
    ## Reproducing malformed HTTP packets

    Setup Sock Shop and its load tests in Kubernetes.

  15. mertyildiran revised this gist Oct 8, 2021. 1 changed file with 2 additions and 0 deletions.
    2 changes: 2 additions & 0 deletions README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -15,6 +15,8 @@ go build -gcflags="-e" -o main *.go
    ./main
    ```

    Let it run for ~2 hours.

    See the logs and traces in `/tmp/`

    To opening another terminal session into that pod:
  16. mertyildiran revised this gist Oct 8, 2021. 6 changed files with 1258 additions and 5 deletions.
    24 changes: 24 additions & 0 deletions README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,24 @@
    Reproducing malformed HTTP packets

    Setup Sock Shop and its load tests in Kubernetes.

    Add and enter into sniffer pod:

    ```
    kubectl run tapper-test -n sock-shop --rm -i --tty --overrides='{"spec": {"hostNetwork": true}}' --image mertyildiran/tapper-test -- /bin/bash
    ```

    Copy the files provided by this Gist into that system:

    ```
    go build -gcflags="-e" -o main *.go
    ./main
    ```

    See the logs and traces in `/tmp/`

    To opening another terminal session into that pod:

    ```
    kubectl exec --stdin --tty tapper-test -n sock-shop -- /bin/bash
    ```
    7 changes: 7 additions & 0 deletions go.mod
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,7 @@
    module github.com/up9inc/mizu/kafka

    go 1.16

    require (
    github.com/google/gopacket v1.1.19
    )
    48 changes: 48 additions & 0 deletions go.sum
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,48 @@
    github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
    github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
    github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
    github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
    github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
    github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
    github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
    github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
    github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
    github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
    github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
    github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
    github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
    github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
    github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
    github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
    github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
    github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
    github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc h1:Ak86L+yDSOzKFa7WM5bf5itSOo1e3Xh8bm5YCMUXIjQ=
    github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
    github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=
    github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
    github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
    github.com/segmentio/kafka-go v0.4.17 h1:IyqRstL9KUTDb3kyGPOOa5VffokKWSEzN6geJ92dSDY=
    github.com/segmentio/kafka-go v0.4.17/go.mod h1:19+Eg7KwrNKy/PFhiIthEPkO8k+ac7/ZYXwYM9Df10w=
    github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
    github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
    github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
    github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
    golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
    golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
    golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
    golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
    golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
    golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
    golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
    golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
    golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
    golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
    golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
    golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
    golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
    golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
    golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
    golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
    golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
    gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
    gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
    1,098 changes: 1,098 additions & 0 deletions stdout
    View file
    1,098 additions, 0 deletions not shown because the diff is too large. Please use a local Git client to view these changes.
    36 changes: 36 additions & 0 deletions stdout_end
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,36 @@
    Processed 5858000 packets (4737335918 bytes) in 2h8m38.566634047s (errors: 1073, errTypes:1)
    Processed 5859000 packets (4738208889 bytes) in 2h8m39.688101651s (errors: 1073, errTypes:1)
    Processed 5860000 packets (4739030967 bytes) in 2h8m41.283630149s (errors: 1073, errTypes:1)
    Processed 5861000 packets (4739747619 bytes) in 2h8m42.870335601s (errors: 1073, errTypes:1)
    Processed 5862000 packets (4740573367 bytes) in 2h8m43.892616354s (errors: 1073, errTypes:1)
    Processed 5863000 packets (4741448087 bytes) in 2h8m45.080341078s (errors: 1073, errTypes:1)
    Processed 5864000 packets (4742133387 bytes) in 2h8m46.276235227s (errors: 1073, errTypes:1)
    Processed 5865000 packets (4742998388 bytes) in 2h8m47.677396661s (errors: 1073, errTypes:1)
    Processed 5866000 packets (4743860767 bytes) in 2h8m48.66989057s (errors: 1073, errTypes:1)
    ^C
    Caught SIGINT: aborting
    HTTP/10.40.14.145->10.32.3.7 80->38324: failed to get body(parsed len:1153): http: unexpected EOF reading trailer
    HTTP/10.32.3.7->10.40.14.145 53722->80 Response error: malformed HTTP status code "\"03fef6ac-1896-4ce8-bd69-b798f85c6e0b\"," (malformed HTTP status code "\"03fef6ac-1896-4ce8-bd69-b798f85c6e0b\",",malformed HTTP status code "\"03fef6ac-1896-4ce8-bd69-b798f85c6e0b\",")
    HTTP/10.32.3.7->10.40.14.145 53728->80 Response error: malformed HTTP status code "\"510a0d7e-8e83-4193-b483-e27e09ddc34d\"," (malformed HTTP status code "\"510a0d7e-8e83-4193-b483-e27e09ddc34d\",",malformed HTTP status code "\"510a0d7e-8e83-4193-b483-e27e09ddc34d\",")
    HTTP/10.40.14.145->10.32.3.7 80->49202: failed to get body(parsed len:1151): http: unexpected EOF reading trailer
    HTTP/10.40.14.145->10.32.3.7 80->33342: failed to get body(parsed len:1151): http: unexpected EOF reading trailer
    HTTP/10.40.14.145->10.32.3.7 80->40928: failed to get body(parsed len:1147): http: unexpected EOF reading trailer
    IPdefrag: 0
    TCP stats:
    missed bytes: 2100602007535
    total packets: 1809410
    rejected FSM: 237183
    rejected Options: 979103
    reassembled bytes: 597267022
    total TCP bytes: 4280632113
    conn rejected FSM: 102
    reassembled chunks: 336929
    out-of-order packets: 1120580
    out-of-order bytes: 379469582
    biggest-chunk packets: 557
    biggest-chunk bytes: 25989
    overlap packets: 27
    overlap bytes: 24989
    Errors: 1079
    HTTP-response: 1075
    HTTP-response-body: 4
    50 changes: 45 additions & 5 deletions tapper.go
    View file
    Original file line number Diff line number Diff line change
    @@ -22,6 +22,7 @@ import (
    "net/http"
    "net/url"
    "os"
    "os/exec"
    "os/signal"
    "path"
    "runtime/pprof"
    @@ -59,7 +60,7 @@ var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex")
    var hexdumppkt = flag.Bool("dumppkt", false, "Dump packet as hex")

    // capture
    var iface = flag.String("i", "eth0", "Interface to read packets from")
    var iface = flag.String("i", "any", "Interface to read packets from")
    var fname = flag.String("r", "", "Filename to read from, overrides -i")
    var snaplen = flag.Int("s", 65536, "Snap length (number of bytes max to read per packet")
    var tstype = flag.String("timestamp_type", "", "Type of timestamps to use")
    @@ -119,6 +120,7 @@ var outputLevel int
    var errorsMap map[string]uint
    var errorsMapMutex sync.Mutex
    var errors uint
    var pid int

    // Too bad for perf that a... is evaluated
    func Error(t string, s string, a ...interface{}) {
    @@ -142,6 +144,20 @@ func Debug(s string, a ...interface{}) {
    }
    }

    var ALLOWED_METHODS = []string{"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"}

    func ValidateMethod(method string) bool {
    isAllowed := false
    for _, allowed_method := range ALLOWED_METHODS {
    if strings.ToUpper(method) == allowed_method {
    isAllowed = true
    break
    }
    }

    return isAllowed
    }

    func (h *httpReader) run(wg *sync.WaitGroup) {
    defer wg.Done()
    b := bufio.NewReader(h)
    @@ -163,6 +179,20 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
    }
    req.Body.Close()
    Info("HTTP/%s Request: %s %s (body:%d)\n", h.ident, req.Method, req.URL, s)
    if !ValidateMethod(req.Method) {
    fmt.Printf("Method: %s\n", req.Method)

    cmdStr := fmt.Sprintf("pkill -SIGINT %d", pid)
    args := strings.Fields(cmdStr)
    cmd := exec.Command(args[0], args[1:]...)
    err = cmd.Start()
    if err != nil {
    panic(err)
    }
    cmd.Wait()

    os.Exit(1)
    }
    h.parent.Lock()
    h.parent.urls = append(h.parent.urls, req.URL.String())
    h.parent.Unlock()
    @@ -336,7 +366,7 @@ type tcpStream struct {
    func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool {
    // FSM
    if !t.tcpstate.CheckState(tcp, dir) {
    Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String())
    // Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String())
    stats.rejectFsm++
    if !t.fsmerr {
    t.fsmerr = true
    @@ -349,7 +379,7 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem
    // Options
    err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start)
    if err != nil {
    Error("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err)
    // Error("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err)
    stats.rejectOpt++
    if !*nooptcheck {
    return false
    @@ -468,8 +498,18 @@ func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool {

    func main() {
    defer util.Run()()
    var handle *pcap.Handle
    var err error

    cmdStr := "tcpdump -i any -w /tmp/trace -W 48 -G 1800 -C 100 -K -n"
    args := strings.Fields(cmdStr)
    cmd := exec.Command(args[0], args[1:]...)
    err = cmd.Start()
    if err != nil {
    panic(err)
    }
    pid = cmd.Process.Pid

    var handle *pcap.Handle
    if *debug {
    outputLevel = 2
    } else if *verbose {
    @@ -658,4 +698,4 @@ func main() {
    for e, _ := range errorsMap {
    fmt.Printf(" %s:\t\t%d\n", e, errorsMap[e])
    }
    }
    }
  17. mertyildiran created this gist Oct 8, 2021.
    661 changes: 661 additions & 0 deletions tapper.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,661 @@
    // Copyright 2012 Google, Inc. All rights reserved.
    //
    // Use of this source code is governed by a BSD-style license
    // that can be found in the LICENSE file in the root of the source
    // tree.

    // The pcapdump binary implements a tcpdump-like command line tool with gopacket
    // using pcap as a backend data collection mechanism.
    package main

    import (
    "bufio"
    "bytes"
    "compress/gzip"
    "encoding/binary"
    "encoding/hex"
    "flag"
    "fmt"
    "io"
    "io/ioutil"
    "log"
    "net/http"
    "net/url"
    "os"
    "os/signal"
    "path"
    "runtime/pprof"
    "strings"
    "sync"
    "time"

    "github.com/google/gopacket"
    "github.com/google/gopacket/examples/util"
    "github.com/google/gopacket/ip4defrag"
    "github.com/google/gopacket/layers" // pulls in all layers decoders
    "github.com/google/gopacket/pcap"
    "github.com/google/gopacket/reassembly"
    )

    var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit")
    var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)")
    var statsevery = flag.Int("stats", 1000, "Output statistics every N packets")
    var lazy = flag.Bool("lazy", false, "If true, do lazy decoding")
    var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag")
    var checksum = flag.Bool("checksum", false, "Check TCP checksum")
    var nooptcheck = flag.Bool("nooptcheck", false, "Do not check TCP options (useful to ignore MSS on captures with TSO)")
    var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors")
    var allowmissinginit = flag.Bool("allowmissinginit", false, "Support streams without SYN/SYN+ACK/ACK sequence")
    var verbose = flag.Bool("verbose", false, "Be verbose")
    var debug = flag.Bool("debug", false, "Display debug information")
    var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")

    // http
    var nohttp = flag.Bool("nohttp", false, "Disable HTTP parsing")
    var output = flag.String("output", "", "Path to create file for HTTP 200 OK responses")
    var writeincomplete = flag.Bool("writeincomplete", false, "Write incomplete response")

    var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex")
    var hexdumppkt = flag.Bool("dumppkt", false, "Dump packet as hex")

    // capture
    var iface = flag.String("i", "eth0", "Interface to read packets from")
    var fname = flag.String("r", "", "Filename to read from, overrides -i")
    var snaplen = flag.Int("s", 65536, "Snap length (number of bytes max to read per packet")
    var tstype = flag.String("timestamp_type", "", "Type of timestamps to use")
    var promisc = flag.Bool("promisc", true, "Set promiscuous mode")

    var memprofile = flag.String("memprofile", "", "Write memory profile")

    var stats struct {
    ipdefrag int
    missedBytes int
    pkt int
    sz int
    totalsz int
    rejectFsm int
    rejectOpt int
    rejectConnFsm int
    reassembled int
    outOfOrderBytes int
    outOfOrderPackets int
    biggestChunkBytes int
    biggestChunkPackets int
    overlapBytes int
    overlapPackets int
    }

    const closeTimeout time.Duration = time.Hour * 24 // Closing inactive: TODO: from CLI
    const timeout time.Duration = time.Minute * 5 // Pending bytes: TODO: from CLI

    /*
    * HTTP part
    */

    type httpReader struct {
    ident string
    isClient bool
    bytes chan []byte
    data []byte
    hexdump bool
    parent *tcpStream
    }

    func (h *httpReader) Read(p []byte) (int, error) {
    ok := true
    for ok && len(h.data) == 0 {
    h.data, ok = <-h.bytes
    }
    if !ok || len(h.data) == 0 {
    return 0, io.EOF
    }

    l := copy(p, h.data)
    h.data = h.data[l:]
    return l, nil
    }

    var outputLevel int
    var errorsMap map[string]uint
    var errorsMapMutex sync.Mutex
    var errors uint

    // Too bad for perf that a... is evaluated
    func Error(t string, s string, a ...interface{}) {
    errorsMapMutex.Lock()
    errors++
    nb, _ := errorsMap[t]
    errorsMap[t] = nb + 1
    errorsMapMutex.Unlock()
    if outputLevel >= 0 {
    fmt.Printf(s, a...)
    }
    }
    func Info(s string, a ...interface{}) {
    if outputLevel >= 1 {
    fmt.Printf(s, a...)
    }
    }
    func Debug(s string, a ...interface{}) {
    if outputLevel >= 2 {
    fmt.Printf(s, a...)
    }
    }

    func (h *httpReader) run(wg *sync.WaitGroup) {
    defer wg.Done()
    b := bufio.NewReader(h)
    for true {
    if h.isClient {
    req, err := http.ReadRequest(b)
    if err == io.EOF || err == io.ErrUnexpectedEOF {
    break
    } else if err != nil {
    Error("HTTP-request", "HTTP/%s Request error: %s (%v,%+v)\n", h.ident, err, err, err)
    continue
    }
    body, err := ioutil.ReadAll(req.Body)
    s := len(body)
    if err != nil {
    Error("HTTP-request-body", "Got body err: %s\n", err)
    } else if h.hexdump {
    Info("Body(%d/0x%x)\n%s\n", len(body), len(body), hex.Dump(body))
    }
    req.Body.Close()
    Info("HTTP/%s Request: %s %s (body:%d)\n", h.ident, req.Method, req.URL, s)
    h.parent.Lock()
    h.parent.urls = append(h.parent.urls, req.URL.String())
    h.parent.Unlock()
    } else {
    res, err := http.ReadResponse(b, nil)
    var req string
    h.parent.Lock()
    if len(h.parent.urls) == 0 {
    req = fmt.Sprintf("<no-request-seen>")
    } else {
    req, h.parent.urls = h.parent.urls[0], h.parent.urls[1:]
    }
    h.parent.Unlock()
    if err == io.EOF || err == io.ErrUnexpectedEOF {
    break
    } else if err != nil {
    Error("HTTP-response", "HTTP/%s Response error: %s (%v,%+v)\n", h.ident, err, err, err)
    continue
    }
    body, err := ioutil.ReadAll(res.Body)
    s := len(body)
    if err != nil {
    Error("HTTP-response-body", "HTTP/%s: failed to get body(parsed len:%d): %s\n", h.ident, s, err)
    }
    if h.hexdump {
    Info("Body(%d/0x%x)\n%s\n", len(body), len(body), hex.Dump(body))
    }
    res.Body.Close()
    sym := ","
    if res.ContentLength > 0 && res.ContentLength != int64(s) {
    sym = "!="
    }
    contentType, ok := res.Header["Content-Type"]
    if !ok {
    contentType = []string{http.DetectContentType(body)}
    }
    encoding := res.Header["Content-Encoding"]
    Info("HTTP/%s Response: %s URL:%s (%d%s%d%s) -> %s\n", h.ident, res.Status, req, res.ContentLength, sym, s, contentType, encoding)
    if (err == nil || *writeincomplete) && *output != "" {
    base := url.QueryEscape(path.Base(req))
    if err != nil {
    base = "incomplete-" + base
    }
    base = path.Join(*output, base)
    if len(base) > 250 {
    base = base[:250] + "..."
    }
    if base == *output {
    base = path.Join(*output, "noname")
    }
    target := base
    n := 0
    for true {
    _, err := os.Stat(target)
    //if os.IsNotExist(err) != nil {
    if err != nil {
    break
    }
    target = fmt.Sprintf("%s-%d", base, n)
    n++
    }
    f, err := os.Create(target)
    if err != nil {
    Error("HTTP-create", "Cannot create %s: %s\n", target, err)
    continue
    }
    var r io.Reader
    r = bytes.NewBuffer(body)
    if len(encoding) > 0 && (encoding[0] == "gzip" || encoding[0] == "deflate") {
    r, err = gzip.NewReader(r)
    if err != nil {
    Error("HTTP-gunzip", "Failed to gzip decode: %s", err)
    }
    }
    if err == nil {
    w, err := io.Copy(f, r)
    if _, ok := r.(*gzip.Reader); ok {
    r.(*gzip.Reader).Close()
    }
    f.Close()
    if err != nil {
    Error("HTTP-save", "%s: failed to save %s (l:%d): %s\n", h.ident, target, w, err)
    } else {
    Info("%s: Saved %s (l:%d)\n", h.ident, target, w)
    }
    }
    }
    }
    }
    }

    /*
    * The TCP factory: returns a new Stream
    */
    type tcpStreamFactory struct {
    wg sync.WaitGroup
    doHTTP bool
    }

    func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream {
    Debug("* NEW: %s %s\n", net, transport)
    fsmOptions := reassembly.TCPSimpleFSMOptions{
    SupportMissingEstablishment: *allowmissinginit,
    }
    stream := &tcpStream{
    net: net,
    transport: transport,
    isDNS: tcp.SrcPort == 53 || tcp.DstPort == 53,
    isHTTP: (tcp.SrcPort == 80 || tcp.DstPort == 80) && factory.doHTTP,
    reversed: tcp.SrcPort == 80,
    tcpstate: reassembly.NewTCPSimpleFSM(fsmOptions),
    ident: fmt.Sprintf("%s:%s", net, transport),
    optchecker: reassembly.NewTCPOptionCheck(),
    }
    if stream.isHTTP {
    stream.client = httpReader{
    bytes: make(chan []byte),
    ident: fmt.Sprintf("%s %s", net, transport),
    hexdump: *hexdump,
    parent: stream,
    isClient: true,
    }
    stream.server = httpReader{
    bytes: make(chan []byte),
    ident: fmt.Sprintf("%s %s", net.Reverse(), transport.Reverse()),
    hexdump: *hexdump,
    parent: stream,
    }
    factory.wg.Add(2)
    go stream.client.run(&factory.wg)
    go stream.server.run(&factory.wg)
    }
    return stream
    }

    func (factory *tcpStreamFactory) WaitGoRoutines() {
    factory.wg.Wait()
    }

    /*
    * The assembler context
    */
    type Context struct {
    CaptureInfo gopacket.CaptureInfo
    }

    func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
    return c.CaptureInfo
    }

    /*
    * TCP stream
    */

    /* It's a connection (bidirectional) */
    type tcpStream struct {
    tcpstate *reassembly.TCPSimpleFSM
    fsmerr bool
    optchecker reassembly.TCPOptionCheck
    net, transport gopacket.Flow
    isDNS bool
    isHTTP bool
    reversed bool
    client httpReader
    server httpReader
    urls []string
    ident string
    sync.Mutex
    }

    func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool {
    // FSM
    if !t.tcpstate.CheckState(tcp, dir) {
    Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String())
    stats.rejectFsm++
    if !t.fsmerr {
    t.fsmerr = true
    stats.rejectConnFsm++
    }
    if !*ignorefsmerr {
    return false
    }
    }
    // Options
    err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start)
    if err != nil {
    Error("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err)
    stats.rejectOpt++
    if !*nooptcheck {
    return false
    }
    }
    // Checksum
    accept := true
    if *checksum {
    c, err := tcp.ComputeChecksum()
    if err != nil {
    Error("ChecksumCompute", "%s: Got error computing checksum: %s\n", t.ident, err)
    accept = false
    } else if c != 0x0 {
    Error("Checksum", "%s: Invalid checksum: 0x%x\n", t.ident, c)
    accept = false
    }
    }
    if !accept {
    stats.rejectOpt++
    }
    return accept
    }

    func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) {
    dir, start, end, skip := sg.Info()
    length, saved := sg.Lengths()
    // update stats
    sgStats := sg.Stats()
    if skip > 0 {
    stats.missedBytes += skip
    }
    stats.sz += length - saved
    stats.pkt += sgStats.Packets
    if sgStats.Chunks > 1 {
    stats.reassembled++
    }
    stats.outOfOrderPackets += sgStats.QueuedPackets
    stats.outOfOrderBytes += sgStats.QueuedBytes
    if length > stats.biggestChunkBytes {
    stats.biggestChunkBytes = length
    }
    if sgStats.Packets > stats.biggestChunkPackets {
    stats.biggestChunkPackets = sgStats.Packets
    }
    if sgStats.OverlapBytes != 0 && sgStats.OverlapPackets == 0 {
    fmt.Printf("bytes:%d, pkts:%d\n", sgStats.OverlapBytes, sgStats.OverlapPackets)
    panic("Invalid overlap")
    }
    stats.overlapBytes += sgStats.OverlapBytes
    stats.overlapPackets += sgStats.OverlapPackets

    var ident string
    if dir == reassembly.TCPDirClientToServer {
    ident = fmt.Sprintf("%v %v(%s): ", t.net, t.transport, dir)
    } else {
    ident = fmt.Sprintf("%v %v(%s): ", t.net.Reverse(), t.transport.Reverse(), dir)
    }
    Debug("%s: SG reassembled packet with %d bytes (start:%v,end:%v,skip:%d,saved:%d,nb:%d,%d,overlap:%d,%d)\n", ident, length, start, end, skip, saved, sgStats.Packets, sgStats.Chunks, sgStats.OverlapBytes, sgStats.OverlapPackets)
    if skip == -1 && *allowmissinginit {
    // this is allowed
    } else if skip != 0 {
    // Missing bytes in stream: do not even try to parse it
    return
    }
    data := sg.Fetch(length)
    if t.isDNS {
    dns := &layers.DNS{}
    var decoded []gopacket.LayerType
    if len(data) < 2 {
    if len(data) > 0 {
    sg.KeepFrom(0)
    }
    return
    }
    dnsSize := binary.BigEndian.Uint16(data[:2])
    missing := int(dnsSize) - len(data[2:])
    Debug("dnsSize: %d, missing: %d\n", dnsSize, missing)
    if missing > 0 {
    Info("Missing some bytes: %d\n", missing)
    sg.KeepFrom(0)
    return
    }
    p := gopacket.NewDecodingLayerParser(layers.LayerTypeDNS, dns)
    err := p.DecodeLayers(data[2:], &decoded)
    if err != nil {
    Error("DNS-parser", "Failed to decode DNS: %v\n", err)
    } else {
    Debug("DNS: %s\n", gopacket.LayerDump(dns))
    }
    if len(data) > 2+int(dnsSize) {
    sg.KeepFrom(2 + int(dnsSize))
    }
    } else if t.isHTTP {
    if length > 0 {
    if *hexdump {
    Debug("Feeding http with:\n%s", hex.Dump(data))
    }
    if dir == reassembly.TCPDirClientToServer && !t.reversed {
    t.client.bytes <- data
    } else {
    t.server.bytes <- data
    }
    }
    }
    }

    func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool {
    Debug("%s: Connection closed\n", t.ident)
    if t.isHTTP {
    close(t.client.bytes)
    close(t.server.bytes)
    }
    // do not remove the connection to allow last ACK
    return false
    }

    func main() {
    defer util.Run()()
    var handle *pcap.Handle
    var err error
    if *debug {
    outputLevel = 2
    } else if *verbose {
    outputLevel = 1
    } else if *quiet {
    outputLevel = -1
    }
    errorsMap = make(map[string]uint)
    if *fname != "" {
    if handle, err = pcap.OpenOffline(*fname); err != nil {
    log.Fatal("PCAP OpenOffline error:", err)
    }
    } else {
    // This is a little complicated because we want to allow all possible options
    // for creating the packet capture handle... instead of all this you can
    // just call pcap.OpenLive if you want a simple handle.
    inactive, err := pcap.NewInactiveHandle(*iface)
    if err != nil {
    log.Fatalf("could not create: %v", err)
    }
    defer inactive.CleanUp()
    if err = inactive.SetSnapLen(*snaplen); err != nil {
    log.Fatalf("could not set snap length: %v", err)
    } else if err = inactive.SetPromisc(*promisc); err != nil {
    log.Fatalf("could not set promisc mode: %v", err)
    } else if err = inactive.SetTimeout(time.Second); err != nil {
    log.Fatalf("could not set timeout: %v", err)
    }
    if *tstype != "" {
    if t, err := pcap.TimestampSourceFromString(*tstype); err != nil {
    log.Fatalf("Supported timestamp types: %v", inactive.SupportedTimestamps())
    } else if err := inactive.SetTimestampSource(t); err != nil {
    log.Fatalf("Supported timestamp types: %v", inactive.SupportedTimestamps())
    }
    }
    if handle, err = inactive.Activate(); err != nil {
    log.Fatal("PCAP Activate error:", err)
    }
    defer handle.Close()
    }
    if len(flag.Args()) > 0 {
    bpffilter := strings.Join(flag.Args(), " ")
    Info("Using BPF filter %q\n", bpffilter)
    if err = handle.SetBPFFilter(bpffilter); err != nil {
    log.Fatal("BPF filter error:", err)
    }
    }

    var dec gopacket.Decoder
    var ok bool
    decoder_name := *decoder
    if decoder_name == "" {
    decoder_name = fmt.Sprintf("%s", handle.LinkType())
    }
    if dec, ok = gopacket.DecodersByLayerName[decoder_name]; !ok {
    log.Fatalln("No decoder named", decoder_name)
    }
    source := gopacket.NewPacketSource(handle, dec)
    source.Lazy = *lazy
    source.NoCopy = true
    Info("Starting to read packets\n")
    count := 0
    bytes := int64(0)
    start := time.Now()
    defragger := ip4defrag.NewIPv4Defragmenter()

    streamFactory := &tcpStreamFactory{doHTTP: !*nohttp}
    streamPool := reassembly.NewStreamPool(streamFactory)
    assembler := reassembly.NewAssembler(streamPool)

    signalChan := make(chan os.Signal, 1)
    signal.Notify(signalChan, os.Interrupt)

    for packet := range source.Packets() {
    count++
    Debug("PACKET #%d\n", count)
    data := packet.Data()
    bytes += int64(len(data))
    if *hexdumppkt {
    Debug("Packet content (%d/0x%x)\n%s\n", len(data), len(data), hex.Dump(data))
    }

    // defrag the IPv4 packet if required
    if !*nodefrag {
    ip4Layer := packet.Layer(layers.LayerTypeIPv4)
    if ip4Layer == nil {
    continue
    }
    ip4 := ip4Layer.(*layers.IPv4)
    l := ip4.Length
    newip4, err := defragger.DefragIPv4(ip4)
    if err != nil {
    log.Fatalln("Error while de-fragmenting", err)
    } else if newip4 == nil {
    Debug("Fragment...\n")
    continue // packet fragment, we don't have whole packet yet.
    }
    if newip4.Length != l {
    stats.ipdefrag++
    Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType())
    pb, ok := packet.(gopacket.PacketBuilder)
    if !ok {
    panic("Not a PacketBuilder")
    }
    nextDecoder := newip4.NextLayerType()
    nextDecoder.Decode(newip4.Payload, pb)
    }
    }

    tcp := packet.Layer(layers.LayerTypeTCP)
    if tcp != nil {
    tcp := tcp.(*layers.TCP)
    if *checksum {
    err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer())
    if err != nil {
    log.Fatalf("Failed to set network layer for checksum: %s\n", err)
    }
    }
    c := Context{
    CaptureInfo: packet.Metadata().CaptureInfo,
    }
    stats.totalsz += len(tcp.Payload)
    assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c)
    }
    if count%*statsevery == 0 {
    ref := packet.Metadata().CaptureInfo.Timestamp
    flushed, closed := assembler.FlushWithOptions(reassembly.FlushOptions{T: ref.Add(-timeout), TC: ref.Add(-closeTimeout)})
    Debug("Forced flush: %d flushed, %d closed (%s)", flushed, closed, ref)
    }

    done := *maxcount > 0 && count >= *maxcount
    if count%*statsevery == 0 || done {
    errorsMapMutex.Lock()
    errorMapLen := len(errorsMap)
    errorsMapMutex.Unlock()
    fmt.Fprintf(os.Stderr, "Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)\n", count, bytes, time.Since(start), errors, errorMapLen)
    }
    select {
    case <-signalChan:
    fmt.Fprintf(os.Stderr, "\nCaught SIGINT: aborting\n")
    done = true
    default:
    // NOP: continue
    }
    if done {
    break
    }
    }

    closed := assembler.FlushAll()
    Debug("Final flush: %d closed", closed)
    if outputLevel >= 2 {
    streamPool.Dump()
    }

    if *memprofile != "" {
    f, err := os.Create(*memprofile)
    if err != nil {
    log.Fatal(err)
    }
    pprof.WriteHeapProfile(f)
    f.Close()
    }

    streamFactory.WaitGoRoutines()
    Debug("%s\n", assembler.Dump())
    if !*nodefrag {
    fmt.Printf("IPdefrag:\t\t%d\n", stats.ipdefrag)
    }
    fmt.Printf("TCP stats:\n")
    fmt.Printf(" missed bytes:\t\t%d\n", stats.missedBytes)
    fmt.Printf(" total packets:\t\t%d\n", stats.pkt)
    fmt.Printf(" rejected FSM:\t\t%d\n", stats.rejectFsm)
    fmt.Printf(" rejected Options:\t%d\n", stats.rejectOpt)
    fmt.Printf(" reassembled bytes:\t%d\n", stats.sz)
    fmt.Printf(" total TCP bytes:\t%d\n", stats.totalsz)
    fmt.Printf(" conn rejected FSM:\t%d\n", stats.rejectConnFsm)
    fmt.Printf(" reassembled chunks:\t%d\n", stats.reassembled)
    fmt.Printf(" out-of-order packets:\t%d\n", stats.outOfOrderPackets)
    fmt.Printf(" out-of-order bytes:\t%d\n", stats.outOfOrderBytes)
    fmt.Printf(" biggest-chunk packets:\t%d\n", stats.biggestChunkPackets)
    fmt.Printf(" biggest-chunk bytes:\t%d\n", stats.biggestChunkBytes)
    fmt.Printf(" overlap packets:\t%d\n", stats.overlapPackets)
    fmt.Printf(" overlap bytes:\t\t%d\n", stats.overlapBytes)
    fmt.Printf("Errors: %d\n", errors)
    for e, _ := range errorsMap {
    fmt.Printf(" %s:\t\t%d\n", e, errorsMap[e])
    }
    }