Skip to content

Instantly share code, notes, and snippets.

@nmicic
Last active August 16, 2022 21:52
Show Gist options
  • Select an option

  • Save nmicic/d9f4b704f39d67a1e6ddaed33c06bbe7 to your computer and use it in GitHub Desktop.

Select an option

Save nmicic/d9f4b704f39d67a1e6ddaed33c06bbe7 to your computer and use it in GitHub Desktop.
process_pcaps_vlan3000+vlan_2000_RTT_issue
#!/bin/sh
FILE=$1
tshark -nr $FILE -q -z io,stat,1,"COUNT(tcp)tcp","COUNT(tcp.analysis.retransmission)tcp.analysis.retransmission","COUNT(tcp.analysis.fast_retransmission)tcp.analysis.fast_retransmission","COUNT(tcp.analysis.spurious_retransmission)tcp.analysis.spurious_retransmission","COUNT(tcp.analysis.duplicate_ack)tcp.analysis.duplicate_ack","COUNT(tcp.analysis.lost_segment)tcp.analysis.lost_segment","COUNT(tcp.analysis.ack_lost_segment)tcp.analysis.ack_lost_segment","COUNT(tcp.analysis.rto)tcp.analysis.rto","COUNT(tcp.analysis.out_of_order)tcp.analysis.out_of_order","MIN(tcp.window_size)tcp.window_size","AVG(tcp.window_size)tcp.window_size","MAX(tcp.window_size)","COUNT(tcp.analysis.window_full)tcp.analysis.window_full","COUNT(tcp.analysis.zero_window)tcp.analysis.zero_window","MIN(tcp.analysis.ack_rtt)tcp.analysis.ack_rtt","AVG(tcp.analysis.ack_rtt)tcp.analysis.ack_rtt","MAX(tcp.analysis.ack_rtt)tcp.analysis.ack_rtt","MIN(tcp.analysis.initial_rtt)tcp.analysis.initial_rtt","AVG(tcp.analysis.initial_rtt)tcp.analysis.initial_rtt","MAX(tcp.analysis.initial_rtt)tcp.analysis.initial_rtt","AVG(tcp.analysis.rto)tcp.analysis.rto","MAX(tcp.analysis.bytes_in_flight)tcp.analysis.bytes_in_flight" > "$FILE"_CUSP.txt
#!/bin/bash
mkdir TXT
for i in re*pcap*;do ./tshark2tcp_fields.sh $i;done
mv *txt TXT
mergecap -w rennes.vlan2000.pcap rennes.cisco.rtt.vlan2000.*.pcap*
mergecap -w rennes.vlan3000.pcap rennes.cisco.rtt.vlan3000.*.pcap*
editcap -C 20:36 rennes.vlan3000.pcap rennes.vlan3000_gtp_stripped.pcap
tshark -nr rennes.vlan3000_gtp_stripped.pcap -Y"ip.addr==100.88.0.0/13 || ip.addr==100.80.0.0/13 || ip.addr==100.72.0.0/13 || ip.addr==100.64.0.0/13" -w rennes.vlan3000_gtp_stripped_sgiwappsp.pcap
./tshark2tcp_fields.sh rennes.vlan2000.pcap
./tshark2tcp_fields.sh rennes.vlan3000_gtp_stripped_sgiwappsp.pcap
mergecap -w rennes.cisco.rtt.merged_ALL_striped.pcap rennes.vlan2000.pcap rennes.vlan3000_gtp_stripped_sgiwappsp.pcap
BEGIN=$( cat begin_pcap.*.rennes.vlan2000.pcap begin_pcap.*.rennes.vlan3000_gtp_stripped_sgiwappsp.pcap | cut -f1 -d_ | sort -n | tail -1 )
END=$( cat end_pcap.*.rennes.vlan2000.pcap end_pcap.*.rennes.vlan3000_gtp_stripped_sgiwappsp.pcap | cut -f1 -d_ | sort -n | head -1 )
tshark -nr rennes.cisco.rtt.merged_ALL_striped.pcap -Y"frame.time_epoch>$BEGIN && frame.time_epoch<$END" -s65535 -w rennes.cisco.rtt.merged_ALL_time_sync.pcap
./tshark2tcp_fields.sh rennes.cisco.rtt.merged_ALL_time_sync.pcap
ln -s rennes.cisco.rtt.merged_ALL_time_sync.pcap_tcp.txt rennes.cisco.rtt.merged_ALL_striped.Fields_New.txt
./run.sh
./RTT_avg.sh rennes.vlan2000.pcap
./RTT_avg.sh rennes.vlan3000_gtp_stripped_sgiwappsp.pcap
./CUSP.sh rennes.vlan2000.pcap
./CUSP.sh rennes.vlan3000_gtp_stripped_sgiwappsp.pcap
#!/bin/bash
FILE=$1
tshark -nr $FILE -q -z io,stat,10,"COUNT(tcp)tcp","AVG(tcp.analysis.ack_rtt)tcp.analysis.ack_rtt","AVG(tcp.analysis.initial_rtt)tcp.analysis.initial_rtt" > "$FILE"_RTT_avg10.txt
#!/bin/bash
PCAPTXTFILE="rennes.cisco.rtt.merged_ALL_striped.Fields_New.txt"
### Please don't change below.
FLOWID="$PCAPTXTFILE"_flowid.txt
DELTAFILE="delta_time9.txt"
cat "$PCAPTXTFILE" | awk -F'_' 'BEGIN{id=0} {key=$4"_"$5"_"$6"_"$7"_"$8"_"$9"_"$10"_"$11"_"$12"_"$13"_"$14;if(flowid[key]==""){id++;flowid[key]=id};print $1" " $2 " " $3 " " key " " flowid[key] " " NF}' | sort -n -k 5,2 > "$FLOWID"
cat "$FLOWID" | sort -n -k 5 > "$FLOWID"_sort-n-k5
cat "$FLOWID"_sort-n-k5 | awk '{key=$4;if(prev4==$4 && seen[key]!=key ){seen[key]=key;delta=sprintf("%f",$1-prev1);print delta" "prev1" "$1" "prev2" "$2" "prev3" "$3" "prev4" "$4" "$5"==="};prev1=$1;prev2=$2;prev3=$3;prev4=$4}' > delta_time9.txt
cat delta_time9.txt | awk '{if($6 < $7 ){hund=sprintf("%d",(($1*1000)/100));second=sprintf("%d",$2-1654864387.757585000);csv[second,hund]++;allsec[second]=second;allhund[hund]=hund;if(debug==1){print hund/10 " === " $1 " === " second " === " csv[second,hund]}}} END{printf "bucket100ms/second,";for(j in allhund){printf "%f,",j/10;};print "";for(i in allsec){printf "%s,",i;for(j in allhund){printf("%d,",csv[i,j])}print ""} }' > SFR_RTT_downlink_vlan3000_to_vlan2000.csv
cat delta_time9.txt | awk '{if($6 < $7 ){delay=substr($1,0,3);second=sprintf("%d",$2-1654864387.757585000);csv[second,delay]++;allsec[second]=second;alldelay[delay]=delay;if(debug==1){print delay " === " $1 " === " second " === " csv[second,delay]}}} END{printf "bucket100ms/second,";for(j in alldelay){printf "%f,",j;};print "";for(i in allsec){printf "%s,",i;for(j in alldelay){printf("%d,",csv[i,j])}print ""} }' > SFR_RTT_downlink_vlan3000_to_vlan2000_100ms_buckets_per_second.csv
cat delta_time9.txt | awk '{if($6 < $7 ){delay=substr($1,0,4);second=sprintf("%d",$2-1654864387.757585000);csv[second,delay]++;allsec[second]=second;alldelay[delay]=delay;if(debug==1){print delay " === " $1 " === " second " === " csv[second,delay]}}} END{printf "bucket10ms/second,";for(j in alldelay){printf "%f,",j;};print "";for(i in allsec){printf "%s,",i;for(j in alldelay){printf("%d,",csv[i,j])}print ""} }' > SFR_RTT_downlink_vlan3000_to_vlan2000_10ms_buckets_per_second.csv
cat "$FLOWID"_sort-n-k5 | awk '{key=$5;flow[key]=$0;count[key]++} END{for(i in count){if((count[i]%2)>0){print flow[i]" "count[i]}}}' > drops_full_all_out.txt
cat "$FLOWID"_sort-n-k5 | awk '{print $4}' | sort | uniq -c | sort -n > seen_number_per_pkt_key.txt
cat seen_number_per_pkt_key.txt | awk '{print $1}' | sort -n | uniq -c > drops-summary.txt
cat seen_number_per_pkt_key.txt | awk '{if($1==2){print $0}}' > not_drops.txt
cat seen_number_per_pkt_key.txt | awk '{if($1==1){print $0}}' > drops_confirmed.txt
cat drops_confirmed.txt | awk '{print $2}' | awk -F"_" '{print $1}' | sort | uniq -c | sort -n -r > drops_top_source_ip.txt
cat drops_confirmed.txt | awk '{print $2}' | awk -F"_" '{print $2}' | sort | uniq -c | sort -n -r > drops_top_destin_ip.txt
cat drops_confirmed.txt | awk '{print $2}' | awk -F"_" '{print $1" "$2}' | sort | uniq -c | sort -n -r > drops_top_ips_src_dst.txt
cat drops_full_all_out.txt | awk '{if($3==3000){print $0}}' | cut -f1 -d. | uniq -c > drops_per_second_vlan3000.txt
cat drops_full_all_out.txt | awk '{if($3==2000){print $0}}' | cut -f1 -d. | uniq -c > drops_per_second_vlan2000.txt
cat drops_full_all_out.txt | cut -f1 -d. | uniq -c > drops_per_second.txt
cat drops_full_all_out.txt | awk '{print $3}' | sort | uniq -c > drops_per_vlan.txt
# The following below are useful without "| wc -l"to actyally see which packets are delayed above some value:
cat delta_time9.txt | awk '{if($1<0.001 && $6 < $7 ){print $0}}' | wc -l > delta_time9.txt_summary_delay_less_then_0.001s.txt
cat delta_time9.txt | awk '{if($1<0.001 && $6 < $7 ){print $0}}' > delta_time9.txt_pkts_delay_less_then_0.001s.txt
cat delta_time9.txt | awk '{if($1>0.001 && $6 < $7 ){print $0}}' | wc -l > delta_time9.txt_summary_delay_more_then_0.001.txt
cat delta_time9.txt | awk '{if($1>0.001 && $6 < $7 ){print $0}}' > delta_time9.txt_pkts_delay_more_then_0.001.txt
cat delta_time9.txt | awk '{if($1>0.01 && $6 < $7 ){print $0}}' | wc -l > delta_time9.txt_summary_delay_more_then_0.01.txt
cat delta_time9.txt | awk '{if($1>0.01 && $6 < $7 ){print $0}}' > delta_time9.txt_pkts_delay_more_then_0.01.txt
cat delta_time9.txt | awk '{if($1>0.1 && $6 < $7 ){print $0}}' | wc -l > delta_time9.txt_summary_delay_more_then_0.1.txt
cat delta_time9.txt | awk '{if($1>0.1 && $6 < $7 ){print $0}}' > delta_time9.txt_pkts_delay_more_then_0.1.txt
cat delta_time9.txt | awk '{if($1>0.2 && $6 < $7 ){print $0}}' | wc -l > delta_time9.txt_summary_delay_more_then_0.2.txt
cat delta_time9.txt | awk '{if($1>0.2 && $6 < $7 ){print $0}}' > delta_time9.txt_pkts_delay_more_then_0.2.txt
cat delta_time9.txt | awk '{if($1>0.3 && $6 < $7 ){print $0}}' | wc -l > delta_time9.txt_summary_delay_more_then_0.3.txt
cat delta_time9.txt | awk '{if($1>0.3 && $6 < $7 ){print $0}}' > delta_time9.txt_pkts_delay_more_then_0.3.txt
cat delta_time9.txt | awk '{if($1>0.4 && $6 < $7 ){print $0}}' | wc -l > delta_time9.txt_summary_delay_more_then_0.4.txt
cat delta_time9.txt | awk '{if($1>0.4 && $6 < $7 ){print $0}}' > delta_time9.txt_pkts_delay_more_then_0.4.txt
cat delta_time9.txt | awk '{if($1>0.5 && $6 < $7 ){print $0}}' | wc -l > delta_time9.txt_summary_delay_more_then_0.5.txt
cat delta_time9.txt | awk '{if($1>0.5 && $6 < $7 ){print $0}}' > delta_time9.txt_pkts_delay_more_then_0.5.txt
cat delta_time9.txt | awk '{if($1>1 && $6 < $7 ){print $0}}' | wc -l > delta_time9.txt_summary_delay_more_then_1.txt
cat delta_time9.txt | awk '{if($1>1 && $6 < $7 ){print $0}}' > delta_time9.txt_pkts_delay_more_then_1.txt
cat delta_time9.txt | awk '{if($1>10 && $6 < $7 ){print $0}}' | wc -l > delta_time9.txt_summary_delay_more_then_10.txt
cat delta_time9.txt | awk '{if($1>10 && $6 < $7 ){print $0}}' > delta_time9.txt_pkts_delay_more_then_10.txt
cat delta_time9.txt | awk '{print $8}' | awk -F"_" '{flw=$1"_"$2"_"$3"_"$4;rev=$2"_"$1"_"$4"_"$3;MAP[flw]=rev;FLOW[flw]++;FLOW[rev]++;FLOW[rev]--;TOTAL[flw]=FLOW[flw]+FLOW[rev]} END{for(i in TOTAL){j=MAP[i];print TOTAL[i]" flw:" FLOW[i] " rev:" FLOW[j] " " i}}' | grep -v , > delta_time9.txt_number_of_packets_per_5_tuple_both_directions.txt
cat delta_time9.txt | grep -v , | awk '{delay=$1;bucket=substr(delay,0,3);split($8,key,"_");fwd=key[1]"_"key[2]"_"key[3]"_"key[4];rev=key[2]"_"key[1]"_"key[4]"_"key[3];MAP[fwd]=rev;FLOW[fwd]++;FLOW[rev]++;FLOW[rev]--;TOTAL[fwd]=FLOW[fwd]+FLOW[rev];if(MaxDelay[fwd]<delay){MaxDelay[fwd]=delay}} END{for(i in TOTAL){j=MAP[i];split(i,tuple,"_");print TOTAL[i]" fwd: " FLOW[i] " rev: "FLOW[j] " maxdelay: " MaxDelay[i] " :: " i " :: \"ip.addr=="tuple[1]" && ip.addr=="tuple[2]" && tcp.port=="tuple[3]" && tcp.port=="tuple[4]"\""}}' | sort -n -k 7 -r > max_delay_per_5_tuple.txt
cat "$FLOWID" drops_full_all_out.txt delta_time9.txt | grep -v , | awk 'BEGIN{id=0} {if(NF==6){vlan=$3;flowid=$5;split($4,key,"_");fwd=key[1]"_"key[2]"_"key[4]"_"key[5];PKTVLAN[fwd,vlan]++};if(NF==7){vlan=$3;split($4,key,"_");fwd=key[1]"_"key[2]"_"key[4]"_"key[5];DROPS[fwd]++;DROPVLAN[fwd,vlan]++};if(NF==10 && $6 < $7){delay=$1;bucket=substr(delay,0,3);split($8,key,"_");fwd=key[1]"_"key[2]"_"key[4]"_"key[5];DROPS[fwd]=DROPS[fwd]+0;rev=key[2]"_"key[1]"_"key[5]"_"key[4];if(FLID[fwd]==""&& FLID[rev]==""){FLID[fwd]=id;FLID[rev]=id;id++};MAP[fwd]=rev;FLOW[fwd]++;if(FLOW[rev]==""){FLOW[rev]=0};TOTAL[fwd]=FLOW[fwd]+FLOW[rev];TotalDelay[fwd]=TotalDelay[fwd]+delay;if(MaxDelay[fwd]<delay){MaxDelay[fwd]=delay};syn=key[8];if(syn==1){SYNSTAT[fwd]++}else{SYNSTAT[fwd]=SYNSTAT[fwd]+0}}} END{for(i in TOTAL){j=MAP[i];split(i,tuple,"_");print "total_pkts: "TOTAL[i]" fwd: " FLOW[i] " rev: "FLOW[j] " maxdelay: " MaxDelay[i] " totaldelay: "TotalDelay[i]" :: SynSeen: " SYNSTAT[i]" vlan3000: " PKTVLAN[i,"3000"]" vlan2000: "PKTVLAN[i,"2000"]" delta: "PKTVLAN[i,"3000"]-PKTVLAN[i,"2000"] " drops: " DROPS[i]" , 5tuple: " i " :: \"ip.addr=="tuple[1]" && ip.addr=="tuple[2]" && tcp.port=="tuple[3]" && tcp.port=="tuple[4]"\""}}' | sort -n -k 8 -r > max_total_delay_pkts_stats_per_5_tuple.txt
#1655384036.046224000 65 2000 185.86.138.124_100.64.165.125_0x000050d2_443_53572_3928723957_3076994248_0_1_1821627086_514889170 37 14
cat delta_time9.txt | awk '{delay=substr($1,0,4);print delay}' | sort | uniq -c | sort -n -k 2 > delta_time9.txt_summary_delay_10ms_buckets.txt
cat delta_time9.txt | awk '{delay=substr($1,0,3);print delay}' | sort | uniq -c | sort -n -k 2 > delta_time9.txt_summary_delay_100ms_buckets.txt
cat delta_time9.txt | awk '{if($6 < $7 ){sum=sum+$1;cnt=cnt+1}} END{printf "%f %f %d",sum/cnt,sum,cnt}' > delta_time9.txt_summary_pkts_all.txt
cat delta_time9.txt | awk '{if($1>1 && $6 < $7 ){sum=sum+$1;cnt=cnt+1}} END{printf "%f %f %d",sum/cnt,sum,cnt}' > delta_time9.txt_summary_pkts_more_then_1s.txt
cat delta_time9.txt | awk '{if($1>0.1 && $6 < $7 ){sum=sum+$1;cnt=cnt+1}} END{printf "%f %f %d",sum/cnt,sum,cnt}' > delta_time9.txt_summary_pkts_more_then_0.1.txt
cat delta_time9.txt | awk '{if($1>0.1 && $1<1 && $6 < $7 ){sum=sum+$1;cnt=cnt+1}} END{printf "%f %f %d",sum/cnt,sum,cnt}' > delta_time9.txt_summary_pkts_more_then_0.1_less_1s.txt
cat delta_time9.txt | awk '{if($6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{print $4}' | sort | uniq -c | sort -n -r | head -20 > top_ports_100ms_delay_or_more.txt
cat delta_time9.txt | awk '{if($1>0.01 && $6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{print $4}' | sort | uniq -c | sort -n -r > top_ports_10ms_delay_or_more.txt
cat delta_time9.txt | awk '{if($6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{print $4}' | sort | uniq -c | sort -n -r | head -20 > top_ports_all_packets.txt
cat delta_time9.txt | awk '{if($1>0.1 && $6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{if($4=="443"){print $1}}' | awk -F. '{print $1"."$2"."$3".0/24"}' | sort | uniq -c | sort -n -r | head -20 > top_C_classes_port_443.txt
cat delta_time9.txt | awk '{if($1>0.1 && $6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{if($4=="5228"){print $1}}' | awk -F. '{print $1"."$2"."$3".0/24"}' | sort | uniq -c | sort -n -r | head -40 > top_C_classes_port_5228.txt
cat delta_time9.txt | awk '{if($1>0.1 && $6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{if($4=="5222"){print $1}}' | awk -F. '{print $1"."$2"."$3".0/24"}' | sort | uniq -c | sort -n -r | head -40 > top_C_classes_port_5222.txt
cat delta_time9.txt | awk '{if($1>0.1 && $6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{if($4=="5223"){print $1}}' | awk -F. '{print $1"."$2"."$3".0/24"}' | sort | uniq -c | sort -n -r | head -40 > top_C_classes_port_5223.txt
cat delta_time9.txt | awk '{if($6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{print "SYN:"$8" ACK:"$9}' | sort | uniq -c | sort -n -r > distribution_of_SYN_ACK_in_downlink_2000_to_3000_vlan.txt
cat delta_time9.txt | awk '{if($1>0.1 && $6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{print "SYN:"$8" ACK:"$9}' | sort | uniq -c | sort -n -r > distribution_of_SYN_ACK_100ms_delay_or_more.txt
cat delta_time9.txt | awk '{if($1>0.05 && $6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{print "SYN:"$8" ACK:"$9}' | sort | uniq -c | sort -n -r > distribution_of_SYN_ACK_50ms_delay_or_more.txt
cat delta_time9.txt | awk '{if($1>0.01 && $6 < $7 ){print $0}}' | awk -F"2000 3000 " '{print $2}' | awk -F_ '{print "SYN:"$8" ACK:"$9}' | sort | uniq -c | sort -n -r > distribution_of_SYN_ACK_10ms_delay_or_more.txt
grep " 3000 " drops_full_all_out.txt | awk -F_ '{if($2 ~ /^100\./)print $0}' > missing_packets_in_vlan_2000_as_they_are_seen_in_vlan3000_already_traversed_CUPS.txt
grep " 3000 " drops_full_all_out.txt | awk -F_ '{if($2 ~ /^100\./)print $0}' | awk '{print $4}' | awk -F_ '{print "ip.src=="$1" && ip.dst=="$2" && ip.id=="$3" && tcp.srcport=="$4" && tcp.dstport=="$5}' | sort | uniq > missing_packets_in_vlan_2000_as_they_are_seen_in_vlan3000_already_traversed_CUPS_filters.txt
grep " 2000 100." drops_full_all_out.txt | awk '{print $4}' > missing_packets_in_vlan_3000_as_they_are_seen_in_vlan2000_already_traversed_CUPS.txt
grep " 2000 100." drops_full_all_out.txt | awk '{print $4}' | awk -F_ '{print "ip.src=="$1" && ip.dst=="$2" && ip.id=="$3" && tcp.srcport=="$4" && tcp.dstport=="$5}' | sort | uniq > missing_packets_in_vlan_3000_as_they_are_seen_in_vlan2000_already_traversed_CUPS_filters.txt
echo done
#!/bin/bash
FILE=$1
cat /dev/null > "$FILE"_tcp.txt
tshark -Tfields -Eseparator='_' -e frame.time_epoch -e frame.number -e vlan.id -e ip.src -e ip.dst -e tcp.srcport -e tcp.dstport -e ip.id -e tcp.seq -e tcp.ack -e tcp.flags.syn -e tcp.flags.ack -e tcp.options.timestamp.tsval -e tcp.options.timestamp.tsecr -nr "$FILE" -Ytcp -o tcp.relative_sequence_numbers:FALSE > "$FILE"_tcp.txt
head -1 "$FILE"_tcp.txt | cut -f 1 -d';' > begin_pcap.`date +%s`."$FILE"
tail -1 "$FILE"_tcp.txt | cut -f 1 -d';' > end_pcap.`date +%s`."$FILE"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment