gistfile1.txt

[nkodner@c1 ~]$ sh -x measure_awk_performance.sh flat_community_cumulative_week_fct1.out.gz.bak
+ FILENAME=flat_community_cumulative_week_fct1.out.gz.bak
+ ls -ltrh /var/opt/sports_dw/live/nz_sports/done/flat_community_cumulative_week_fct1.out.gz.bak
-rwxrwxrwx 1 dwsports dwsports 4.6G Jul 21 10:16 /var/opt/sports_dw/live/nz_sports/done/flat_community_cumulative_week_fct1.out.gz.bak
+ echo 'zcat > /dev/null'
zcat > /dev/null
+ zcat /var/opt/sports_dw/live/nz_sports/done/flat_community_cumulative_week_fct1.out.gz.bak

real    3m49.037s
user    3m44.276s
sys     0m3.901s
+ echo 'zcat | wc -l'
zcat | wc -l
+ zcat /var/opt/sports_dw/live/nz_sports/done/flat_community_cumulative_week_fct1.out.gz.bak
+ wc -l
399665406

real    4m42.093s
user    4m19.686s
sys     0m30.836s
+ echo 'using pipe and tee'
using pipe and tee
+ mkfifo f
mkfifo: cannot create fifo `f': File exists
+ wc -l
+ zcat /var/opt/sports_dw/live/nz_sports/done/flat_community_cumulative_week_fct1.out.gz.bak
+ tee f

real    5m0.181s
user    3m50.145s
sys     0m55.174s
+ cat num_rows.out
399665406
+ [[ -f f ]]
+ echo ''

+ echo ''

+ echo 'awk method'
awk method
+ zcat /var/opt/sports_dw/live/nz_sports/done/flat_community_cumulative_week_fct1.out.gz.bak
+ awk '{print $0} END {print NR > "num_rows.out"}'

real    4m30.879s
user    6m23.139s
sys     1m12.458s
+ cat num_rows.out
399665406