Skip to content

Instantly share code, notes, and snippets.

@datherra

datherra/looping_accesslogs.sh

Created May 25, 2012 18:15
Show Gist options
  • Save datherra/2789612 to your computer and use it in GitHub Desktop.
Save datherra/2789612 to your computer and use it in GitHub Desktop.
Download ZIP
looping through some access_logs
Raw
looping_accesslogs.sh
#!/usr/bin/sh
for sistema in sistema1 sistema2 sistema3 sistema4 sistema5 ; do
for host in `ls -d $sistema/sne*`; do
# for host in `ls -d sne*`; do
for file in `ls $host`; do
echo $host/$file | egrep -q gz$
if [ $? -eq 0 ] ; then
gzcat $host/$file| awk '{if ($1 ~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/) {
sub(/\[/,"",$4)
sub(/\"/,"",$6)
$6 = substr($6,1,7)
sub(/\"/,"",$7)
sub(/\:/,"",$7)
sub(/-/,"0",$10)
print $4","$6",\""$7"\","$9","$10}
else {
sub(/\[/,"",$5)
sub(/\"/,"",$8)
$8 = substr($8,1,7)
sub(/\"/,"",$9)
sub(/\:/,"",$9)
sub(/-/,"0",$12)
print $5","$8",\""$9"\","$11","$12}
}' \
| grep "/2012" | awk '{ FS=":" ; print $1 " " $2 ":"$3":"$4 }' \
| host_sistema=`echo $host|tr "/" ","` \
| awk -v host_sistema=$host_sistema '{ host_sistema=toupper(host_sistema); print host_sistema","$0}' >> loader_source.txt
continue
fi
cat $host/$file | awk '{if ($1 ~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/) {
sub(/\[/,"",$4)
sub(/\"/,"",$6)
$6 = substr($6,1,7)
sub(/\"/,"",$7)
sub(/\:/,"",$7)
sub(/-/,"0",$10)
print $4","$6",\""$7"\","$9","$10}
else {
sub(/\[/,"",$5)
sub(/\"/,"",$8)
$8 = substr($8,1,7)
sub(/\"/,"",$9)
sub(/\:/,"",$9)
sub(/-/,"0",$12)
print $5","$8",\""$9"\","$11","$12}
}' \
| grep "/2012" | awk '{ FS=":" ; print $1 " " $2 ":"$3":"$4 }' \
| host_sistema=`echo $host|tr "/" ","` \
| awk -v host_sistema=$host_sistema '{ host_sistema=toupper(host_sistema); print host_sistema","$0}' >> loader_source.txt
done
done
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment