BigQuery + Icecast2 logs

icecast2-bq-table-schema

[  { "type": "TIMESTAMP", "name": "time" },
   { "type": "STRING",    "name": "host" },
   { "type": "STRING",    "name": "user" },
   { "type": "STRING",    "name": "method" },
   { "type": "STRING",    "name": "path" },
   { "type": "STRING",    "name": "code" },
   { "type": "INTEGER",   "name": "size" },
   { "type": "STRING",    "name": "referer" },
   { "type": "STRING",    "name": "agent" },
   { "type": "STRING",    "name": "duration" } ]

icecast2-td-agent.conf

<source>
  @type tail
  # Need this to load older logs
  read_from_head true
  path /var/log/icecast/access.log
  pos_file /var/log/td-agent/icecast2.access_log.pos
  <parse>
    @type regexp
    expression ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")? (?<duration>[^ ]*)$
    time_format %d/%b/%Y:%H:%M:%S %z
  </parse>
  tag bq.icecast2.access
</source>

<match bq.icecast2.access>
  @type bigquery_insert
  # Authenticate with BigQuery using the VM's service account, this run inside GCP vm
  auth_method compute_engine
  project [MY_PROJECT_NAME]
  dataset fluentd
  table icecast2
  fetch_schema true
  <inject>
    # Convert fluentd timestamp into TIMESTAMP string
    time_key time
    time_type string
    time_format %Y-%m-%dT%H:%M:%S.%NZ
  </inject>
</match>