Terraform code to configure GCS bucket usage logs and send them to BigQuery

gcs-usage-logs-transfer-bigquery.tf

# Terraform code to configure GCS bucket usage logs and send them to BigQuery.
#
# References:
# * https://cloud.google.com/storage/docs/access-logs
# * https://cloud.google.com/bigquery-transfer/docs/cloud-storage-transfer
#
# Notes:
# * both resources must reside on same location
# * cloud_storage_usage_schema_v0.json comes from http://storage.googleapis.com/pub/cloud_storage_usage_schema_v0.json

resource "google_storage_bucket" "foo" {
  name     = "foo"
  location = "EU"
}

resource "google_storage_bucket" "foo-logs" {
  name     = "foo_logs"
  location = "EU"
}

resource "google_storage_bucket_iam_member" "foo-logs-gcsa" {
  bucket = "${google_storage_bucket.foo-logs.name}"
  role   = "roles/storage.legacyBucketWriter"
  member = "group:[email protected]"
}

resource "google_bigquery_dataset" "foo-logs" {
  dataset_id = "foo-logs"
  location   = "EU"
}

resource "google_bigquery_table" "foo-logs-usage" {
  dataset_id = "${google_bigquery_dataset.foo-logs.dataset_id}"
  table_id   = "usage_logs"

  schema = "${file("cloud_storage_usage_schema_v0.json")}"
}

resource "google_bigquery_data_transfer_config" "foo-logs-usage" {
  display_name           = "$foo-logs-usage transfer"
  location               = "EU"
  data_source_id         = "google_cloud_storage"
  destination_dataset_id = "${google_bigquery_dataset.foo-logs.dataset_id}"
  schedule               = "every 1 hours"

  params = {
    data_path_template              = "gs://${google_storage_bucket.foo-logs.name}/*_usage_*"
    destination_table_name_template = "log_usage"
    file_format                     = "CSV"
    skip_leading_rows               = "1"
    delete_source_files             = "true" # remove them to save costs, keep them for later re-import
  }
}