GoogleCloudStorageToBigQueryOperatorのサンプル

GoogleCloudStorageToBigQueryOperator_sample.py

import os
import datetime

from airflow import DAG
from airflow.contrib.operators.gcs_to_bq import GoogleCloudStorageToBigQueryOperator

default_args = {
    "start_date": datetime.datetime(2018, 1, 1),
}

ENV = os.environ["ENV"]

with DAG(
        "sample_dag",
        schedule_interval=None,
        catchup=False,
        default_args=default_args) as dag:


    task = GoogleCloudStorageToBigQueryOperator(
      task_id="load_person",
      location="asia-northeast1",
      write_disposition="WRITE_TRUNCATE",
      source_format="CSV",
      bucket="sample-bucket",
      source_objects=[f"{table_name}.csv.gz"],
      destination_project_dataset_table=f"momota.{table_name}",
      # schema_object= f"{table_name}.json", # schema_fieldsの代替としてjsonファイルの利用も可能
      schema_fields = [
        {"description":"id", "name":"id", "type":"string"},
        {"description":"名前", "name":"name", "type":"string"},
        {"description":"誕生日", "name":"birthday", "type":"datetime"}
      ]
    )