YAML config for reading some BigQuery tables

config.yaml

# [required] The GCP project id (not the number). You can find this in the GCP console.
project: grey-sort-challenge

# [required] The type of runner. One of:
# - dataflow (runs on GCP)
# - local (runs on local machine)
runner: dataflow

# The actual tables to copy. Options:
#
# [required] source: The source table in format [PROJECT]:[DATASET].[TABLE]
# [required] target: The target table in format [PROJECT]:[DATASET].[TABLE]
# [optional] numWorkers: The initial number of workers in the Dataflow cluster. Default is 1.
# [optional] maxNumWorkers: The max number of workers in the Dataflow cluster. Default is 3.
# [optional] workerMachineType: The type of the workers. Default is n1-standard-1.
# [optional] zone: The zone where Dataflow cluster will spin up. Default is australia-southeast1-a.
# [optional] detectSchema: Either true or false. Use this as a workaround for complex schemas. Default is true.
# [optional] writeDisposition: Either truncate or append. Default is truncate.
# [optional] targetDatasetLocation: If the target dataset in BigQuery does not exist, use this to specify the location.
copies:
- source: grey-sort-challenge:src_US_dataset.src_US_table
  target: grey-sort-challenge:a_EU_dataset.i_was_copied_from_us_to_eur
  workerMachineType: n1-standard-2
  zone: us-central1-a
  writeDisposition: truncate

- source: grey-sort-challenge:src_EU_dataset.src_EU_table
  target: grey-sort-challenge:a_US_dataset.i_was_copied_from_eu_to_us
  numWorkers: 2
  maxNumWorkers: 5
  zone: europe-north1-a
  writeDisposition: append

- source: grey-sort-challenge:src_EU_dataset.src_EU_table
  target: grey-sort-challenge:a_Tokyo_dataset.i_was_copied_from_eu_to_tokyo
  detectSchema: false