Skip to content

Instantly share code, notes, and snippets.

@a-mpch
Created September 14, 2020 21:02
Show Gist options
  • Save a-mpch/57c221f5ddf5f445c03a1fbb73ebcbb6 to your computer and use it in GitHub Desktop.
Save a-mpch/57c221f5ddf5f445c03a1fbb73ebcbb6 to your computer and use it in GitHub Desktop.
Run Pipeline different runners
if options.view_as(StandardOptions).runner in ("DirectRunner", None):
pubsub_id = None
with beam.Pipeline(options=options) as p:
_run_pipeline(p, pubsub_id, file_processing_options, gcp_project)
elif options.view_as(StandardOptions).runner == "DataflowRunner":
pipeline = beam.Pipeline(options=options)
_run_pipeline(pipeline, pubsub_id, file_processing_options, gcp_project)
pipeline.run()
else:
raise Exception("Unknown runner")
def _run_pipeline(pipeline, pubsub_id, file_options, gcp_project):
print("Here the pipeline")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment