mauliksoneji

class BigQueryClient(object):

    def __init__(self, project_id):
        self.project_id = project_id

    def _get_conf(self, bucket, dataset_id, table_id):
        return {
            "fs.gs.project.id": self.project_id,
            "mapred.bq.project.id": self.project_id,  # default project
            "mapred.bq.gcs.bucket": bucket,  # gcs bucket holding the temperory path

stefanthoss

import json
from pyspark.sql.types import *

# Define the schema
schema = StructType(
    [StructField("name", StringType(), True), StructField("age", IntegerType(), True)]
)

# Write the schema
with open("schema.json", "w") as f: