This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from google.cloud import bigquery | |
client = bigquery.Client() | |
# returns a Query Job object | |
# https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.job.QueryJob.html#google.cloud.bigquery.job.QueryJob | |
query = client.query( | |
""" | |
SELECT * FROM ... |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import logging | |
import json | |
import apache_beam as beam | |
from apache_beam.options.pipeline_options import PipelineOptions, StandardOptions | |
# pub/sub subscription in the format: projects/<your-project>/subscriptions/<subscription-name> | |
sub = '' | |
# bigquery table in the format: project:dataset.table |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from datetime import datetime | |
import logging | |
import random | |
from apache_beam import DoFn, GroupByKey, io, ParDo, Pipeline, PTransform, WindowInto, WithKeys | |
from apache_beam.options.pipeline_options import PipelineOptions | |
from apache_beam.transforms.window import FixedWindows | |
# your pub/sub topic, format: projects/<your-project-name>/topics/<your-topic-name> | |
input_topic = '' |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import re | |
import apache_beam as beam | |
from apache_beam.options.pipeline_options import PipelineOptions | |
# Input and output files should be Cloud Storage locations | |
# beginning with gs:// | |
input_file = 'gs://<your-bucket-name>/kinglear.txt' | |
output_path = 'gs://<your-bucket-name>/counts.txt' | |
# Replace <your-bucket-name>, <your-project-name> and <your-sa-email> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from google.cloud import bigquery | |
dataset = "btlearningpath" | |
table = "city_speeds_from_function" | |
def add_to_bq(data, context): | |
source_bucket = data['bucket'] | |
source_file = data['name'] | |
uri = 'gs://{}/{}'.format(source_bucket, source_file) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
### BigQuery schema | |
country:STRING, | |
country_code:STRING, | |
region:STRING, | |
region_code:STRING, | |
city:STRING, | |
date:DATE, | |
download_kbps:FLOAT, | |
upload_kbps:FLOAT, |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import csv | |
from google.cloud import bigquery | |
from time import sleep | |
from random import random | |
client = bigquery.Client() | |
table_id = "ab-academy-demo.btlearningpath.city_speeds_bq_stream" | |
filename = "city_daily_speeds-10k.csv" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import csv | |
import json | |
from concurrent import futures | |
from google.cloud import pubsub_v1 | |
from typing import Callable | |
from time import sleep | |
from random import random | |
project_id = "ab-academy-demo" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$('textarea').keyup(function() { | |
var characterCount = $(this).val().length, | |
current = $('#current'), | |
maximum = $('#maximum'), | |
theCount = $('#the-count'); | |
current.text(characterCount); | |
NewerOlder