Anna Geller anna-geller
🎯
anna-geller
/ 03_dashboards.py
Created
October 25, 2021 18:13
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import prefect | |
| from prefect import task, Flow | |
| from prefect.storage import GitHub | |
| from prefect.run_configs import LocalRun | |
| FLOW_NAME = "03_dashboards" | |
| STORAGE = GitHub( | |
| repo="anna-geller/flow-of-flows", | |
| path=f"flows/{FLOW_NAME}.py", | |
| access_token_secret="GITHUB_ACCESS_TOKEN", |
anna-geller
/ 04_orchestrating_flow.py
Last active
November 12, 2021 18:28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from prefect import Flow | |
| from prefect.storage import GitHub | |
| from prefect.tasks.prefect import create_flow_run, wait_for_flow_run | |
| from prefect.run_configs import LocalRun | |
| FLOW_NAME = "04_orchestrating_flow" | |
| STORAGE = GitHub( | |
| repo="anna-geller/flow-of-flows", | |
| path=f"flows/{FLOW_NAME}.py", | |
| access_token_secret="GITHUB_ACCESS_TOKEN", |
anna-geller
/ 04_orchestrating_flow_StartFlowRun.py
Created
October 25, 2021 18:37
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from prefect import Flow | |
| from prefect.storage import GitHub | |
| from prefect.tasks.prefect import StartFlowRun | |
| from prefect.run_configs import LocalRun | |
| FLOW_NAME = "04_orchestrating_flow" | |
| STORAGE = GitHub( | |
| repo="anna-geller/flow-of-flows", | |
| path=f"flows/{FLOW_NAME}.py", | |
| access_token_secret="GITHUB_ACCESS_TOKEN", |
anna-geller
/ postgres_task_example.py
Created
November 2, 2021 09:33
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from prefect import Flow, Parameter, task | |
| from prefect.tasks.postgres import PostgresExecute | |
| from prefect.tasks.secrets import PrefectSecret | |
| postgres_execute = PostgresExecute( | |
| db_name="abc", user="peter", host="my_host", port=5432, | |
| ) | |
| @task |
anna-geller
/ 01_demo_data.py
Created
November 8, 2021 23:27
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import requests | |
| url = ( | |
| "https://min-api.cryptocompare.com/data/pricemulti?fsyms=BTC,ETH,REP,DASH&tsyms=USD" | |
| ) | |
| r = requests.get(url) | |
| data = r.json() | |
| print(data) |
anna-geller
/ demo_data.json
Created
November 8, 2021 23:28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "BTC": {"USD": 62289.47}, | |
| "ETH": {"USD": 4600.19}, | |
| "REP": {"USD": 24.92}, | |
| "DASH": {"USD": 192.05} | |
| } |
anna-geller
/ 02_demo_data.py
Created
November 8, 2021 23:28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| btc = data['BTC']['USD'] | |
| eth = data['ETH']['USD'] | |
| rep = data['REP']['USD'] | |
| dash = data['DASH']['USD'] |
anna-geller
/ 03_boto3_ts.py
Created
November 8, 2021 23:29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import boto3 | |
| write_client = boto3.client("timestream-write") | |
| db_arn = write_client.create_database(DatabaseName="demo") | |
| tbl_arn = write_client.create_table( | |
| DatabaseName="demo", | |
| TableName="data", | |
| RetentionProperties={ | |
| "MemoryStoreRetentionPeriodInHours": 1, | |
| "MagneticStoreRetentionPeriodInDays": 365, |
anna-geller
/ 04_records_ts.py
Created
November 8, 2021 23:29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| records = [ | |
| { | |
| "Time": now, | |
| "TimeUnit": "MILLISECONDS", | |
| "Dimensions": [{"Name": "crypto", "Value": "BTC"}], | |
| "MeasureName": "Price", | |
| "MeasureValue": str(btc), | |
| "MeasureValueType": "DOUBLE", | |
| }, | |
| { |
anna-geller
/ 05_boto3_ts.py
Created
November 8, 2021 23:30
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import boto3 | |
| write_client = boto3.client("timestream-write") | |
| rejected_records = write_client.write_records( | |
| DatabaseName="demo", TableName="data", Records=records, CommonAttributes={} | |
| ) | |
| print(rejected_records) |