zkan

import random

from airflow.operators.dummy_operator import DummyOperator
from airflow.models import DAG
from airflow.utils.dates import days_ago

with DAG(
   "big_dag",
   start_date=days_ago(1),
   schedule_interval=None,

zkan

# -*- coding: utf-8 -*-
from airflow.operators.http_operator import SimpleHttpOperator
from airflow.operators.postgres_operator import PostgresOperator
from airflow.operators.dummy_operator import DummyOperator

from airflow.hooks.postgres_hook import PostgresHook

from airflow.models import Variable, DAG

from datetime import date, datetime, timedelta

zkan

from airflow.models import Variable

# Common (Not-so-nice way)
# 3 DB connections when the file is parsed
var1 = Variable.get("var1")
var2 = Variable.get("var2")
var3 = Variable.get("var3")

# Recommended Way
# Just 1 Database call

zkan

# Instructions
# 1. Write down Python code in say() method
#    - This method might have one parameter `number` which possibly 1-99
#    - This method need to return an English word for provided number e.g. If the number is 3, should return "three"
# 2. More examples on test data (numbers and words) are in the "Test Data Example" section
# 3. How to run this file through unit tests
#    - Open the terminal
#    - Run this command "python <path/to/file.py>" e.g. python PyConUS2021Challenge.py
#    - If the tests are failed, it will display "FAILED (failures=16)" for example, on the terminal
#    - If the tests are pass, it will display "Ran 16 tests in 0.001s OK"

zkan

admin > connections > +

conn_type = s3
conn_name = local_minio

Extra: a JSON object with the following properties: 
 {
    "aws_access_key_id":"your_minio_access_key",
    "aws_secret_access_key": "your_minio_secret_key",
    "host": "http://127.0.0.1:9000"

zkan

FROM jupyter/pyspark-notebook
USER root
# Add essential packages
RUN apt-get update && apt-get install -y build-essential curl git gnupg2 nano apt-transport-https software-properties-common
# Set locale
RUN apt-get update && apt-get install -y locales \
    && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
    && locale-gen
# Add config to Jupyter notebook
COPY jupyter/jupyter_notebook_config.py /home/jovyan/.jupyter/

zkan

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "PublicReadGetObject",
            "Effect": "Allow",
            "Principal": "*",
            "Action": [
                "s3:GetObject"
            ],

zkan

import duckdb
from google.cloud import bigquery

bqclient = bigquery.Client()
table = bigquery.TableReference.from_string(
    "bigquery-public-data.utility_us.country_code_iso"
)
rows = bqclient.list_rows(table)
country_code_iso = rows.to_arrow(create_bqstorage_client=True)
cursor = duckdb.connect()

zkan

layout	title	description	tags
default	SQL Style Guide	A guide to writing clean, clear, and consistent SQL.	data process

Purpose

zkan

MongoDB Shell Commands Cheat Sheet.

This is a Cheat Sheet for interacting with the Mongo Shell ( mongo on your command line). This is for MongoDB Community Edition.

Preface:

Mongo Manual can help you with getting started using the Shell.

FAQ for MongoDB Fundamentals and other FAQs can be found in the side-bar after visiting that link.