mehd-io

Define the custom function in the dag :

sc = SlackClient(my_slack_token)

def slack_failed_task(contextDictionary, **kwargs):
        base_url = mybasedUrl
        task_instance = contextDictionary.get('task_instance')
        log_url = '{task_instance.log_url}'.format(**locals())
 failed_alert = sc.api_call('chat.postMessage',

mehd-io

# List hadoop conf while running a pyspark job

hadoopConf = {}
iterator = sc._jsc.hadoopConfiguration().iterator()

while iterator.hasNext():
    prop = iterator.next()
    hadoopConf[prop.getKey()] = prop.getValue()

for item in sorted(hadoopConf.items()):

mehd-io

{
	"name": "Python 3.8",
	"build": {
		"dockerfile": "../docker/dev.Dockerfile",
		"context": "..",
		// Update 'VARIANT' to pick a Python version. Rebuild the container 
		// if it already exists to update. Available variants: 3, 3.6, 3.7, 3.8 
		"args": { "VARIANT": "3.8" }
	},

mehd-io

#!/bin/bash
# Simple bash script to generate version based on hash of files or git command
ROOT=$(git rev-parse --show-toplevel)

OS=$(uname -s)
if [ "$OS" = "Linux" ]; then
  md5_cmd="md5sum
elif [ "$OS" = "Darwin" ]; then
  md5_cmd="md5 -r"

mehd-io

get-img:
	@echo 🐳 $@
	@if [ "$$DOCKER_LAYER" != "base" ]; then\
		echo 🔨 Pulling/Building base image first...; \
		$(MAKE) pull-img DOCKER_LAYER=base || $(MAKE) _build-img DOCKER_LAYER=base;\
	 fi;\
	 $(MAKE) pull-img || $(MAKE) _build-img 
	@if [ "$$PUSH_IMAGE" = "true" ]; then\
		echo ✨ Pushing the images to docker registry...; \
		$(MAKE) push-img DOCKER_LAYER=base; \

mehd-io

      - name: Prepare CI image
        run: |
          make get-img DOCKER_LAYER=dev PUSH_IMAGE=true

      - name: Test
        run: |
          make test

      - name: Build & Push app image 
        run: |

mehd-io

import pytest
import tftest
from pathlib import Path


@pytest.fixture
def plan():
    file_path = Path(__file__).resolve()
    base_dir = file_path.parent.parent.parent.absolute()
    tf = tftest.TerraformTest(tfdir="terraform", basedir=base_dir)

mehd-io

import json

import pytest
from google.auth.transport.requests import AuthorizedSession

from .cloud_run_client import request_wrapper


@pytest.mark.integration
def test_health_check(auth_session: AuthorizedSession):

mehd-io

from pathlib import Path

import pytest
import tftest
from google.auth.transport.requests import AuthorizedSession

from .cloud_run_client import (
    get_auth_session,
    get_service_account_file_path,
    get_token_credentials_from_service_account,

mehd-io

#!/bin/bash
# You can put this in your .bashrc or .zshrc
function csv_to_parquet() {
    file_path="$1"
    duckdb -c "COPY (SELECT * FROM read_csv_auto('$file_path')) TO '${file_path%.*}.parquet' (FORMAT PARQUET);"
}