fclesio

import os
import boto3

COMPUTE_ENVIRONMENT_VERSION = 'v0_01'

aws_iam_boto_client \
    = boto3.client('iam')

aws_batch_boto_client \
    = boto3.client('batch')

fclesio

readinessProbe:
  exec:
    command:
    - python
    - check_integrity.py
  initialDelaySeconds: 60
  periodSeconds: 900
  timeoutSeconds: 10
  successThreshold: 1
  failureThreshold: 3

fclesio

import hashlib
import sys
from module import download_model_locally

def check_model_md5(model_name, api_production_artifact_md5):
  """Ref: https://stackoverflow.com/a/21565932/"""
  
  """This function will download the model """
  download_model_locally_to(model_name)
    

fclesio

apiVersion: v1
kind: Pod
metadata:
  labels:
    test: liveness-ml-application
  name: liveness-ml-application
spec:
  containers:
  - name: ml_application
    image: internal_registry/ml_application

fclesio

apiVersion: v1
kind: Pod
metadata:
  labels:
    test: liveness-ml-application
  name: liveness-ml-application
spec:
  containers:
  - name: ml_application
    image: internal_registry/ml_application

fclesio

docker build -f dags/docker_job/Dockerfile -t docker_image_task . && \
docker-compose up -d

fclesio

docker-compose up airflow-init

fclesio

from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.operators.docker_operator import DockerOperator
from airflow.operators.python_operator import BranchPythonOperator
from airflow.operators.dummy_operator import DummyOperator

default_args = {
'owner'                 : 'airflow',
'description'           : 'Use of the DockerOperator',

fclesio

from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.operators.docker_operator import DockerOperator
from airflow.operators.python_operator import BranchPythonOperator
from airflow.operators.dummy_operator import DummyOperator

default_args = {
'owner'                 : 'airflow',
'description'           : 'Use of the DockerOperator',

fclesio

import pandas as pd
import fasttext

df = pd.read_csv('B2W-Reviews01.csv', sep=';')

df['label'] = '__label__' + df['site_category_lv1']

df = df.sample(n=10000)

df[['label', 'review_text']].to_csv('gfjakhsdgkfjhgsd.txt', index=None, header=None)