nehiljain

from fireworks.client import Fireworks
import os
from dotenv import load_dotenv
import random
import json
from pydantic import BaseModel, Field
from enum import Enum

class DateTimeCategory(str, Enum):
    BASIC_OPERATIONS = "Basic Operations"

nehiljain

Home

Search in docs... K

Home

Docs Dashboard

Your API Key

nehiljain

import instructor
import base64
import logging
import os
from openai import AsyncOpenAI, OpenAI
import asyncio
from pydantic import Field, BaseModel
from typing import List
from dotenv import find_dotenv, load_dotenv

nehiljain

#! /usr/bin/env sh


DIR=$(dirname "$0")
cd "$DIR"
echo $DIR

[ -d temp-dco ] && rm -rf temp-dco
git clone https://github.com/nehiljain/data-cost-optimizer.git temp-dco
cd "$DIR/temp-dco"

nehiljain

Outline

https://pretalx.com/apache-airflow-summit-bay-area-2020/talk/review/Q3WNKPGR7LYYNMZTGLBSJTSC9NXAKBJ7

Introduction

Common problems we face

* Custom code for data source. 1:1 mappings for source to destinations
* ETLs bottleneck, not fast as biz landscape changes are
* Depletion o f trust. Wrong decisions made, bad data in production

nehiljain

'''This is the extension on the alembic API to facilitate the custom autogenerate script'''
import json
import logging
import os

import boto3

from alembic.autogenerate import comparators, renderers
from alembic.operations import MigrateOperation, Operations
from alembic.operations.ops import CreateTableOp, ModifyTableOps

nehiljain

Project Title

One Paragraph of project description goes here

Getting Started

These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.

Prerequisites

nehiljain

files:
  "/etc/td-agent/td-agent.conf":
    owner: root
    group: root
    content: |
      <source>
        @type tail
        path /var/log/eb-docker/containers/log.file
        exclude_path ["/var/log/eb-docker/containers/exception.file"]
        pos_file /var/log/td-agent/fluentd.log.pos

nehiljain

provider "aws" {
  region = "us-east-1"
}

resource "aws_s3_bucket" "pipeline_bucket" {
  bucket        = "pipeline-logs"
  acl           = "private"
  force_destroy = false
}

nehiljain

from datetime import datetime, timedelta
from airflow.models import DAG
from airflow.operators.s3_key_sensor import S3KeySensor
from airflow.operators.python_operator import PythonOperator
from airflow.utils.dates import days_ago
schedule = timedelta(minutes=5)
args = {
 'owner': 'airflow',
 'start_date': days_ago(1),
 'depends_on_past': False,