edvardm

### resource/common.just
# std set of rules for all projects. Should be overridden in
# project-specific justfiles, preferably using same signature if possible
# (think this as an interface in programming languages). 
# Basically it means that if a base rule doesn't accept parameters, 
# overridden version should also do without those unless necessary

# DISCLAIMER: it's probably quite hard to end up with general, valid "signature" for all rules
# without making it insufficient for some users and unnecessary complex for others, so maybe
# rule names themselves and their precense could be considered to be more strict than 

edvardm

module Main where

import           Control.Applicative (liftA2)
import           Data.Char           (toLower, toUpper)
import           Data.Function       (on)
import qualified Data.Map.Strict     as Map
import           Text.Parsec
import           Text.Parsec.String  (Parser)

data Color

edvardm

from typing import Any, TypeVar, Callable, Generator
from contextlib import contextmanager

T = TypeVar("T")


class BufferedOp:
    """Class for adding elements to a _buffer and flushing when full.

    Note that last batch may is not usually flushed automatically as usually

edvardm

# Credits to answer in https://stackoverflow.com/questions/53173927/pandas-extensive-describe-include-count-the-null-values

def stats(df: pd.DataFrame) -> pd.DataFrame:
    """Like describe() but return dataframe, with datatypes and ratio of NaN values"""
    st = df.describe()
    total_n = len(df)

    st.loc["dtype"] = df.dtypes
    st.loc["n"] = total_n
    st.loc["nan_n"] = total_n - df.count()  # number of NaNs

edvardm

# Using sample data from https://www.gov.uk/government/statistical-data-sets/price-paid-data-downloads
# See https://wiki.postgresql.org/wiki/Sample_Databases for table schema & instructions how to populate

# Simple script to test the idea of keeping Queue full of database rows ready for consumption, using appropriate-sized
# chunks of rows

import multiprocessing as mp
import queue

import psycopg2

edvardm

import time
import sys

def exp_backoff_retry(
    fun=None,
    *,
    retry_exceptions=(BaseException,),
    max_retries=3,
    exp_base=2.0,
    max_seconds=300,

edvardm

from contracts import contract, new_contract
import pytest
from contracts.interface import ContractNotRespected
import json

# There are probably bugs, but this should check that given
# argument is a valid object which can be encoded as json.
#
# It assumes the following Python structure, if type hints
# could be recursive:

edvardm

SELECT h.schema_name,
     h.table_name,
     h.id AS table_id,
     h.associated_table_prefix,
     ((round(row_estimate.row_estimate::numeric / 1000000, 3))::text || 'M') as estimated_rows
    FROM _timescaledb_catalog.hypertable h
      CROSS JOIN LATERAL ( SELECT sum(cl.reltuples) AS row_estimate
            FROM _timescaledb_catalog.chunk c
            JOIN pg_class cl ON cl.relname = c.table_name
            WHERE c.hypertable_id = h.id

edvardm

#!/usr/bin/env bash

  set -eu
  PYTHON_VERSION=3.7.5
  POETRY_URL=https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py

  has() {
      type $1 > /dev/null 2>&1
  }

edvardm

#!/usr/bin/env bash

  set -eu
  PYTHON_VERSION=3.7.5
  POETRY_URL=https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py

  has() {
      type $1 > /dev/null 2>&1
  }