skliarpawlo

from datasets import load_dataset
import transformers
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer

import ray
from ray import tune, air
from ray.train.huggingface import HuggingFaceTrainer
from ray.air.config import ScalingConfig
import os

skliarpawlo

;;
;; Matcher should recognize and destruct URL by:
;; host: domain
;; path: parts, splitted with "/"
;; queryparam: name/value pairs of query
;; (see examples below)
;;
;; Each string that is started from "?" is a "bind"
;; (recognize matcher) should return nil or seq of binds
;;

skliarpawlo

============================= test session starts ==============================
platform darwin -- Python 3.5.2, pytest-3.2.3, py-1.7.0, pluggy-0.4.0 -- /usr/local/bin/python3.5
cachedir: ../.cache
rootdir: /private/var/tmp/_bazel_pavloskliar/4ebdf5e551906a3d6bd81a9029c30c48/execroot/__main__/bazel-out/darwin-fastbuild/bin/tool_castor/castor_integration_tests.runfiles/__main__, inifile: tox.ini
collecting ... INFO:dd.datadogpy:No agent or invalid configuration file found
collected 63 items

tests/integration/test_activities_merger.py::TestActivitiesMergerWorkflow::test_abort_with_corrupted_delta INFO:root:Starting the new global session for <class 'castor.TestCastorSession'>
Warning: Ignoring non-spark config property: es.nodes.wan.only=true
https://nexus.tubularlabs.net/repository/libs-release-local/ added as a remote repository with the name: repo-1

skliarpawlo

...
  emacs:
    build:
      dockerfile: /Users/psk/dev/dotfiles/ide/Dockerfile
      context: /Users/psk/dev/dotfiles/ide/
      args:
        - SUBDIR=tool-castor
    environment:
      - DISPLAY=$DISPLAY
      - SUBDIR=tool-castor

skliarpawlo

docker run --name mysql-test -e MYSQL_DATABASE='tube_test' -e MYSQL_USER='jenkins' -e MYSQL_PASSWORD='jenkins' -e MYSQL_ALLOW_EMPTY_PASSWORD='yes' -d mysql:5.5
docker run --name redis-test -d redis
docker run --name cassandra-test -m 500M --memory-reservation 100M --memory-swap 100M -e "MAX_HEAP_SIZE=300M" -e "HEAP_NEWSIZE=100M" -d cassandra
docker run --name rabbitmq-test -d rabbitmq:3.6.2-management

# docker rm $(docker ps -a -q)
# docker rmi $(docker images | grep "^<none>" | awk "{print $3}")
# docker volume rm $(docker volume ls -qf dangling=true)

skliarpawlo

#!/bin/bash

# $1 - session name
# $2 - num of processes
# $3 - cmd

tmux new-session -s $1 -d
for i in `seq $2` 
do
   tmux select-pane -t $1.0

skliarpawlo

#!/usr/bin/env python3
import argparse
import os

from pyspark import SparkContext, SparkConf
from pyspark.sql import HiveContext

args_parser = argparse.ArgumentParser('YT Activities join')
args_parser.add_argument(
    '--drain',

skliarpawlo

from itertools import product


def generate_from_pattern(pattern, open_sep='{{', close_sep='}}'):
    """ Generates all possible variations that fits pattern.

    For example:
    >>> generate_from_pattern('number {{9-11}}')
    ['number 9', 'number 10', 'number 11']
    >>> generate_from_pattern('number {{1-3,11}}')

skliarpawlo

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import sys


EMO = os.environ.get('TUBULAR_COMMIT_EMOJI', '👮 ')
SHORT_DESCRIPTION_LINE_REGEX = re.compile('^((?:[A-Z]+-\d+)|FIX|HOTFIX|IMP) (.{1,50})$')

skliarpawlo

def _get_json_from_string(s):
    """Parses first found json object from string (piece of html).

    >>> _get_json_from_string('{"a": "b"}')
    {u'a': u'b'}
    >>> _get_json_from_string('dskljasd{"a": "b"}asdkljasd')
    {u'a': u'b'}
    >>> _get_json_from_string('require(asd=[{"a": "b"}, {"c":"d"}]spamspamspam)')
    [{u'a': u'b'}, {u'c': u'd'}]
    >>> _get_json_from_string('{"a": [1,2,{"b":"d[1,2,3]sd"}]}')