proger

#
# cookbook filename: hist.awk
#
function max(arr, big)
{
    big = 0;
    for (i in user)
    {
        if (user[i] > big) { big=user[i];}
    }

proger

"""
parallel prefix scan

Up-down algorithm illustration: https://www.cs.princeton.edu/courses/archive/fall13/cos326/lec/23-parallel-scan.pdf
Up-down algorithm and recurrence reductions by Guy Blelloch: https://www.cs.cmu.edu/~guyb/papers/Ble93.pdf

See also:
Generic Functional Parallel Algorithms: Scan and FFT by Conal Elliott
http://conal.net/papers/generic-parallel-functional/generic-parallel-functional.pdf
"""

proger

  1805215   stud76       gpu      vae PD Tomorr 00:53 Tomorr 02:53   2:00:00      1     (Resources)     15321
  1805216   stud76       gpu      vae PD Tomorr 00:53 Tomorr 02:53   2:00:00      1      (Priority)     15321
  1805217   stud76       gpu      vae PD Tomorr 00:53 Tomorr 02:53   2:00:00      1      (Priority)     15321
  1805218   stud76       gpu      vae PD Tomorr 00:53 Tomorr 02:53   2:00:00      1      (Priority)     15321
  1805219   stud76       gpu      vae PD Tomorr 00:53 Tomorr 02:53   2:00:00      1      (Priority)     15321
  1805220   stud76       gpu      vae PD Tomorr 00:53 Tomorr 02:53   2:00:00      1      (Priority)     15321
  1805240   stud76       gpu      vae PD Tomorr 08:53 Tomorr 10:53   2:00:00      1      (Priority)     15319
  1805241   stud76       gpu      vae PD Tomorr 08:53 Tomorr 10:53   2:00:00      1      (Priority)     15319
  1805242   stud76       gpu      vae PD Tomorr 08:53 Tomorr 10:53   2:00:00      1      (Priority)     15319
  1805243   stud76       gpu      

proger

import math
import torch
import torch.nn as nn
from fast_pytorch_kmeans import KMeans
import numpy as np # seeding np.random.choice inside KMeans


#
# XXX: forward algorithm must be computed with double precision for stable convergence
#

proger

from fastapi import FastAPI, File, UploadFile
from fastapi.responses import StreamingResponse, HTMLResponse
from pydub.silence import split_on_silence
from pydub import AudioSegment
import random
import string

app = FastAPI()

@app.post("/upload")

proger

import argparse
import json

import ftfy
import sentencepiece as spm


parser = argparse.ArgumentParser()
parser.add_argument('--vocab_size', type=int, default=50304, help='vocab size')
parser.add_argument('--model', type=str, default='out.model', help='model name')

proger

"""
This module runs batches of requests concurrently.
Each batch is sent over one connection using HTTP/1.1 Pipelining.

To send one batch, use tokenize_batch(batch, batch_size) where a batch is a list of strings.
batch_size must be <= 100, this coincides with a limit of requests per one connection of api_nlp_uk.

>>> response_batch = await tokenize_batch(line_batch, len(line_batch))

See tokenize_file() for a concurrent usage scenario.

proger

# saves the dataset to a binary file for training. following was helpful:
# https://github.com/HazyResearch/flash-attention/blob/main/training/src/datamodules/language_modeling_hf.py
# prepare.py in karpathy/nanoGPT

import numpy as np
import sentencepiece as spm
from datasets import load_dataset, Value, Features
from datasets import disable_caching
import random
from tqdm import tqdm

proger

--- /Users/proger/nlp_uk/100k_test_nlp_uk_nonwu.txt	2022-10-15 17:44:52.000000000 +0200
+++ 100k_test_choppa.txt	2022-10-15 17:37:28.000000000 +0200
@@ -106,8 +106,7 @@
 — Стривай, стривай, — спиняє Беймбета Бондаренко, який нічого не може второпати. 
 — Який "конный милиционер"? 
 А що таке "левосерый"? 
-— Слідчий так казав, так писав; суд казав — "левосерый", десять років давав... 
-— розводить руками Беймбет і знову повторює: — Я ні в кінній, ні в пішій міліції не служив. 
+— Слідчий так казав, так писав; суд казав — "левосерый", десять років давав... — розводить руками Беймбет і знову повторює: — Я ні в кінній, ні в пішій міліції не служив. 
 Бачачи, що з уст Кунанбаєва нічого не добереш, Бондаренко каже: — Доведеться до трудила [7] йти — розпитати про твою справу, бо щось ти не так кажеш. 

proger

bench-experiment-dirty-AccurateSrxTextIterator-1024 1033.50s 44639
bench-experiment-dirty-AccurateSrxTextIterator-768 806.57s 45076
bench-experiment-dirty-AccurateSrxTextIterator-512 549.58s 17863
bench-main-dirty-SrxTextIterator-1024 361.15s 44257
bench-pull-4-gb9fa368-dirty-SrxTextIterator-1024 349.77s 44257
bench-experiment-dirty-SrxTextIterator-1024 347.87s 44257
bench-experiment-dirty-AccurateSrxTextIterator-256 288.40s 1
bench-pull-4-gb9fa368-dirty-SrxTextIterator-768 267.55s 44257
bench-main-dirty-SrxTextIterator-768 266.79s 44257
bench-experiment-dirty-SrxTextIterator-768 262.96s 44257