ozancaglayan

from typing import List, Optional, Tuple, Dict

from torch.utils.data import Dataset
from torch.utils.data.sampler import RandomSampler, Sampler


class MultiTaskBatchSampler(Sampler):
    def __init__(self, datasets: Dict[str, Dataset],
                 batch_size: int,
                 epoch_size: Optional[int] = None,

ozancaglayan

--- rawdata-v4-1882.txt	2018-08-13 15:23:10.065617891 +0300
+++ rawdata-v5.txt	2018-08-13 15:18:22.598301871 +0300
@@ -113,8 +113,7 @@
 112;Halil Kuru;_halilkuru;23 Haz 2014;@06melihgokcek adlı kullanıcıya yanıt olarak @06melihgokcek başgan hackerlara terörist dedin. RTE'ye bir rica etde İSİD'e de terör örgütü deyiversin. Akhack ve redhack hangi kategoride?;0 yanıt 0 retweet 0 beğenme
 113;B a t ı n.wav;batinmuzik;23 Haz 2014;Düne kadar Redhack taraftar olanlar,bugün Redhack'in terör örgütü olduğunu söyleyen ve img'i hackleyenlerle gurur duyuyor.Ne döneksiniz.;0 yanıt 0 retweet 0 beğenme
 114;Kızıl;zapatistass;21 Haz 2014;Bir kuruluş kalbi sosyalizm düşmanlığıyla tutuşmuş insanlardan kuruluysa tabiki redhack'a terör örgütü der.;0 yanıt 0 retweet 0 beğenme
-115;Banu Barlas;banubarlass;17 Haz 2014;Flash flash flash Twitter geziye destek veren RedHack siber terör örgütünün adresini sildi http://www.akdenizpress.com/haber/65/twitter-redhacki-sildi ...
-2 retweet 0 beğenme
+115;Banu Barlas;banubarlass;17 Haz 2014;

ozancaglayan

# -*- coding: utf-8 -*-
#
# author: Ozan Caglayan


from collections import OrderedDict

import torch
from torchvision import models
from torchvision.models.vgg import cfg as vgg_cfg

ozancaglayan

###############
# SINFO aliases
###############
# Detailed sinfo
alias si="sinfo -o '%8P %10n %.11T %.4c %.8z %.6m %12G %10l %10L %10O %20E' -S '-P'"
# sinfo only on GPU partition
alias sig="si -p gpu"
# sinfo only on CPU partition
alias sic="si -p cpu"

ozancaglayan

#!/usr/bin/env python3

import time
import atexit
from collections import OrderedDict

from py3nvml import py3nvml as nv

from prometheus_client import Gauge, CollectorRegistry
from prometheus_client import write_to_textfile

ozancaglayan

class WeightedBatchSampler(Sampler):
    def __init__(self, n_elems, batch_size,
                 initial_p=None, epoch_p_reset=False):
        self.n_elems = n_elems
        self.batch_size = batch_size
        self.epoch_p_reset = epoch_p_reset

        self.n_batches = math.ceil(self.n_elems / self.batch_size)

        if initial_p is None:

ozancaglayan

#!/usr/bin/env python3
import re
import math
import argparse

from collections import defaultdict

LOG_2 = math.log(2)

###################################################################################

ozancaglayan

adam-512emb-1000lstm-wdecay-feature-reshape.sampling         Bleu_1:  0.46259  Bleu_2:  0.27595  Bleu_3:  0.16279  Bleu_4:  0.09660  CIDEr:  0.22156  METEOR:  0.22565  ROUGE_L:  0.33607
adam-512emb-1000lstm-feature-reshape.sampling                Bleu_1:  0.45442  Bleu_2:  0.26771  Bleu_3:  0.15705  Bleu_4:  0.09146  CIDEr:  0.19169  METEOR:  0.21855  ROUGE_L:  0.32499
adam-512emb-1000lstm-wdecay-feature-reshape.beam12.1best     Bleu_1:  0.45167  Bleu_2:  0.26816  Bleu_3:  0.15995  Bleu_4:  0.09787  CIDEr:  0.26480  METEOR:  0.24464  ROUGE_L:  0.34140
adam-512emb-1000lstm-feature-reshape-noprevctx.sampling      Bleu_1:  0.44216  Bleu_2:  0.25580  Bleu_3:  0.14835  Bleu_4:  0.08502  CIDEr:  0.18997  METEOR:  0.22077  ROUGE_L:  0.32038
adam-512emb-1000lstm-feature-reshape.beam12.1best            Bleu_1:  0.44151  Bleu_2:  0.25829  Bleu_3:  0.15098  Bleu_4:  0.09008  CIDEr:  0.24039  METEOR:  0.23844  ROUGE_L:  0.33360
adam-512emb-1000lstm-feature-reshape-noprevctx.beam12.1best  Bleu_1:  0.43102  Bleu_2:  0.2487

ozancaglayan

       Wemb_enc, -0.05387, 0.05577
       Wemb_dec, -0.05280, 0.05521
      encoder_W, -0.05016, 0.04749
      encoder_b, 0.00000, 0.00000
      encoder_U, -0.17179, 0.16453
     encoder_Wx, -0.04784, 0.04707
     encoder_bx, 0.00000, 0.00000
     encoder_Ux, -0.15236, 0.15887
    encoder_r_W, -0.04918, 0.04668
    encoder_r_b, 0.00000, 0.00000

ozancaglayan

local argparse = require "argparse"
local moses = require "moses"

local parser = argparse("build_dictionary", "example")

parser:argument("input", "Input text file"):args('+')
parser:option('-o --output', 'Output directory', '.')
parser:option('-m --minfreq', 'Filter out words occuring < m times.', 0)

local args = parser:parse()