filter_two_dataset.py

# coding: utf-8

# In[1]:

get_ipython().magic(u'load_ext autoreload')
get_ipython().magic(u'autoreload 2')
import sys
sys.path.append("/afs/inf.ed.ac.uk/user/v/v1sjastr/scratch/v1sjastr/go-deep/")
import numpy as np
from training_data import datasets
from training_data.datasets import *
from go import liberty_representation
from training_data import datasets
import gomill
from gomill import boards
from sgflib.sgflib import *
from training_data.preprocessing import *
from training_data.datasets import *
from go.liberty_representation import *
from go.data_processing import *
from go.board import *
from itertools import *
import itertools
from matplotlib.pylab import *
import scipy
import matplotlib
from go.utils import *
import matplotlib.pyplot as plt
rng = np.random.RandomState(seed=777)


# In[ ]:

data_staszek = GoGodLiberties(n_test=0.0, which="scored")
X_staszek, Y_staszek = data_staszek.get_data(which="train", group_by_games=True, 
                                             target=[WINNER, WINNER_MARGIN, WINNER_MARGIN_WITHOUT_KOMI])
meta_staszek = data_staszek.get_meta(which="train")


# In[ ]:

targets_igor = [WINNER, GNUGO_BLACK_MARGIN, GNUGO_BLACK_MARGIN_WITHOUT_KOMI]


# In[ ]:

data_igor = GoGodRegionsWinners(n_test=0.0, which="scored")
X_igor, Y_igor = data_igor.get_data(which="train", group_by_games=True, 
                                             target=targets_igor)
meta_igor = data_igor.get_meta(which="train")


# In[ ]:

len(meta_igor)


# In[ ]:

len(meta_staszek)


# In[ ]:

meta['filename'][0]


# In[ ]:

igor_sgf_to_id = {"/".join(meta['filename'].split("/")[-2:]): id for id, meta in enumerate(meta_igor)}


# In[ ]:

igor_sgf_to_id.values()[0]


# In[ ]:

id_staszek_to_igor = [igor_sgf_to_id.get("/".join(meta['filename'].split("/")[-2:]), -1) for meta in meta_staszek]


# In[ ]:

winner = [Y_igor[targets_igor.index(WINNER)][id_staszek_to_igor[id]]           for id in range(len(X_staszek)) if id_staszek_to_igor[id] != -1]
gnugo_black_margin = [Y_igor[targets_igor.index(GNUGO_BLACK_MARGIN)][id_staszek_to_igor[id]]                      for id in range(len(X_staszek)) if id_staszek_to_igor[id] != -1]
gnugo_black_margin_without_komi = [Y_igor[targets_igor.index(GNUGO_BLACK_MARGIN_WITHOUT_KOMI)][id_staszek_to_igor[id]]                      for id in range(len(X_staszek)) if id_staszek_to_igor[id] != -1]


# In[ ]:

X_staszek_filtered = [X_staszek[id][-1:] for id in range(len(X_staszek)) if id_staszek_to_igor[id] != -1]


# In[ ]:

winner = np.hstack(winner).reshape(-1,).astype(np.int8)
gnugo_black_margin =  np.hstack(gnugo_black_margin).reshape(-1,).astype(np.float32)
gnugo_black_margin_without_komi =  np.hstack(gnugo_black_margin_without_komi).reshape(-1,).astype(np.uint8)


# In[ ]:

len(X_staszek_filtered)


# In[ ]:

positions = np.vstack(X_staszek_filtered).astype(np.uint8)


# In[ ]:

games_filtered = {"positions": positions, 
                       "gnugo_black_margin": gnugo_black_margin, 
                  "gnugo_black_margin_without_komi": gnugo_black_margin_without_komi,
                        "winner": winner}
game_lengths = [len(X) for X in X_staszek_filtered]


# In[ ]:

meta_filtered = [meta_staszek[id] for id in range(len(X_staszek)) if id_staszek_to_igor[id] != -1]


# In[ ]:

positions = positions.reshape(-1,3,46)


# In[ ]:

winner.shape, gnugo_black_margin.shape, positions.shape, gnugo_black_margin_without_komi.shape


# In[ ]:

save_compressed_dataset(meta=meta_filtered,
                        game_lengths=game_lengths,
                        fields=games_filtered ,
                        file_name="/afs/inf.ed.ac.uk/user/v/v1sjastr/scratch/v1sjastr/gogod_summer_scored_gnugo_liberties.npz")