kastnerkyle

#!/usr/bin/env python3
"""
To use:
1. install/set-up the google cloud api and dependencies listed on https://github.com/GoogleCloudPlatform/python-docs-samples/tree/master/texttospeech/cloud-client
2. install pandoc and pypandoc, also tqdm
3. create and download a service_account.json ("Service account key") from https://console.cloud.google.com/apis/credentials
4. run GOOGLE_APPLICATION_CREDENTIALS=service_account.json python make_audiobook.py book_name.epub
"""
import re
import sys

kastnerkyle

#!/usr/bin/env python3
"""
To use:
1. install/set-up the google cloud api and dependencies listed on https://github.com/GoogleCloudPlatform/python-docs-samples/tree/master/texttospeech/cloud-client
2. install pandoc and pypandoc, also tqdm
3. create and download a service_account.json ("Service account key") from https://console.cloud.google.com/apis/credentials
4. run GOOGLE_APPLICATION_CREDENTIALS=service_account.json python make_audiobook.py book_name.epub
"""
import re
import sys

kastnerkyle

Some planets may orbit a supermassive black hole instead of a star
This is almost certainly not what Denisovans looked like
UN climate summit: Scientists' messages to world leaders
A hat that zaps the scalp with electricity helps reverse male balding
Man sees the world in miniature after a stroke damages his brain
Do dads matter? Anna Machin on the fascinating science of fatherhood
People like the idea of a carbon tax - if the money is put to good use
Climate change will boost risk of extreme flooding in northern Europe
Fast swimming fish robot could perform underwater surveillance
Ad Astra: Pirates and space monkeys can't save dull space psychodrama

kastnerkyle

Code

https://gist.github.com/victor-shepardson/5b3d3087dc2b4817b9bffdb8e87a57c4

Setup

I'm using Ubuntu 16.04 with a GTX 1060

install CUDA

kastnerkyle

import jax
import jax.numpy as np
from jax import grad, jit
from jax.scipy.special import logsumexp


def dadashi_fig2d():
    """ Figure 2 d) of
    ''The Value Function Polytope in Reinforcement Learning''
    by Dadashi et al. (2019) https://arxiv.org/abs/1901.11524

kastnerkyle

kastnerkyle

# Author: Kyle Kastner
# License: BSD 3-Clause

# based on minigo implementation
# https://github.com/tensorflow/minigo/blob/master/mcts.py 
# Useful discussion of the benefits
# http://www.moderndescartes.com/essays/agz/
# single player tweaks based on
# https://tmoer.github.io/AlphaZero/

kastnerkyle

# Author: Kyle Kastner
# License: BSD 3-Clause

# based on minigo implementation
# https://github.com/tensorflow/minigo/blob/master/mcts.py 
# Useful discussion of the benefits
# http://www.moderndescartes.com/essays/agz/

# See survey
# http://mcts.ai/pubs/mcts-survey-master.pdf

kastnerkyle

# Copyright (c) 2019-present, Thomas Wolf.
# All rights reserved. This source code is licensed under the MIT-style license.
""" A very small and self-contained gist to train a GPT-2 transformer model on wikitext-103 """
import os
from collections import namedtuple
from tqdm import tqdm
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from ignite.engine import Engine, Events

kastnerkyle

def top_k_top_p_filtering(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')):
    """ Filter a distribution of logits using top-k and/or nucleus (top-p) filtering
        Args:
            logits: logits distribution shape (..., vocabulary size)
            top_k >0: keep only top k tokens with highest probability (top-k filtering).
            top_p >0.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering).
    """
    top_k = min(top_k, logits.size(-1))  # Safety check
    if top_k > 0:
        # Remove all tokens with a probability less than the last token of the top-k