epwalsh

local transformer_model = 'bert-base-cased';
local epochs = 3;
local batch_size = 8;
local train_path = "https://allennlp.s3.amazonaws.com/datasets/squad/squad-train-v1.1.json";
local dev_path = "https://allennlp.s3.amazonaws.com/datasets/squad/squad-dev-v1.1.json";

{
  "dataset_reader": {
    "type": "transformer_squad",
    "transformer_model_name": transformer_model,

epwalsh

transformer_dir = "~/my-cool-transformer"

transformer_embedder.transformer_model.save_pretrained(transformer_dir)
tokenizer.tokenizer.save_pretrained(transformer_dir)

epwalsh

from allennlp.common.params import Params
from allennlp.common.plugins import import_plugins
from allennlp.data.tokenizers import Tokenizer, PretrainedTransformerTokenizer
from allennlp.models import load_archive
from allennlp.modules.token_embedders import PretrainedTransformerEmbedder

# Change this to your serialization directory.
serialization_dir = "~/my-trained-model"

# Make sure all of the classes our model and tokenizer use are registered.

epwalsh

"""
Proposal for new DatasetReader API.

For this to work, all `Instance`s would have to be efficiently serializable.
So `TextField`s, for example, shouldn't contain `TokenIndexer`s.

The flow of data would look like this (boxes represent separate Python processes):

```
           +-----------------------------------------------+

epwalsh

local transformer_model = 'bert-base-cased';

local epochs = 1;
local batch_size = 8;

{
  "dataset_reader": {
      "type": "transformer_squad",
      "transformer_model_name": transformer_model,
      "skip_invalid_examples": true,

epwalsh

#!/bin/bash

label_names=(
	'Status: Changes Requested' 
	'Status: Do Not Merge' 
	'Status: Help Wanted' 
	'Status: In Progress' 
	'Status: Mergeable' 
	'Status: Review Needed' 
	'Type: Bug' 

epwalsh

class CopyNetSeq2Seq(Model):

    # snip...
    
    def _get_ll_contrib(self,
                        generation_scores: torch.Tensor,
                        generation_scores_mask: torch.Tensor,
                        copy_scores: torch.Tensor,
                        target_tokens: torch.Tensor,
                        target_to_source: torch.Tensor,

epwalsh

class CopyNetSeq2Seq(Model):

    # snip...
    
    def _decoder_step(self,
                      last_predictions: torch.Tensor,
                      selective_weights: torch.Tensor,
                      state: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
        # shape: (group_size, max_input_sequence_length, encoder_output_dim)
        encoder_outputs_mask = state["source_mask"].float()

epwalsh

class CopyNetSeq2Seq(Model):

    # snip...
    
    def _get_copy_scores(self, state: Dict[str, torch.Tensor]) -> torch.Tensor:
        # NOTE: here `trimmed_source_length` refers to the input sequence length minus 2,
        # so that the special START and END tokens in the source are ignored. We also need to
        # ignore PAD tokens, but that happens elsewhere using a mask.
        # shape: (batch_size, trimmed_source_length, encoder_output_dim)
        trimmed_encoder_outputs = state["encoder_outputs"][:, 1:-1]

epwalsh

class CopyNetSeq2Seq(Model):
    
    # snip...
    
    def _get_generation_scores(self, state: Dict[str, torch.Tensor]) -> torch.Tensor:
        # `self._output_generation_layer` is just a PyTorch linear layer with an input
        # dimension equal to the decoder hidden state size, and an output dimension
        # equal to the size of the target vocabulary.
        return self._output_generation_layer(state["decoder_hidden"])