lm

from models.transformer_fused import Model
from models.transformer_lm import TransformerLM
lm = TransformerLM('lm', out_voc, **{
        "hid_size": 256,
        "ff_size": 1024,
        "num_heads": 4,
        "num_layers": 4,
        "rescale_emb": True,
        "relu_dropout": 0.0,
        "res_dropout": 0.0,
        "attn_dropout": 0.0,
        "inp_emb_bias": True,
        "res_steps": "nlda",
        "normalize_out": True,
        "force_bos": True
})

lm_weights = np.load("../../pretrained_lm_default_params-24999.npz")
ops = []
for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, lm.name):
    if w.name in lm_weights:
        ops.append(tf.assign(w, lm_weights[w.name]))
    else:
        print(w.name, 'not initialized')
        
sess.run(ops);