Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save mladvladimir/afba73e36bcfa65974a87e28301ddfec to your computer and use it in GitHub Desktop.
Save mladvladimir/afba73e36bcfa65974a87e28301ddfec to your computer and use it in GitHub Desktop.
let mut batch_tokens = Vec::new();
let mut longest_seq = 0;
for idx in &length_sorted_idx[batch_start..batch_end] {
let sentence = &sentences[*idx];
println!("pa gde se gubis koji kurac - longest_seq {:?}", longest_seq);
let tokens = self.bert.tokenize(&sentence);
let mut longest_seq = &longest_seq.max(tokens.len());
batch_tokens.push(tokens);
}
@pradovic
Copy link

pradovic commented Apr 5, 2020

let mut batch_tokens = Vec::new();
let mut longest_seq = 0;

for idx in &length_sorted_idx[batch_start..batch_end] {
    let sentence = &sentences[*idx];
    println!("pa gde se gubis koji kurac - longest_seq {:?}", longest_seq);
    let tokens = self.bert.tokenize(&sentence);
    longest_seq = &longest_seq.max(tokens.len());
    batch_tokens.push(tokens);
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment