embed

cn8.rs

 //encode 
 let tokens = self
    .tokenizer
    .encode_batch(sentences, true)
    .map_err(anyhow::Error::msg)?;
    
//collect
let token_ids = tokens
    .iter()
    .map(|tokens| {
        let tokens = tokens.get_ids().to_vec();
        Ok(Tensor::new(tokens.as_slice(), &self.device)?)
    })
    .collect::<anyhow::Result<Vec<_>>>()?;
    
//embed
let token_ids = Tensor::stack(&token_ids, 0)?;
let token_type_ids = token_ids.zeros_like()?;
let embeddings = self.model.forward(&token_ids, &token_type_ids)?;