miguelmota

// explicity setting the content-length will disable the "transfer-encoding: chunked" header
req.ContentLength = int64(len(requestBody))

// you can also remove the transfer-encoding header with this,
// however setting the content-length header is still required.
req.TransferEncoding = []string{"identity"}

ctlllll

import tiktoken
import langdetect
T = tiktoken.get_encoding("o200k_base")

length_dict = {}

for i in range(T.n_vocab):
    try:
        length_dict[i] = len(T.decode([i]))
    except: