bpe.py

bpemb_ru = BPEmb(lang='ru', dim=50)


def extract_text(json_data):
    ru_text = list(filter(lambda x: any(1040 <= ord(y) <= 1103 for y in x), json_data.split('"')))
    return ' '.join(ru_text)


def embed_text(text):
    ids = bpemb_ru.encode_ids(text)
    # ids = list(filter(lambda x: x in ids_white_list, ids))
    return bpemb_ru.emb.vectors[ids].sum(axis=0)
    
def preprocess_stories_description(df):
    story_emb_df = pd.DataFrame(np.vstack(df.story_json.map(extract_text).map(embed_text))).add_prefix('story_bpe__')
    story_emb_df['story_id'] = df['story_id']
    return story_emb_df