quora_check_coverage

check_coverage.py

def check_coverage(text, embeddings_dict):
    known_words, unknown_words = {}, {};
    total_known, total_unknown = 0, 0;
    
    for sentence in text:
        for word in sentence.split(' '):
            if word in known_words:
                total_known = total_known + 1;
            elif word in embeddings_dict:
                known_words[word] = embeddings_dict[word];
                total_known = total_known + 1;
            else:
                unknown_words[word] = None;
                total_unknown = total_unknown + 1;
            
    print('Total coverage of Vocabulary %.2f'%(len(known_words) / len(embeddings_dict)))
    print('Total coverage of Dataset %.2f'%(total_known / (total_known + total_unknown)));
    
    return known_words, unknown_words;