manmohan24nov

>>> from spacy_langdetect import LanguageDetector
>>> import spacy
>>> nlp = spacy.load('en')  # 1
>>> nlp.add_pipe(LanguageDetector(), name='language_detector', last=True) #2
>>> text_content = "Er lebt mit seinen Eltern und seiner Schwester in Berlin."
>>> doc = nlp(text_content) #3
>>> detect_language = doc._.language #4
>>> print(detect_language)
{'language': 'de', 'score': 0.9999958526911192}

manmohan24nov

>>> text_content = "Er lebt mit seinen Eltern und seiner Schwester in Berlin. Welcome, to this world of Data Scientist. Today is a lovely day."
>>> doc = nlp(text_content)
>>> detect_language = doc._.language
>>> print(detect_language)
{'language': 'en', 'score': 0.8571372625765084}

manmohan24nov

>>> from textblob import TextBlob
>>> text = "это компьютерный портал для гиков. It was a beautiful day ."
>>> lang = TextBlob(text)
>>> print(lang.detect_language())
ru

manmohan24nov

>>> from googletrans import Translator
>>> translator = Translator()
>>> translator.detect('이 문장은 한글로 쓰여졌습니다.')
<Detected lang=ko confidence=0.27041003>

manmohan24nov

language_short_name = {'aa':'Afar','ab':'Abkhazian','af':'Afrikaans','ak':'Akan','sq':'Albanian','am':'Amharic','ar':'Arabic',
'an':'Aragonese','hy':'Armenian','as':'Assamese','av':'Avaric','ae':'Avestan','ay':'Aymara','az':'Azerbaijani','ba':'Bashkir',
'bm':'Bambara','eu':'Basque','be':'Belarusian','bn':'Bengali','bh':'Bihari languages','bi':'Bislama','bo':'Tibetan','bs':'Bosnian',
'br':'Breton','bg':'Bulgarian','my':'Burmese','ca':'Catalan; Valencian','cs':'Czech','ch':'Chamorro','ce':'Chechen','zh':'Chinese',
'cu':'Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic','cv':'Chuvash','kw':'Cornish','co':'Corsican',
'cr':'Cree','cy':'Welsh','cs':'Czech','da':'Danish','de':'German','dv':'Divehi; Dhivehi; Maldivian','nl':'Dutch; Flemish','dz':'Dzongkha',
'el':'Greek-Modern (1453-)','en':'English','eo':'Esperanto','et':'Estonian','eu':'Basque','ee':'Ewe','fo':'Faroese','fa':'Persian',
'fj':'Fijian','fi':'Finnish','fr':'French','fy':'Western Frisian','ff':'Fulah','Ga':'Georgian','gd':

manmohan24nov

>>> import spacy
>>> nlp = spacy.load("en_core_sci_lg")
>>> text = """spaCy is an open-source software library for advanced natural language processing, 
written in the programming languages Python and Cython. The library is published under the MIT license
and its main developers are Matthew Honnibal and Ines Montani, the founders of the software company Explosion."""
>>> doc = nlp(text)
>>> print(doc.ents)
(spaCy, open-source software library, written, programming languages,
 Python, Cython, library, MIT, license, developers, Matthew Honnibal, 
 Ines, Montani, founders, software company)

manmohan24nov

>>> import yake
>>> kw_extractor = yake.KeywordExtractor()
>>> text = """spaCy is an open-source software library for advanced natural language processing, written in the programming languages Python and Cython. The library is published under the MIT license and its main developers are Matthew Honnibal and Ines Montani, the founders of the software company Explosion."""
>>> language = "en"
>>> max_ngram_size = 3
>>> deduplication_threshold = 0.9
>>> numOfKeywords = 20
>>> custom_kw_extractor = yake.KeywordExtractor(lan=language, n=max_ngram_size, dedupLim=deduplication_threshold, top=numOfKeywords, features=None)
>>> keywords = custom_kw_extractor.extract_keywords(text)
>>> for kw in keywords:

manmohan24nov

>>> from rake_nltk import Rake
>>> rake_nltk_var = Rake()
>>> text = """spaCy is an open-source software library for advanced natural language processing,
written in the programming languages Python and Cython. The library is published under the MIT license
and its main developers are Matthew Honnibal and Ines Montani, the founders of the software company Explosion."""
>>> rake_nltk_var.extract_keywords_from_text(text)
>>> keyword_extracted = rake_nltk_var.get_ranked_phrases()
>>> print(keyword_extracted)
['advanced natural language processing', 'software company explosion', 
 'programming languages python', 'source software library', 'mit license',

manmohan24nov

>>> from gensim.summarization import keywords
>>> text = """spaCy is an open-source software library for advanced natural language processing, 
written in the programming languages Python and Cython. The library is published under the MIT license 
and its main developers are Matthew Honnibal and Ines Montani, the founders of the software company Explosion."""
>>> print(keywords(text))
language
languages
software
company

manmohan24nov

>>> from nltk.tokenize import word_tokenize
>>> text_data = "Life is what happens when you're busy making other plans."
>>> duplicate_data = "what happens when you're busy"
>>> original_tokens = word_tokenize(text_data)
>>> duplicate_tokens = word_tokenize(duplicate_data)
>>> # Convert all the characters to lower case because this method is case sensitive.
>>> original_tokens = [token.lower() for token in original_tokens]
>>> duplicate_tokens = [token.lower() for token in duplicate_tokens]
>>> original_trigrams = []
>>> for i in range(len(original_tokens) - 2):