lilacs snippets

information_extraction.py

from lilacs.reasoning.extraction import LILACSextractor

LILACS = LILACSextractor()

test_text = """London is the capital and most populous city of England and the United Kingdom.
          Standing on the River Thames in the south east of the island of Great Britain, London has been a major settlement for 2 millennia.
          It was founded by the Romans, who named it Londinium. 
          London's ancient core, the City of London, which covers an area of only 1.12 square miles (2.9 km2), largely retains its medieval boundaries. 
          Since at least the 19th century, "London" has also referred to the metropolis around this core, historically split between Middlesex, Essex, Surrey, Kent and Hertfordshire, which today largely makes up Greater London, a region governed by the Mayor of London and the London Assembly."""

facts = LILACS.extract_facts(test_text)
"""
[
 'Since at least the 19th century, "London" has also referred to the '
 'metropolis around this core, historically split between Middlesex, Essex, '
 'Surrey, Kent and Hertfordshire, which today largely makes up Greater London, '
 'a region governed by the Mayor of London and the London Assembly.',

 'London is the capital and most populous city of England and the United '
 'Kingdom.',

 'standing on the River Thames in the south east of the island of Great '
 'Britain, London has been a major settlement for 2 millennia.',

 "London's ancient core, the City of London, which covers an area of only 1.12 "
 'square miles (2.9 km2), largely retains its medieval boundaries.'
]
"""

facts = LILACS.extract_facts(test_text, "Romans")
"""
['It was founded by the Romans, who named it Londinium.']
"""

triples = LILACS.extract_semi_structured_statements(test_text, "london")
"""
[('London', 'is', 'the capital and most populous city of England and the United Kingdom.'),
 ('London', 'has been', 'a major settlement for 2 millennia.')]
"""

entities = LILACS.extract(test_text)
for ent in entities:
    print(ent + " " + str(entities[ent]["tags"]))
"""
"London ['Capital City', 'Noun', 'entity']"
"England ['Noun', 'entity']"
"United Kingdom ['Noun', 'entity']"
"River Thames ['Noun', 'entity']"
"south east ['entity']"
"Great Britain ['Noun', 'entity']"
"2 ['Dimensionless_quantity', 'entity']"
"Romans ['Noun', 'entity']"
"Londinium ['Noun', 'entity']"
"City of London ['entity']"
"only 1.12 square miles ['entity']"
"2.9 km2 ['Area:Square_kilometre', 'entity']"
"at least the 19th century ['entity']"
"metropolis ['entity']"
"Middlesex ['Noun', 'entity']"
"Essex ['Noun', 'entity']"
"Surrey ['Noun', 'entity']"
"Kent ['Noun', 'entity']"
"Hertfordshire ['Noun', 'entity']"
"today ['entity']"
"Greater London ['Noun', 'entity']"
"London Assembly ['Noun', 'entity']"
"City ['Noun']"
"Mayor ['Noun']"
"1.12 square miles ['Area:Square_mile']"
"""
# NOTE dbpedia is SLOW
# to include it in results above
# entities = LILACS.extract(big_text, dbpedia=True)

entities = LILACS.extract_dbpedia(test_text)
"""
[{'confidence': 0.9986392327537942,
  'data': {'facts': ['Since at least the 19th century, "London" has also '
                     'referred to the metropolis around this core, '
                     'historically split between Middlesex, Essex, Surrey, '
                     'Kent and Hertfordshire, which today largely makes up '
                     'Greater London, a region governed by the Mayor of London '
                     'and the London Assembly.',
                     'London is the capital and most populous city of England '
                     'and the United Kingdom.',
                     'standing on the River Thames in the south east of the '
                     'island of Great Britain, London has been a major '
                     'settlement for 2 millennia.'],
           'uri': 'http://dbpedia.org/resource/London'},
  'tags': ['Wikidata:Q486972',
           'Schema:Place',
           'DBpedia:Settlement',
           'DBpedia:PopulatedPlace',
           'DBpedia:Place',
           'DBpedia:Location'],
  'value': 'London'},
 {'confidence': 0.9987980803946171,
  'data': {'facts': [],
           'uri': 'http://dbpedia.org/resource/United_Kingdom'},
  'tags': ['Wikidata:Q6256',
           'Schema:Place',
           'Schema:Country',
           'DBpedia:PopulatedPlace',
           'DBpedia:Place',
           'DBpedia:Location',
           'DBpedia:Country'],
  'value': 'United Kingdom'},
 {'confidence': 0.9999999999907914,
  'data': {'facts': ['standing on the River Thames in the south east of the '
                     'island of Great Britain, London has been a major '
                     'settlement for 2 millennia.'],
           'uri': 'http://dbpedia.org/resource/River_Thames'},
  'tags': ['Wikidata:Q47521',
           'Wikidata:Q4022',
           'Schema:RiverBodyOfWater',
           'Schema:Place',
           'Schema:BodyOfWater',
           'DBpedia:Stream',
           'DBpedia:River',
           'DBpedia:Place',
           'DBpedia:NaturalPlace',
           'DBpedia:Location',
           'DBpedia:BodyOfWater'],
  'value': 'River Thames'},
  
(....)

 {'confidence': 0.9999981762260153,
  'data': {'facts': ['standing on the River Thames in the south east of the '
                     'island of Great Britain, London has been a major '
                     'settlement for 2 millennia.'],
           'uri': 'http://dbpedia.org/resource/Greater_London'},
  'tags': ['Wikidata:Q3455524',
           'Schema:Place',
           'Schema:AdministrativeArea',
           'DBpedia:Region',
           'DBpedia:PopulatedPlace',
           'DBpedia:Place',
           'DBpedia:Location',
           'DBpedia:AdministrativeRegion'],
  'value': 'Greater London'},
 {'confidence': 0.9999996204112885,
  'data': {'facts': [],
           'uri': 'http://dbpedia.org/resource/London_Assembly'},
  'tags': ['Wikidata:Q43229',
           'Wikidata:Q24229398',
           'Wikidata:Q11204',
           'DUL:SocialPerson',
           'DUL:Agent',
           'Schema:Organization',
           'DBpedia:Organisation',
           'DBpedia:Legislature',
           'DBpedia:Agent'],
  'value': 'London Assembly'}
]
"""

entities = LILACS.extract_entities(test_text)  # entity candidates
"""
[{'confidence': 1,
  'data': {'facts': ['Since at least the 19th century, "London" has also '
                     'referred to the metropolis around this core, '
                     'historically split between Middlesex, Essex, Surrey, '
                     'Kent and Hertfordshire, which today largely makes up '
                     'Greater London, a region governed by the Mayor of London '
                     'and the London Assembly.',
                     'London is the capital and most populous city of England '
                     'and the United Kingdom.',
                     'standing on the River Thames in the south east of the '
                     'island of Great Britain, London has been a major '
                     'settlement for 2 millennia.']},
  'tags': ['entity'],
  'value': 'London'},
 {'confidence': 1,
  'data': {'facts': ['London is the capital and most populous city of England '
                     'and the United Kingdom.']},
  'tags': ['entity'],
  'value': 'England'},
 {'confidence': 1,
  'data': {'facts': []},
  'tags': ['entity'],
  'value': 'United Kingdom'},
 {'confidence': 1,
  'data': {'facts': ['standing on the River Thames in the south east of the '
                     'island of Great Britain, London has been a major '
                     'settlement for 2 millennia.']},
  'tags': ['entity'],
  'value': 'River Thames'},
 (...)
"""

entities = LILACS.extract_keywords(test_text)  # Rake Keywords
"""
[{'confidence': 1,
  'data': {'facts': [], 'keyword_score': 9.0},
  'tags': ['keyword'],
  'value': 'today largely makes'},
 {'confidence': 1,
  'data': {'facts': ['London is the capital and most populous city of England '
                     'and the United Kingdom.'],
           'keyword_score': 4.0},
  'tags': ['keyword'],
  'value': 'populous city'},
 {'confidence': 1,
  'data': {'facts': [], 'keyword_score': 4.0},
  'tags': ['keyword'],
  'value': 'united kingdom'},
 {'confidence': 1,
  'data': {'facts': ['standing on the River Thames in the south east of the '
                     'island of Great Britain, London has been a major '
                     'settlement for 2 millennia.'],
           'keyword_score': 4.0},
  'tags': ['keyword'],
  'value': 'river thames'},
 {'confidence': 1,
  'data': {'facts': ['standing on the River Thames in the south east of the '
                     'island of Great Britain, London has been a major '
                     'settlement for 2 millennia.'],
           'keyword_score': 4.0},
  'tags': ['keyword'],
  'value': 'south east'},
 (...)
"""

entities = LILACS.extract_location(test_text)  # wordlist, country names and  capital cities only
"""
[{'confidence': 0.9,
  'data': {'country_code': 'GB',
           'country_name': 'United Kingdom',
           'facts': ['Since at least the 19th century, "London" has also '
                     'referred to the metropolis around this core, '
                     'historically split between Middlesex, Essex, Surrey, '
                     'Kent and Hertfordshire, which today largely makes up '
                     'Greater London, a region governed by the Mayor of London '
                     'and the London Assembly.',
                     'London is the capital and most populous city of England '
                     'and the United Kingdom.',
                     'standing on the River Thames in the south east of the '
                     'island of Great Britain, London has been a major '
                     'settlement for 2 millennia.'],
           'hemisphere': 'north',
           'name': 'London'},
  'tags': ['Capital City'],
  'value': 'London'}]
"""

entities = LILACS.extract_units(test_text)
"""
TODO BUG in quantulum 3
"""

normalization.py

from lilacs.parse import SentenceParser

LILACS = SentenceParser()


s = "I love my dogs"
print(SentenceParser.normalize(s, make_singular=True))
"""
I love my dog
"""
print(SentenceParser.normalize(s, make_singular=True, remove_pronouns=True))
"""
I love dog
"""

s = "My sister has a dog. She loves him."
print(SentenceParser.normalize(s, solve_corefs=True))
"""
My sister has dog. She loves a dog.
"""
# MISSED
# She -> My sister


s = """
London is the capital and most populous city of England and the United Kingdom.
It was founded by the Romans,who named it Londinium.
"""
print(SentenceParser.normalize(s, solve_corefs=True, remove_articles=True))
"""
London is capital and most populous city of England and United Kingdom.
London was founded by Romans, who named London Londinium.
"""
# MISSED
# who -> the romans


print(SentenceParser.process_text(s))
"""
['london capital populous city england united kingdom', 
'found romans name londinium']
"""

sentence_parsing.py

from lilacs.parse import SentenceParser

LILACS = SentenceParser()

# Chunking text
text = "turn on the light and turn it green"
big_text = """London is the capital and most populous city of England and 
the United Kingdom. Standing on the River Thames in the south east of the 
island of Great Britain, London has been a major settlement for 2 millennia. It was founded by the Romans, who named it Londinium."""


sentences = LILACS.split_sentences(big_text, solve_corefs=True)
"""
['London is the capital and most populous city of England and the United Kingdom.', 
'Standing on the River Thames in the south east of the island of Great Britain, London has been a major settlement for 2 millennia.', 
'London was founded by the Romans, who named London Londinium.']
"""
sentences = LILACS.chunk(big_text, solve_corefs=True)
"""
['London is the capital', 
'most populous city of England', 
'the United Kingdom.', 
'Standing on the River Thames in the south east of the island of Great Britain',
'London has been a major settlement for 2 millennia.', 
'London was founded by the Romans', 
'who named London Londinium.']
"""

commands = LILACS.chunk(text, solve_corefs=True)
"""
['turn on the light', 'turn the light green']
"""

# handling verbs
assert LILACS.is_passive('Mistakes were made.')
assert not LILACS.is_passive('I made mistakes.')
# NOTE Notable fail case. Fix me. I think it is because the 'to be' verb is omitted.
# assert t.is_passive('guy shot by police')

assert LILACS.change_tense("I am making dinner",
                           "past") == "I was making dinner"
assert LILACS.change_tense("I am making dinner",
                           "future") == "I will be making dinner"
assert LILACS.change_tense("I am making dinner",
                           "present") == "I am making dinner"


# word metrics
n_words = LILACS.count_unique_words(big_text)
""" 38 """

bag = LILACS.bag_of_words(big_text)
"""
[('London', 2), ('capital', 1), ('populous', 1), ('city', 1), ('United', 1), 
('Kingdom', 1), ('stand', 1), ('River', 1), ('Thames', 1), ('south', 1), 
('east', 1), ('island', 1), ('Great', 1), ('Britain', 1), (...)"""

pos_tags = LILACS.tag_sentence(sentences[0])
"""
[('London', 'NP-HL'), ('is', 'BEZ'), ('the', 'AT'), ('capital', 'NN')]
"""

rank = LILACS.text_rank(big_text)
"""
[('major settlement', 0.04612352789123868), 
('populous city', 0.04092495866435582), 
('River Thames', 0.03982333027329801), 
('United Kingdom', 0.03941882384916587), 
('Great Britain', 0.03937889638376672)]
"""

rank = LILACS.ngram_rank(big_text)
"""
[('major settlement', 0.2220317260467898), 
('great britain', 0.15072790866655575)]
"""

text_analysis.py

from lilacs.reasoning import LILACSTextAnalyzer

LILACS = LILACSTextAnalyzer()

# word2vec

text1 = "my birthday is next wednesday"
text2 = "i have a dog"
score = LILACS.similarity_match(text1, text2)
# 0.5224385790619714

text3 = "i was born in january"
score = LILACS.similarity_match(text1, text3)
# 0.7512508036193444

text4 = "trains run at 100km/h"
score = LILACS.similarity_match(text1, text4)
# 0.44370032064121794



# Readability
#   https://en.wikipedia.org/wiki/Readability

text = "I am some really difficult text to read because I use obnoxiously " \
     "large words."

"""The "Flesch–Kincaid Grade Level Formula" instead presents a score as a 
U.S. grade level"""
score = LILACSTextAnalyzer.grade_level(text)
# 8.412857142857145

"""
Score 	        School level 	    Notes
100.00–90.00 	5th grade 	        Very easy to read. Easily understood by an average 11-year-old student.
90.0–80.0 	    6th grade 	        Easy to read. Conversational English for consumers.
80.0–70.0 	    7th grade 	        Fairly easy to read.
70.0–60.0   	8th & 9th grade 	Plain English. Easily understood by 13- to 15-year-old students.
60.0–50.0   	10th to 12th grade 	Fairly difficult to read.
50.0–30.0   	College 	        Difficult to read.
30.0–0.0 	    College graduate 	Very difficult to read. Best understood by university graduates. 
"""
score = LILACSTextAnalyzer.reading_ease(text)
# 59.68214285714288 - 10th to 12th grade