Created
October 8, 2024 15:46
-
-
Save JarbasAl/0a8a25b9442ad7e709b73bbda1a20582 to your computer and use it in GitHub Desktop.
extracting numbers from portuguese text
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import enum | |
| from dataclasses import dataclass | |
| from typing import List, Iterable, Union, Optional, Tuple | |
| from quebra_frases import word_tokenize | |
| class DisambiguationStrategy(enum.IntEnum): | |
| ASSUME_ORDINALS = 0 | |
| ASSUME_FRACTIONS = 1 | |
| GUESS = 2 | |
| class NumberScale(enum.IntEnum): | |
| LONG_SCALE = 0 | |
| SHORT_SCALE = 1 | |
| def is_number_word(word: str, scale: NumberScale = NumberScale.LONG_SCALE) -> bool: | |
| if is_numeric(word) or is_fraction(word): | |
| return True | |
| mapping = PortugueseNumberTagger._WORD2NUM | |
| if scale == NumberScale.LONG_SCALE: | |
| mapping.update(PortugueseNumberTagger._WORD2NUM_LONG_SCALE) | |
| else: | |
| mapping.update(PortugueseNumberTagger._WORD2NUM_SHORT_SCALE) | |
| mapping.update(PortugueseNumberTagger._WORD2NUM_FRACTIONS) | |
| return word in mapping | |
| def word2number(word: str, scale: NumberScale = NumberScale.LONG_SCALE) -> Optional[Union[float, int]]: | |
| mapping = PortugueseNumberTagger._WORD2NUM | |
| if scale == NumberScale.LONG_SCALE: | |
| mapping.update(PortugueseNumberTagger._WORD2NUM_LONG_SCALE) | |
| else: | |
| mapping.update(PortugueseNumberTagger._WORD2NUM_SHORT_SCALE) | |
| mapping.update(PortugueseNumberTagger._WORD2NUM_FRACTIONS) | |
| w = mapping.get(word) | |
| if w: | |
| return int(w) if is_integer(w) else float(w) | |
| if is_integer(word): | |
| return int(word) | |
| if is_numeric(word): | |
| return float(word) | |
| if is_fraction(word): | |
| n1, n2 = word.split("/") | |
| return int(n1) / int(n2) | |
| def is_numeric(word: str) -> bool: | |
| """Check if the given word is a numeric value, including floats. | |
| Args: | |
| word (str): The word to check. | |
| Returns: | |
| bool: True if the word is numeric, otherwise False. | |
| """ | |
| # NOTE: we don't use .isdigit() so we can catch floats | |
| try: | |
| float(word) | |
| return True | |
| except: | |
| return False | |
| def is_integer(word: str) -> bool: | |
| """Check if the given word is an integer value, including negative signs | |
| Args: | |
| word (str): The word to check. | |
| Returns: | |
| bool: True if the word is an integer, otherwise False. | |
| """ | |
| # NOTE: we don't use .isdigit() so we can catch negatives | |
| try: | |
| int(word) | |
| return True | |
| except ValueError: | |
| # handle scientific notation (trillions etc) | |
| if word.startswith("1e+"): | |
| return True | |
| return False | |
| def is_fraction(word: str) -> bool: | |
| """Check if the given word is a fraction of the form N1/N2. | |
| Args: | |
| word (str): The word to check. | |
| Returns: | |
| bool: True if the word is a fraction, otherwise False. | |
| """ | |
| if "/" in word and len(word.split("/")) == 2: | |
| n1, n2 = word.split("/") | |
| if is_numeric(n1) and is_numeric(n2): | |
| return True | |
| return False | |
| def get_prev_word(tokens: List[str], idx: int) -> Tuple[Optional[str], int]: | |
| prev = None | |
| index = -1 | |
| for i, w in enumerate(tokens): | |
| if i >= idx: | |
| break | |
| if w: | |
| prev = w | |
| index = i | |
| return prev, index | |
| def get_consecutive_integers(tokens: List[str], start_idx: int) -> Tuple[List[str], int]: | |
| last_index = -1 | |
| numbers = [] | |
| for i, w in enumerate(tokens): | |
| if i <= start_idx: | |
| continue | |
| if is_integer(w): | |
| last_index = i | |
| numbers.append(w) | |
| else: | |
| break | |
| return numbers, last_index | |
| @dataclass | |
| class Span: | |
| """Class representing a span of text with a numeric value. | |
| Attributes: | |
| start_token_idx (int): The starting token index of the span. | |
| end_token_idx (int): The ending token index of the span. | |
| text (str): The original text of the span. | |
| number (Union[int, float]): The numeric value represented by the span. | |
| """ | |
| start_token_idx: int | |
| end_token_idx: int | |
| text: str | |
| number: Union[int, float] | |
| @dataclass | |
| class OrdinalNumberSpan(Span): | |
| start_token_idx: int | |
| end_token_idx: int | |
| text: str | |
| number: Union[int, float] | |
| is_ambiguous: bool | |
| class PortugueseNumberTagger: | |
| """A class for tagging numbers in Portuguese text. | |
| This class converts Portuguese number words and fractions into their | |
| numeric representations and identifies spans of text that represent | |
| numeric values. | |
| """ | |
| _WORD2NUM = { | |
| "zero": "0", | |
| "um": "1", | |
| "dois": "2", | |
| "três": "3", | |
| "quatro": "4", | |
| "cinco": "5", | |
| "seis": "6", | |
| "sete": "7", | |
| "oito": "8", | |
| "nove": "9", | |
| "dez": "10", | |
| "onze": "11", | |
| "doze": "12", | |
| "treze": "13", | |
| "catorze": "14", | |
| "quinze": "15", | |
| "dezasseis": "16", | |
| "dezassete": "17", | |
| "dezoito": "18", | |
| "dezanove": "19", | |
| "vinte": "20", | |
| "trinta": "30", | |
| "quarenta": "40", | |
| "cinquenta": "50", | |
| "sessenta": "60", | |
| "setenta": "70", | |
| "oitenta": "80", | |
| "noventa": "90", | |
| "cem": "100", | |
| "cento": "100", | |
| "duzentos": "200", | |
| "trezentos": "300", | |
| "quatrocentos": "400", | |
| "quinhentos": "500", | |
| "seiscentos": "600", | |
| "setecentos": "700", | |
| "oitocentos": "800", | |
| "novecentos": "900", | |
| "mil": "1000" | |
| } | |
| _WORD2NUM.update({ # add female variants | |
| "uma": "1", | |
| "duas": "2", | |
| "duzentas": "200", | |
| "trezentas": "300", | |
| "quatrocentas": "400", | |
| "quinhentas": "500", | |
| "seiscentas": "600", | |
| "setecentas": "700", | |
| "oitocentas": "800", | |
| "novecentas": "900", | |
| }) | |
| _WORD2NUM_HALFABLE = { | |
| "dúzia": "12", | |
| "centena": "100", | |
| "dúzias": "12", | |
| "centenas": "100", | |
| "milhar": "1000", | |
| "milhares": "1000" | |
| } | |
| _WORD2NUM_LONG_SCALE = { | |
| "milhão": str(1e6), | |
| "bilião": str(1e12), | |
| "trilião": str(1e18), | |
| "quadrilião": str(1e24), | |
| "quintilião": str(1e30), | |
| "sextilião": str(1e36), | |
| "septilião": str(1e42), | |
| "octilião": str(1e48), | |
| "nonilião": str(1e54), | |
| "decilhão": str(1e60), | |
| "undecilião": str(1e66), | |
| "duodecilião": str(1e72), | |
| "tredecilião": str(1e78), | |
| "quattuordecilião": str(1e84), | |
| "quindecilião": str(1e90), | |
| "sexdecilião": str(1e96), | |
| "septendecilião": str(1e102), | |
| "octodecilião": str(1e108), | |
| "novemdecilião": str(1e114), | |
| "vigintilião": str(1e120), | |
| "unvigintilião": str(1e126), | |
| "duovigintilião": str(1e132), | |
| "trevigintilião": str(1e138), | |
| "quattuorvigintilião": str(1e144), | |
| "quinvigintilião": str(1e150), | |
| "sexvigintilião": str(1e156), | |
| "septenvigintilião": str(1e162), | |
| "octovigintilião": str(1e168), | |
| "novemvigintilião": str(1e174), | |
| "trigintilião": str(1e180) | |
| } | |
| _WORD2NUM_LONG_SCALE.update({k.replace("lião", "lhão"): v # add pt-br variants | |
| for k, v in _WORD2NUM_LONG_SCALE.items()}) | |
| _WORD2NUM_LONG_SCALE.update({k[:-2] + "ões": v # add plural variants | |
| for k, v in _WORD2NUM_LONG_SCALE.items()}) | |
| _WORD2NUM_SHORT_SCALE = { | |
| "milhão": str(1e6), | |
| "bilião": str(1e9), | |
| "trilião": str(1e12), | |
| "quadrilião": str(1e15), | |
| "quintilião": str(1e18), | |
| "sextilião": str(1e21), | |
| "septilião": str(1e24), | |
| "octilião": str(1e27), | |
| "nonilião": str(1e30), | |
| "decilhão": str(1e33), | |
| "undecilião": str(1e36), | |
| "duodecilião": str(1e39), | |
| "tredecilião": str(1e42), | |
| "quattuordecilião": str(1e45), | |
| "quindecilião": str(1e48), | |
| "sexdecilião": str(1e51), | |
| "septendecilião": str(1e54), | |
| "octodecilião": str(1e57), | |
| "novemdecilião": str(1e60), | |
| "vigintilião": str(1e63), | |
| "unvigintilião": str(1e66), | |
| "duovigintilião": str(1e69), | |
| "trevigintilião": str(1e72), | |
| "quattuorvigintilião": str(1e75), | |
| "quinvigintilião": str(1e78), | |
| "sexvigintilião": str(1e81), | |
| "septenvigintilião": str(1e84), | |
| "octovigintilião": str(1e87), | |
| "novemvigintilião": str(1e90), | |
| "trigintilião": str(1e93) | |
| } | |
| _WORD2NUM_SHORT_SCALE.update({k.replace("lião", "lhão"): v # add pt-br variants | |
| for k, v in _WORD2NUM_SHORT_SCALE.items()}) | |
| _WORD2NUM_SHORT_SCALE.update({k[:-2] + "ões": v # add plural variants | |
| for k, v in _WORD2NUM_SHORT_SCALE.items()}) | |
| _WORD2NUM_ORDINAL = { | |
| "primeiro": "1", | |
| "segundo": "2", | |
| "terceiro": "3" | |
| } | |
| _WORD2NUM_ORDINAL.update({k[:-1] + "a": v # add female variants | |
| for k, v in _WORD2NUM_ORDINAL.items()}) | |
| _WORD2NUM_ORDINAL.update({k + "s": v # add plural variants | |
| for k, v in _WORD2NUM_ORDINAL.items()}) | |
| _WORD2NUM_HALF = { | |
| "meia": 0.5, | |
| "meio": 0.5 | |
| } | |
| _WORD2NUM_HALF.update({k + "s": v # add plural variants | |
| for k, v in _WORD2NUM_HALF.items()}) | |
| _WORD2NUM_FRACTIONS = { | |
| # "meio": "1/2", # Half | |
| "terço": "1/3" # Third | |
| } | |
| _WORD2NUM_FRACTIONS.update({k[:-1] + "a": v # add female variants | |
| for k, v in _WORD2NUM_FRACTIONS.items()}) | |
| _WORD2NUM_FRACTIONS["metade"] = "1/2" | |
| _WORD2NUM_FRACTIONS.update({k + "s": v # add plural variants | |
| for k, v in _WORD2NUM_FRACTIONS.items()}) | |
| # can mean both a fraction or an ordinal number | |
| _WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL = { | |
| "terço": "1/3", # Third | |
| "quarto": "1/4", # Fourth | |
| "quinto": "1/5", # Fifth | |
| "sexto": "1/6", # Sixth | |
| "sétimo": "1/7", # Seventh | |
| "oitavo": "1/8", # Eighth | |
| "nono": "1/9", # Ninth | |
| "décimo": "1/10", # Tenth | |
| "undécimo": "1/11", # Eleventh | |
| "duodécimo": "1/12", # Twelfth | |
| # multiples of 10 | |
| "vigésimo": "1/20", # Twentieth | |
| "trigésimo": "1/30", # Thirtieth | |
| "quadragésimo": "1/40", # Fortieth | |
| "quinquagésimo": "1/50", # Fiftieth | |
| "sexagésimo": "1/60", # Sixtieth | |
| "septuagésimo": "1/70", # Seventieth | |
| "octogésimo": "1/80", # Eightieth | |
| "nonagésimo": "1/90", # Ninetieth | |
| "centésimo": "1/100", # Hundredth | |
| # multiples of 100 | |
| "duocentésimo": "1/200", # Two-hundredth | |
| "trecentésimo": "1/300", # Three-hundredth | |
| "quadringentésimo": "1/400", # Four-hundredth | |
| "quingentésimo": "1/500", # Five-hundredth | |
| "seiscentésimo": "1/600", # Six-hundredth | |
| "setecentésimo": "1/700", # Seven-hundredth | |
| "oitocentésimo": "1/800", # Eight-hundredth | |
| "novecentésimo": "1/900", # Nine-hundredth | |
| "milésimo": "1/1000" # Thousandth | |
| } | |
| _WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL.update({k[:-1] + "a": v # add female variants | |
| for k, v in _WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL.items()}) | |
| _WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL.update({k + "s": v # add plural variants | |
| for k, v in _WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL.items()}) | |
| _SHORT_AMBIGUOUS_PT = { | |
| "milionésimo": "1/" + str(1e6), | |
| "bilionésimo": "1/" + str(1e9), | |
| "trilionésimo": "1/" + str(1e12), | |
| "quadrilionésimo": "1/" + str(1e15), | |
| "quintilionésimo": "1/" + str(1e18), | |
| "sextilionésimo": "1/" + str(1e21), | |
| "septilionésimo": "1/" + str(1e24), | |
| "octilionésimo": "1/" + str(1e27), | |
| "nonilionésimo": "1/" + str(1e30), | |
| "decilionésimo": "1/" + str(1e33), | |
| "undecilionésimo": "1/" + str(1e36), | |
| "duodecilionésimo": "1/" + str(1e39), | |
| "tredecilionésimo": "1/" + str(1e42), | |
| "quattuordecilionésimo": "1/" + str(1e45), | |
| "quindecilionésimo": "1/" + str(1e48), | |
| "sexdecilionésimo": "1/" + str(1e51), | |
| "septendecilionésimo": "1/" + str(1e54), | |
| "octodecilionésimo": "1/" + str(1e57), | |
| "novemdecilionésimo": "1/" + str(1e60), | |
| "vigintilionésimo": "1/" + str(1e63), | |
| "unvigintilionésimo": "1/" + str(1e66), | |
| "duovigintilionésimo": "1/" + str(1e69), | |
| "trevigintilionésimo": "1/" + str(1e72), | |
| "quattuorvigintilionésimo": "1/" + str(1e75), | |
| "quinvigintilionésimo": "1/" + str(1e78), | |
| "sexvigintilionésimo": "1/" + str(1e81), | |
| "septenvigintilionésimo": "1/" + str(1e84), | |
| "octovigintilionésimo": "1/" + str(1e87), | |
| "novemvigintilionésimo": "1/" + str(1e90), | |
| "trigintilionésimo": "1/" + str(1e93) | |
| } | |
| _LONG_AMBIGUOUS_PT = { | |
| "milionésimo": "1/" + str(1e6), | |
| "bilionésimo": "1/" + str(1e12), | |
| "trilionésimo": "1/" + str(1e18), | |
| "quadrilionésimo": "1/" + str(1e24), | |
| "quintilionésimo": "1/" + str(1e30), | |
| "sextilionésimo": "1/" + str(1e36), | |
| "septilionésimo": "1/" + str(1e42), | |
| "octilionésimo": "1/" + str(1e48), | |
| "nonilionésimo": "1/" + str(1e54), | |
| "decilionésimo": "1/" + str(1e60), | |
| "undecilionésimo": "1/" + str(1e66), | |
| "duodecilionésimo": "1/" + str(1e72), | |
| "tredecilionésimo": "1/" + str(1e78), | |
| "quattuordecilionésimo": "1/" + str(1e84), | |
| "quindecilionésimo": "1/" + str(1e90), | |
| "sexdecilionésimo": "1/" + str(1e96), | |
| "septendecilionésimo": "1/" + str(1e102), | |
| "octodecilionésimo": "1/" + str(1e108), | |
| "novemdecilionésimo": "1/" + str(1e114), | |
| "vigintilionésimo": "1/" + str(1e120), | |
| "unvigintilionésimo": "1/" + str(1e126), | |
| "duovigintilionésimo": "1/" + str(1e132), | |
| "trevigintilionésimo": "1/" + str(1e138), | |
| "quattuorvigintilionésimo": "1/" + str(1e144), | |
| "quinvigintilionésimo": "1/" + str(1e150), | |
| "sexvigintilionésimo": "1/" + str(1e156), | |
| "septenvigintilionésimo": "1/" + str(1e162), | |
| "octovigintilionésimo": "1/" + str(1e168), | |
| "novemvigintilionésimo": "1/" + str(1e174), | |
| "trigintilionésimo": "1/" + str(1e180) | |
| } | |
| @classmethod | |
| def _replace_number_words(cls, tokens: List[str], | |
| scale: NumberScale = NumberScale.LONG_SCALE) -> List[str]: | |
| """Replace number words in the token list with their numeric values. | |
| Args: | |
| tokens (List[str]): A list of tokenized words. | |
| scale (NumberScale): Define whether to use LONG or SHORT scale for larger numbers. | |
| Returns: | |
| List[str]: The token list with number words replaced. | |
| """ | |
| tokens = list(tokens) | |
| mapping = dict(cls._WORD2NUM) | |
| if scale == NumberScale.LONG_SCALE: | |
| mapping.update(cls._WORD2NUM_LONG_SCALE) | |
| else: | |
| mapping.update(cls._WORD2NUM_SHORT_SCALE) | |
| mapping.update(cls._WORD2NUM_FRACTIONS) | |
| mapping.update(cls._WORD2NUM_HALFABLE) | |
| for idx, text in enumerate(tokens): | |
| if text is None: | |
| continue | |
| if text in mapping and text not in cls._WORD2NUM_HALFABLE: | |
| tokens[idx] = mapping[text] | |
| if is_numeric(tokens[idx]): | |
| # check for negative sign | |
| prev_word, prev_idx = get_prev_word(tokens, idx) | |
| if prev_word and prev_word == "menos": | |
| tokens[idx] = "-" + tokens[idx] | |
| tokens[prev_idx] = None | |
| return tokens | |
| @classmethod | |
| def _disambiguate_halfs(cls, tokens: List[str], | |
| scale: NumberScale = NumberScale.LONG_SCALE) -> List[str]: | |
| tokens = list(tokens) | |
| if scale == NumberScale.LONG_SCALE: | |
| mapping = cls._WORD2NUM_LONG_SCALE | |
| else: | |
| mapping = cls._WORD2NUM_SHORT_SCALE | |
| mapping.update(cls._WORD2NUM_HALFABLE) | |
| for idx, text in enumerate(tokens): | |
| if text is None: | |
| continue | |
| prev_w = tokens[idx - 1] if idx > 0 else "" | |
| prev_prev_w = tokens[idx - 2] if idx > 1 else "" | |
| next_w = tokens[idx + 1] if idx + 1 < len(tokens) else "" | |
| next_next_w = tokens[idx + 2] if idx + 2 < len(tokens) else "" | |
| next_next_next_w = tokens[idx + 3] if idx + 3 < len(tokens) else "" | |
| if text in cls._WORD2NUM_HALF: | |
| # Half N and half HALFABLE | |
| if prev_w == "e" and next_w in cls._WORD2NUM_HALFABLE and is_number_word(prev_prev_w, scale): | |
| n = word2number(prev_prev_w, scale) | |
| n2 = 0.5 * int(cls._WORD2NUM_HALFABLE[next_w]) | |
| tokens[idx + 1] = str(n + n2) | |
| tokens[idx] = tokens[idx - 1] = tokens[idx - 2] = None | |
| # HALF HALFABLE | |
| # "meia centena", "meia duzia" -> "half a dozen" | |
| elif next_w in cls._WORD2NUM_HALFABLE: | |
| n = 0.5 * int(cls._WORD2NUM_HALFABLE[next_w]) | |
| tokens[idx + 1] = str(n) | |
| tokens[idx] = None | |
| # HALF HALFABLE and HALF | |
| # "meia duzia e meia"/"half a dozen and a half" -> 12/2 + 6/2 | |
| if next_next_w == "e" and next_next_next_w in cls._WORD2NUM_HALF: | |
| n0 = n + n / 2 | |
| tokens[idx + 1] = tokens[idx + 2] = None | |
| tokens[idx + 3] = str(n0) | |
| # "dois meios" -> "2 halfs" | |
| elif next_w in mapping: | |
| nn = 0.5 * float(mapping[next_w]) | |
| tokens[idx + 1] = str(0.5 * float(mapping[next_w])) | |
| tokens[idx] = None | |
| # handle numbers | |
| elif text in mapping: | |
| n = float(mapping[text]) | |
| # skip numbers, handled elsewhere (fractions) | |
| if ((next_next_next_w in mapping or next_next_w in cls._WORD2NUM) and | |
| next_next_next_w not in cls._WORD2NUM_HALFABLE): | |
| continue | |
| # N HALFABLE and HALF | |
| if is_number_word( | |
| prev_w) and next_w == "e" and next_next_w in cls._WORD2NUM_HALF and next_next_next_w not in cls._WORD2NUM_HALFABLE: | |
| n0 = word2number(prev_w, scale) | |
| tn = n0 * n + 0.5 * n | |
| tokens[idx - 1] = tokens[idx] = tokens[idx + 1] = None | |
| tokens[idx + 2] = str(tn) | |
| # ... and half HALFABLE | |
| elif next_w == "e" and next_next_w in cls._WORD2NUM_HALF and next_next_next_w in cls._WORD2NUM_HALFABLE: | |
| n1 = int(cls._WORD2NUM_HALFABLE[next_next_next_w]) | |
| # N N and half HALFABLE | |
| if is_number_word(prev_w, scale): | |
| n0 = word2number(prev_w, scale) | |
| tokens[idx + 3] = str(n0 * n + 0.5 * n1) | |
| tokens[idx] = tokens[idx + 1] = tokens[idx - 1] = tokens[idx + 2] = None | |
| # N and half HALFABLE | |
| else: | |
| tokens[idx + 3] = str(n + 0.5 * n1) | |
| tokens[idx] = tokens[idx + 1] = tokens[idx + 2] = None | |
| # "dois e meio" -> "2 and a half" | |
| elif next_w == "e" and next_next_w in cls._WORD2NUM_HALF: | |
| # N N and half | |
| if is_number_word(prev_w, scale): | |
| n0 = word2number(prev_w, scale) | |
| tokens[idx + 2] = str(n0 * n + 0.5 * n) | |
| tokens[idx] = tokens[idx - 1] = tokens[idx + 1] = None | |
| # N and half | |
| else: | |
| tokens[idx + 2] = str(n + 0.5 * n) | |
| tokens[idx] = tokens[idx + 1] = None | |
| return tokens | |
| @classmethod | |
| def _disambiguate_fractions(cls, tokens: List[str], | |
| scale: NumberScale = NumberScale.LONG_SCALE) -> List[str]: | |
| """Join fractions represented as words in the token list. | |
| Args: | |
| tokens (List[str]): A list of tokenized words. | |
| Returns: | |
| List[str]: The token list with fractions joined. | |
| """ | |
| mapping = cls._WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL | |
| if scale == NumberScale.SHORT_SCALE: | |
| mapping.update(cls._SHORT_AMBIGUOUS_PT) | |
| else: | |
| mapping.update(cls._LONG_AMBIGUOUS_PT) | |
| tokens = list(tokens) | |
| for idx, text in enumerate(tokens): | |
| if text is None: | |
| continue | |
| prev_w = tokens[idx - 1] if idx > 0 else "" | |
| next_w = tokens[idx + 1] if idx+ 1 < len(tokens) else "" | |
| if text in mapping: | |
| n = int(float(mapping[text][2:])) | |
| # special case, weekdays from monday friday have a ordinal in name | |
| if next_w in ["feira", "feiras"] and 1 < int(n) < 7: | |
| continue | |
| # N * ambiguous -> N / fraction | |
| # eg. "dois vigésimos" -> 2/20 | |
| if is_integer(prev_w): | |
| tokens[idx - 1] = None | |
| tokens[idx] = f"{prev_w}/{n}" | |
| # HALF ambiguous -> 0.5 * FRACTION | |
| # "meio vigésimo" -> 0.5 * 1/20 | |
| elif prev_w in cls._WORD2NUM_HALF: | |
| tokens[idx - 1] = None | |
| tokens[idx] = f"1/{2 * n}" | |
| return tokens | |
| @classmethod | |
| def _disambiguate_ordinals(cls, tokens: List[str], | |
| scale: NumberScale = NumberScale.LONG_SCALE, | |
| ignore_ordinals: bool = False, | |
| ignore_weekdays: bool = True) -> List[str]: | |
| """Disambiguate and join ordinal numbers in the token list. | |
| Args: | |
| tokens (List[str]): A list of tokenized words. | |
| Returns: | |
| List[str]: The token list with ordinals properly handled. | |
| """ | |
| tokens = list(tokens) | |
| start = None | |
| buffer = [] | |
| mapping = cls._WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL | |
| if scale == NumberScale.SHORT_SCALE: | |
| mapping.update(cls._SHORT_AMBIGUOUS_PT) | |
| else: | |
| mapping.update(cls._LONG_AMBIGUOUS_PT) | |
| for idx, text in enumerate(tokens): | |
| if text is None: | |
| continue | |
| next_w = tokens[idx + 1] if idx + 1 < len(tokens) else "" | |
| next_next_w = tokens[idx + 2] if idx + 2 < len(tokens) else "" | |
| # check for ordinal words, NOTE: ambiguos, might also be fractions! | |
| if text in mapping and next_w: | |
| n = int(float(mapping[text][2:])) | |
| if n % 10 == 0: | |
| if start is None: | |
| start = idx | |
| buffer.append(n) | |
| else: # its a fraction | |
| start = None | |
| buffer = [] | |
| # check if we reached the end of this path and should sum numbers | |
| # "vigésimo primeiro" -> 21 | |
| if buffer and next_w in cls._WORD2NUM_ORDINAL: | |
| n = sum(buffer) + int(cls._WORD2NUM_ORDINAL[next_w]) | |
| if ignore_ordinals or (ignore_weekdays and next_next_w in ["feira", "feiras"]): | |
| tokens[idx + 1] = None | |
| else: | |
| tokens[idx + 1] = str(n) | |
| for i in range(start or 0, idx + 1): | |
| tokens[i] = None | |
| return tokens | |
| @classmethod | |
| def _replace_fractions(cls, tokens: List[str], | |
| strategy: DisambiguationStrategy = DisambiguationStrategy.GUESS, | |
| scale: NumberScale = NumberScale.LONG_SCALE) -> List[str]: | |
| """Replace ambiguous words with fractions or ordinals based on strategy. | |
| Args: | |
| tokens (List[str]): A list of tokenized words. | |
| strategy (DisambiguationStrategy): Strategy to resolve ambiguity (assume fractions, ordinals, or guess). | |
| Returns: | |
| List[str]: The token list with ambiguous words resolved. | |
| """ | |
| mapping = cls._WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL | |
| if scale == NumberScale.SHORT_SCALE: | |
| mapping.update(cls._SHORT_AMBIGUOUS_PT) | |
| else: | |
| mapping.update(cls._LONG_AMBIGUOUS_PT) | |
| tokens = list(tokens) | |
| for idx, text in enumerate(tokens): | |
| if text is None: | |
| continue | |
| next_w = tokens[idx + 1] if idx + 1 < len(tokens) else "" | |
| next_next_w = tokens[idx + 2] if idx + 2 < len(tokens) else "" | |
| next_next_next_w = tokens[idx + 3] if idx + 3 < len(tokens) else "" | |
| # check for ordinal words | |
| if text in mapping: | |
| ordinals = False | |
| n = int(float(mapping[text][2:])) | |
| # special case, weekdays from monday friday have a ordinal in name | |
| if next_w in ["feira", "feiras"] and 1 < n < 7: | |
| continue | |
| if strategy == DisambiguationStrategy.ASSUME_FRACTIONS: | |
| ordinals = False | |
| elif strategy == DisambiguationStrategy.ASSUME_ORDINALS: | |
| ordinals = True | |
| elif next_w: # DisambiguationStrategy.GUESS | |
| # these are uncommon and ambiguous cases, but in these specific scenarios lets prefer ordinal numbers | |
| # ORDINAL e N (unlikely to spoken or to be mixing FRACTION e NUMBER) | |
| if next_w == "e" and (is_numeric(next_next_w) or | |
| is_fraction(next_next_w) or | |
| next_next_w in cls._WORD2NUM_HALF): | |
| ordinals = True | |
| # ORDINAL ORDINAL (unlikely to be mixing FRACTION e ORDINAL) | |
| elif next_w in cls._WORD2NUM_ORDINAL: | |
| ordinals = True | |
| # if followed by a number, it's likely a ordinal | |
| elif strategy == DisambiguationStrategy.GUESS and is_number_word(next_w, scale): | |
| ordinals = True | |
| if ordinals: | |
| tokens[idx] = mapping[text][2:] | |
| else: | |
| tokens[idx] = mapping[text] | |
| # check for integer numbers | |
| elif is_integer(text) and next_w: | |
| # special fractions above 11 | |
| # onze avos -> 1/11 | |
| if next_w == "avos": | |
| n = f"1/{text}" # represent as fraction instead | |
| tokens[idx + 1] = n | |
| tokens[idx] = None | |
| # quatro doze avos -> 4/12 | |
| elif next_next_w and next_w.isdigit() and next_next_w == "avos": | |
| n = int(text) / int(next_w) | |
| if isinstance(n, int): | |
| n = str(n) | |
| else: | |
| # if possible represent as fraction instead | |
| n = f"{text}/{next_w}" | |
| tokens[idx + 2] = n | |
| tokens[idx] = tokens[idx + 1] = None | |
| # check for fractions to sum (1 e 1/2 == 1.5) | |
| elif next_w == "e" and is_fraction(next_next_w): | |
| n1, n2 = next_next_w.split("/") | |
| f = int(n1) / int(n2) | |
| n = int(text) + f | |
| # TODO if possible represent as fraction instead | |
| n = str(n) | |
| tokens[idx + 2] = n | |
| tokens[idx] = tokens[idx + 1] = None | |
| # check for summable half (1 e meio -> 1.5) | |
| elif next_w == "e" and next_next_w in cls._WORD2NUM_HALF: | |
| n = int(text) + 0.5 | |
| # TODO if possible represent as fraction instead | |
| tokens[idx + 2] = str(n) | |
| tokens[idx] = tokens[idx + 1] = None | |
| # check for multipliable half (2 meios -> 1 * 0.5) | |
| elif next_w in cls._WORD2NUM_HALF: | |
| n = int(text) * 0.5 | |
| # TODO if possible represent as fraction instead | |
| tokens[idx + 1] = str(n) | |
| tokens[idx] = None | |
| # check for fractions to multiply ( 1 meio == 1/2) | |
| elif is_fraction(next_w): | |
| n1, n2 = next_w.split("/") | |
| f = float(n1) / float(n2) | |
| n = int(text) * f | |
| # TODO if possible represent as fraction instead | |
| tokens[idx + 1] = str(n) | |
| tokens[idx] = None | |
| # FRACTION of N | |
| if next_next_w == "de" and is_number_word(next_next_next_w, scale): | |
| n0 = word2number(next_next_next_w, scale) | |
| tokens[idx + 1] = tokens[idx + 2] = None | |
| tokens[idx + 3] = str(n * n0) | |
| return tokens | |
| @classmethod | |
| def _join_consecutive_numbers(cls, tokens: List[str]) -> List[str]: | |
| """Join consecutive numbers represented as words in the token list. | |
| Args: | |
| tokens (List[str]): A list of tokenized words. | |
| Returns: | |
| List[str]: The token list with consecutive numbers joined. | |
| """ | |
| tokens = list(tokens) | |
| for idx, text in enumerate(tokens): | |
| if text is None: | |
| continue | |
| next_w = tokens[idx + 1] if idx + 1 < len(tokens) else "" | |
| next_next_w = tokens[idx + 2] if idx + 2 < len(tokens) else "" | |
| next_next_next_w = tokens[idx + 3] if idx + 3 < len(tokens) else "" | |
| if text.endswith(".0"): | |
| text = text[:-2] | |
| if next_w and next_w.endswith(".0"): | |
| next_w = next_w[:-2] | |
| if next_next_w and next_next_w.endswith(".0"): | |
| next_next_w = next_next_w[:-2] | |
| # check for integer numbers | |
| if is_integer(text) and next_w: | |
| # check for punctuation | |
| if next_next_w and next_w in ["ponto", "vírgula"] and next_next_w.isdigit(): | |
| nums, last_idx = get_consecutive_integers(tokens, idx + 1) | |
| nums = "".join([str(_) for _ in nums]) | |
| tokens[last_idx] = f"{text}.{nums}" | |
| for i in range(idx, last_idx): | |
| tokens[i] = None | |
| # check for consecutive numbers that should be multiplied | |
| # eg, "2 1000" -> "2000" | |
| elif is_integer(next_w) and int(float(next_w)) > int(text) and int(text) != 0: | |
| tokens[idx + 1] = str(int(text) * int(float(next_w))) | |
| tokens[idx] = None | |
| # check for consecutive number words that should be multiplied | |
| # eg, "2 dozens" -> "2 * 12" | |
| # "mil milhares" -> 1000 * 1000 | |
| elif next_w in cls._WORD2NUM_HALFABLE and int(text) != 0: | |
| nn = cls._WORD2NUM_HALFABLE[next_w] | |
| tokens[idx + 1] = str(int(text) * int(nn)) | |
| tokens[idx] = None | |
| # check for consecutive numbers that should be added | |
| # eg, "1000 200" -> "1200" | |
| elif next_w.isdigit() and int(next_w) < int(text): | |
| tokens[idx + 1] = str(int(text) + int(float(next_w))) | |
| tokens[idx] = None | |
| # check for consecutive numbers that should be added with "e" binder word | |
| # eg, "20 e 2" -> "22" | |
| elif (next_w == "e" and | |
| next_next_w and next_next_w.isdigit() and | |
| int(text) >= 20 and | |
| int(next_next_w) < int(text)): | |
| tokens[idx + 2] = str(int(text) + int(next_next_w)) | |
| tokens[idx] = tokens[idx + 1] = None | |
| return tokens | |
| @classmethod | |
| def tag_numbers(cls, tokens: List[str], | |
| strategy: DisambiguationStrategy = DisambiguationStrategy.GUESS, | |
| scale: NumberScale = NumberScale.LONG_SCALE, | |
| ignore_ordinals: bool = False, | |
| ignore_weekdays: bool = True) -> Iterable[Span]: | |
| """Tag numbers in the given list of tokens and yield spans. | |
| Args: | |
| tokens (List[str]): The list of tokenized words to tag. | |
| strategy (DisambiguationStrategy): Strategy for handling ambiguous number formats. | |
| scale (NumberScale): Define whether to use LONG or SHORT scale for larger numbers. | |
| Yields: | |
| Iterable[Span]: Spans containing the original text and numeric values. | |
| """ | |
| originals = tokens | |
| # NOTE - order is important here | |
| tokens = cls._disambiguate_halfs(tokens) | |
| print(tokens) | |
| tokens = cls._replace_number_words(tokens, scale) | |
| print(tokens) | |
| tokens = cls._disambiguate_fractions(tokens, scale) # ordinal vs fraction, when we are sure it's fraction | |
| print(tokens) | |
| tokens = cls._disambiguate_ordinals(tokens, scale, | |
| ignore_ordinals, | |
| ignore_weekdays) # ordinal vs fraction, when we are sure it's ordinal | |
| print(tokens) | |
| tokens = cls._replace_fractions(tokens, strategy, scale) | |
| print(tokens) | |
| tokens = cls._join_consecutive_numbers(tokens) | |
| print(tokens) | |
| start = None | |
| for idx, text in enumerate(tokens): | |
| if text is None: | |
| if start is None: | |
| start = idx | |
| continue | |
| # check for ordinals | |
| if text in cls._WORD2NUM_ORDINAL: | |
| if ignore_ordinals: | |
| continue | |
| n = cls._WORD2NUM_ORDINAL[text] | |
| yield Span(idx, idx, text, int(n)) | |
| # check for fractions of the form N1/N2 | |
| elif "/" in text and len(text.split("/")) == 2: | |
| n1, n2 = text.split("/") | |
| if is_numeric(n1) and is_numeric(n2): | |
| t = " ".join(originals[start:idx + 1]) if start is not None and idx > start else originals[idx] | |
| yield Span(idx if start is None else start, idx, t, float(n1) / float(n2)) | |
| # check for integer numbers | |
| elif is_integer(text): | |
| t = " ".join(originals[start:idx + 1]) if start is not None and idx > start else originals[idx] | |
| yield Span(idx if start is None else start, idx, t, int(float(text))) | |
| # check for floating point numbers | |
| elif is_numeric(text): | |
| t = " ".join(originals[start:idx + 1]) if start is not None and idx > start else originals[idx] | |
| yield Span(idx if start is None else start, idx, t, float(text)) | |
| # check for literal "half" | |
| elif text in cls._WORD2NUM_HALF: | |
| yield Span(idx, idx, text, 0.5) | |
| start = None | |
| class PortugueseOrdinalNumberTagger: | |
| """A class for tagging ORDINAL numbers in Portuguese text. | |
| This class converts Portuguese number words and fractions into their | |
| numeric representations and identifies spans of text that represent | |
| numeric values. | |
| """ | |
| _WORD2NUM_ORDINAL = { | |
| "primeiro": "1", | |
| "segundo": "2", | |
| "terceiro": "3" | |
| } | |
| _WORD2NUM_ORDINAL.update({k[:-1] + "a": v # add female variants | |
| for k, v in _WORD2NUM_ORDINAL.items()}) | |
| _WORD2NUM_ORDINAL.update({k + "s": v # add plural variants | |
| for k, v in _WORD2NUM_ORDINAL.items()}) | |
| # can mean both a fraction or an ordinal number | |
| _WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL = { | |
| "quarto": "4", # Fourth | |
| "quinto": "5", # Fifth | |
| "sexto": "6", # Sixth | |
| "sétimo": "7", # Seventh | |
| "oitavo": "8", # Eighth | |
| "nono": "9", # Ninth | |
| "décimo": "10", # Tenth | |
| "undécimo": "11", # Eleventh | |
| "duodécimo": "12", # Twelfth | |
| # multiples of 10 | |
| "vigésimo": "20", # Twentieth | |
| "trigésimo": "30", # Thirtieth | |
| "quadragésimo": "40", # Fortieth | |
| "quinquagésimo": "50", # Fiftieth | |
| "sexagésimo": "60", # Sixtieth | |
| "septuagésimo": "70", # Seventieth | |
| "octogésimo": "80", # Eightieth | |
| "nonagésimo": "90", # Ninetieth | |
| # multiples of 100 | |
| "centésimo": "100", # Hundredth | |
| "duocentésimo": "200", # Two-hundredth | |
| "trecentésimo": "300", # Three-hundredth | |
| "quadringentésimo": "400", # Four-hundredth | |
| "quingentésimo": "500", # Five-hundredth | |
| "seiscentésimo": "600", # Six-hundredth | |
| "setecentésimo": "700", # Seven-hundredth | |
| "oitocentésimo": "800", # Eight-hundredth | |
| "novecentésimo": "900", # Nine-hundredth | |
| "milésimo": "1000" # Thousandth | |
| } | |
| _WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL.update({k[:-1] + "a": v # add female variants | |
| for k, v in _WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL.items()}) | |
| _WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL.update({k + "s": v # add plural variants | |
| for k, v in _WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL.items()}) | |
| _SHORT_AMBIGUOUS_PT = { | |
| "milionésimo": str(1e6), | |
| "bilionésimo": str(1e9), | |
| "trilionésimo": str(1e12), | |
| "quadrilionésimo": str(1e15), | |
| "quintilionésimo": str(1e18), | |
| "sextilionésimo": str(1e21), | |
| "septilionésimo": str(1e24), | |
| "octilionésimo": str(1e27), | |
| "nonilionésimo": str(1e30), | |
| "decilionésimo": str(1e33), | |
| "undecilionésimo": str(1e36), | |
| "duodecilionésimo": str(1e39), | |
| "tredecilionésimo": str(1e42), | |
| "quattuordecilionésimo": str(1e45), | |
| "quindecilionésimo": str(1e48), | |
| "sexdecilionésimo": str(1e51), | |
| "septendecilionésimo": str(1e54), | |
| "octodecilionésimo": str(1e57), | |
| "novemdecilionésimo": str(1e60), | |
| "vigintilionésimo": str(1e63), | |
| "unvigintilionésimo": str(1e66), | |
| "duovigintilionésimo": str(1e69), | |
| "trevigintilionésimo": str(1e72), | |
| "quattuorvigintilionésimo": str(1e75), | |
| "quinvigintilionésimo": str(1e78), | |
| "sexvigintilionésimo": str(1e81), | |
| "septenvigintilionésimo": str(1e84), | |
| "octovigintilionésimo": str(1e87), | |
| "novemvigintilionésimo": str(1e90), | |
| "trigintilionésimo": str(1e93) | |
| } | |
| _LONG_AMBIGUOUS_PT = { | |
| "milionésimo": str(1e6), | |
| "bilionésimo": str(1e12), | |
| "trilionésimo": str(1e18), | |
| "quadrilionésimo": str(1e24), | |
| "quintilionésimo": str(1e30), | |
| "sextilionésimo": str(1e36), | |
| "septilionésimo": str(1e42), | |
| "octilionésimo": str(1e48), | |
| "nonilionésimo": str(1e54), | |
| "decilionésimo": str(1e60), | |
| "undecilionésimo": str(1e66), | |
| "duodecilionésimo": str(1e72), | |
| "tredecilionésimo": str(1e78), | |
| "quattuordecilionésimo": str(1e84), | |
| "quindecilionésimo": str(1e90), | |
| "sexdecilionésimo": str(1e96), | |
| "septendecilionésimo": str(1e102), | |
| "octodecilionésimo": str(1e108), | |
| "novemdecilionésimo": str(1e114), | |
| "vigintilionésimo": str(1e120), | |
| "unvigintilionésimo": str(1e126), | |
| "duovigintilionésimo": str(1e132), | |
| "trevigintilionésimo": str(1e138), | |
| "quattuorvigintilionésimo": str(1e144), | |
| "quinvigintilionésimo": str(1e150), | |
| "sexvigintilionésimo": str(1e156), | |
| "septenvigintilionésimo": str(1e162), | |
| "octovigintilionésimo": str(1e168), | |
| "novemvigintilionésimo": str(1e174), | |
| "trigintilionésimo": str(1e180) | |
| } | |
| @classmethod | |
| def _is_fraction(cls, tokens: List[str], idx: int, | |
| scale: NumberScale = NumberScale.LONG_SCALE): | |
| mapping = cls._WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL | |
| if scale == NumberScale.SHORT_SCALE: | |
| mapping.update(cls._SHORT_AMBIGUOUS_PT) | |
| else: | |
| mapping.update(cls._LONG_AMBIGUOUS_PT) | |
| prev = tokens[idx - 1] if idx > 0 else "" | |
| next_w = tokens[idx + 1] if idx + 1 < len(tokens) else "" | |
| w = tokens[idx] | |
| if w in mapping: | |
| # no negative ordinals | |
| if prev == "menos": | |
| return True | |
| # no half ordinals | |
| # "meio vigésimo" -> 0.5 * 1/20 | |
| if prev in PortugueseNumberTagger._WORD2NUM_HALF: | |
| return True | |
| # "a quarta parte" | |
| if next_w == "parte": | |
| return True | |
| # week days are called "ORDINAL feira" starting at monday (2) until friday (6) | |
| if next_w in ["feira", "feiras"] and 1 < int(mapping[w]) < 7: | |
| return False | |
| # if there is a cardinal number before, then it is a fraction being multiplied by it | |
| # "dois quartos"/"two quarters" -> 2 * 1/4 | |
| if prev in PortugueseNumberTagger._WORD2NUM: | |
| return True | |
| return False | |
| @classmethod | |
| def tag_numbers(cls, tokens: List[str], | |
| scale: NumberScale = NumberScale.LONG_SCALE, | |
| ignore_weekdays: bool = True) -> Iterable[OrdinalNumberSpan]: | |
| """Tag numbers in the given list of tokens and yield spans. | |
| Args: | |
| tokens (List[str]): The list of tokenized words to tag. | |
| scale (NumberScale): Define whether to use LONG or SHORT scale for larger numbers. | |
| Yields: | |
| Iterable[Span]: Spans containing the original text and numeric values. | |
| """ | |
| originals = list(tokens) | |
| mapping = cls._WORD2NUM_ORDINAL | |
| mapping.update(cls._WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL) | |
| if scale == NumberScale.SHORT_SCALE: | |
| mapping.update(cls._SHORT_AMBIGUOUS_PT) | |
| else: | |
| mapping.update(cls._LONG_AMBIGUOUS_PT) | |
| for idx, text in enumerate(tokens): | |
| if cls._is_fraction(tokens, idx, scale): | |
| continue | |
| next_w = tokens[idx + 1] if idx + 1 < len(tokens) else "" | |
| # check for ordinals | |
| if text in mapping or isinstance(text, int): | |
| if isinstance(text, int): | |
| n = text | |
| else: | |
| n = int(float(mapping[text])) | |
| tokens[idx] = n | |
| if next_w in mapping: | |
| n2 = int(mapping[next_w]) | |
| if (n2 < n and | |
| n not in [11, 12]): # special naming for these 2 | |
| tokens[idx] = None | |
| tokens[idx + 1] = n + n2 | |
| start = None | |
| for idx, t in enumerate(tokens): | |
| if t is None and start is None: | |
| start = idx | |
| if t is not None and not isinstance(t, str): | |
| if start is None: | |
| text = originals[idx] | |
| ambiguous = (text in cls._SHORT_AMBIGUOUS_PT or | |
| text in cls._WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL) | |
| else: | |
| text = " ".join(originals[start:idx + 1]) | |
| ambiguous = (any([_ in cls._SHORT_AMBIGUOUS_PT | |
| for _ in originals[start:idx + 1]]) or | |
| any([_ in cls._WORD2NUM_AMBIGUOUS_FRACTION_ORDINAL | |
| for _ in originals[start:idx + 1]])) | |
| if any([_ in cls._WORD2NUM_ORDINAL] for _ in originals[start:idx + 1]): | |
| ambiguous = False | |
| # week days are called "ORDINAL feira" starting at monday (2) until friday (6) | |
| next_w = tokens[idx + 1] if idx + 1 < len(tokens) else "" | |
| if next_w in ["feira", "feiras"] and 1 < t < 7: | |
| ambiguous = False | |
| if ignore_weekdays: | |
| continue | |
| # print(text, ambiguous) | |
| yield OrdinalNumberSpan(start_token_idx=start if start is not None else idx, | |
| end_token_idx=idx, text=text, number=t, is_ambiguous=ambiguous) | |
| start = None | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import unittest | |
| from quebra_frases import word_tokenize | |
| from numerico import PortugueseNumberTagger, DisambiguationStrategy, NumberScale | |
| class TestPortugueseNumberTagger(unittest.TestCase): | |
| def setUp(self): | |
| self.tagger = PortugueseNumberTagger() | |
| def test_number_tagging(self): | |
| test_cases = [ | |
| ("teste numero dois", [(2, 2, 'dois', 2)]), | |
| ("duas coisas", [(0, 0, 'duas', 2)]), | |
| ("doze avos", [(0, 1, 'doze avos', 1 / 12)]), | |
| ("quinze doze avos", [(0, 2, 'quinze doze avos', 15 / 12)]), | |
| ("teste numero 1", [(2, 2, '1', 1)]), | |
| ("teste numero 1.5", [(2, 2, '1.5', 1.5)]), | |
| ("teste numero 1/2", [(2, 2, '1/2', 0.5)]), | |
| ("teste numero 1 e meio", [(2, 4, '1 e meio', 1.5)]), | |
| ("teste numero 1 meio", [(2, 3, '1 meio', 0.5)]), | |
| ("teste numero 2 e 2", [(2, 2, '2', 2), | |
| (4, 4, '2', 2)]), | |
| ("teste numero 20 e 2", [(2, 4, '20 e 2', 20 + 2)]), | |
| ("teste numero vinte e 2", [(2, 4, 'vinte e 2', 20 + 2)]), | |
| ("teste numero dois mil", [(2, 3, 'dois mil', 2000)]), | |
| ("teste numero mil duzentos e vinte e dois", [(2, 7, 'mil duzentos e vinte e dois', 1222)]), | |
| ("teste numero quatro mil duzentos e vinte e dois", [(2, 8, 'quatro mil duzentos e vinte e dois', 4222)]), | |
| ("um vigésimo", [(0, 1, 'um vigésimo', 1 / 20)]), | |
| ("uma vigésima", [(0, 1, 'uma vigésima', 1 / 20)]), | |
| ("dois vigésimos", [(0, 1, 'dois vigésimos', 2 / 20)]), | |
| ("duas vigésimas", [(0, 1, 'duas vigésimas', 2 / 20)]), | |
| ("vigésimo primeiro", [(0, 1, 'vigésimo primeiro', 21)]), | |
| ("vigésimo e meio", [(0, 0, 'vigésimo', 20), | |
| (2, 2, 'meio', 0.5)]), | |
| ("duas metades", [(0, 1, 'duas metades', 1)]), | |
| ("menos quatro", [(0, 1, 'menos quatro', -4)]), | |
| ("um ponto dois", [(0, 2, 'um ponto dois', 1.2)]), | |
| ("três vírgula catorze", [(0, 2, 'três vírgula catorze', 3.14)]), | |
| ("menos três vírgula quatro", [(0, 3, 'menos três vírgula quatro', -3.4)]), | |
| ("três menos doze avos", [(0, 0, 'três', 3), | |
| (1, 3, 'menos doze avos', -1 / 12)]), | |
| ("menos três doze avos", [(0, 3, 'menos três doze avos', -3 / 12)]), | |
| ("três vírgula menos sete", [(0, 0, 'três', 3), | |
| (2, 3, 'menos sete', -7)]), | |
| ("dois e vinte", [(0, 0, 'dois', 2), | |
| (2, 2, 'vinte', 20)]), | |
| ("cinco e mil", [(0, 0, 'cinco', 5), | |
| (2, 2, 'mil', 1000)]), | |
| ("cinco mil", [(0, 1, 'cinco mil', 5000)]), | |
| ("um milhar", [(0, 1, 'um milhar', 1000)]), | |
| ("dois milhares", [(0, 1, 'dois milhares', 2000)]), | |
| ] | |
| for input_str, expected in test_cases: | |
| with self.subTest(input=input_str): | |
| toks = word_tokenize(input_str) | |
| spans = list(self.tagger.tag_numbers(toks)) | |
| print(spans) | |
| self.assertEqual(len(spans), len(expected)) | |
| for (start_idx, end_idx, text, number), span in zip(expected, spans): | |
| print(span) | |
| self.assertEqual(span.start_token_idx, start_idx) | |
| self.assertEqual(span.end_token_idx, end_idx) | |
| self.assertEqual(span.text, text) | |
| self.assertEqual(span.number, number) | |
| def test_halfs(self): | |
| cases = [ | |
| "um milhão e meio", | |
| "centena e meia", | |
| "um milhão e meia dúzia", | |
| "meio milhão e meia dúzia", | |
| "dúzia e meia", | |
| "meio milhão", | |
| "dez e meio", | |
| "uma dúzia", | |
| "duas dúzias", | |
| "meia dúzia", | |
| "uma centena", | |
| "meia centena", | |
| "meio milhar de euros", | |
| "um meio", | |
| "dois meios", | |
| "uma e meia", | |
| "vinte e meio", | |
| "duas dúzias e meia", | |
| "meia dúzia e meia", | |
| "um milhão e meia dúzia", | |
| ] | |
| s = NumberScale.LONG_SCALE | |
| target = [ | |
| [(0, 3, 'um milhão e meio', 1500000.0)], | |
| [(0, 2, 'centena e meia', 150.0)], | |
| [(0, 4, 'um milhão e meia dúzia', 1000006.0)], | |
| [(0, 4, 'meio milhão e meia dúzia', 500006.0)], | |
| [(0, 2, 'dúzia e meia', 18.0)], | |
| [(0, 1, 'meio milhão', 500000.0)], | |
| [(0, 2, 'dez e meio', 10.5)], | |
| [(0, 1, 'uma dúzia', 12)], | |
| [(0, 1, 'duas dúzias', 24)], | |
| [(0, 1, 'meia dúzia', 6.0)], | |
| [(0, 1, 'uma centena', 100)], | |
| [(0, 1, 'meia centena', 50.0)], | |
| [(0, 1, 'meio milhar', 500.0)], | |
| [(0, 1, 'um meio', 0.5)], | |
| [(0, 1, 'dois meios', 1.0)], | |
| [(0, 2, 'uma e meia', 1.5)], | |
| [(0, 2, 'vinte e meio', 20.5)], | |
| [(0, 3, 'duas dúzias e meia', 30.0)], | |
| [(0, 3, 'meia dúzia e meia', 9.0)], | |
| [(0, 4, 'um milhão e meia dúzia', 1000006.0)] | |
| ] | |
| for input_str, expected in zip(cases, target): | |
| with self.subTest(input=input_str): | |
| toks = word_tokenize(input_str) | |
| spans = list(self.tagger.tag_numbers(toks, scale=s)) | |
| self.assertEqual(len(spans), len(expected)) | |
| for (start_idx, end_idx, text, number), span in zip(expected, spans): | |
| self.assertEqual(span.start_token_idx, start_idx) | |
| self.assertEqual(span.end_token_idx, end_idx) | |
| self.assertEqual(span.text, text) | |
| self.assertEqual(span.number, number) | |
| def test_scale(self): | |
| cases = ["um milhão", | |
| "mil milhões", | |
| "dois biliões", | |
| "um bilião", | |
| "um bilhão", | |
| "dois bilhões", | |
| "mil biliões", | |
| "um trilião" | |
| ] | |
| s = NumberScale.LONG_SCALE | |
| target = [ | |
| [(0, 1, 'um milhão', 1000000)], | |
| [(0, 1, 'mil milhões', 1000000000)], | |
| [(0, 1, 'dois biliões', 2000000000000)], | |
| [(0, 1, 'um bilião', 1000000000000)], | |
| [(0, 1, 'um bilhão', 1000000000000)], | |
| [(0, 1, 'dois bilhões', 2000000000000)], | |
| [(0, 1, 'mil biliões', 1000 * 1e12)], | |
| [(0, 1, 'um trilião', 1e18)], | |
| ] | |
| for input_str, expected in zip(cases, target): | |
| with self.subTest(input=input_str): | |
| toks = word_tokenize(input_str) | |
| spans = list(self.tagger.tag_numbers(toks, scale=s)) | |
| print(spans) | |
| self.assertEqual(len(spans), len(expected)) | |
| for (start_idx, end_idx, text, number), span in zip(expected, spans): | |
| self.assertEqual(span.start_token_idx, start_idx) | |
| self.assertEqual(span.end_token_idx, end_idx) | |
| self.assertEqual(span.text, text) | |
| self.assertEqual(span.number, number) | |
| s = NumberScale.SHORT_SCALE | |
| target = [ | |
| [(0, 1, 'um milhão', 1000000)], | |
| [(0, 1, 'mil milhões', 1000000000)], | |
| [(0, 1, 'dois biliões', 2000000000)], | |
| [(0, 1, 'um bilião', 1000000000)], | |
| [(0, 1, 'um bilhão', 1000000000)], | |
| [(0, 1, 'dois bilhões', 2000000000)], | |
| [(0, 1, 'mil biliões', 1000 * 1e9)], | |
| [(0, 1, 'um trilião', 1e12)], | |
| ] | |
| for input_str, expected in zip(cases, target): | |
| with self.subTest(input=input_str): | |
| toks = word_tokenize(input_str) | |
| spans = list(self.tagger.tag_numbers(toks, scale=s)) | |
| self.assertEqual(len(spans), len(expected)) | |
| for (start_idx, end_idx, text, number), span in zip(expected, spans): | |
| self.assertEqual(span.start_token_idx, start_idx) | |
| self.assertEqual(span.end_token_idx, end_idx) | |
| self.assertEqual(span.text, text) | |
| self.assertEqual(span.number, number) | |
| def test_strats(self): | |
| cases = [ | |
| "vigésimo primeiro", # always ordinal | |
| "duodécimo primeiro", # AMBIGUOUS + ORDINAL | |
| "vigésimo e primeiro", # AMBIGUOUS e ORDINAL | |
| "vigésimo e um", # AMBIGUOUS e INT | |
| "vigésimo", # AMBIGUOUS | |
| "décimo", # AMBIGUOUS | |
| ] | |
| st = DisambiguationStrategy.GUESS | |
| target = [ | |
| [(0, 1, 'vigésimo primeiro', 21)], | |
| [(0, 0, 'duodécimo', 12), (1, 1, 'primeiro', 1)], | |
| [(0, 0, 'vigésimo', 0.05), (2, 2, 'primeiro', 1)], | |
| [(0, 2, 'vigésimo e um', 21)], | |
| [(0, 0, 'vigésimo', 0.05)], | |
| [(0, 0, 'décimo', 0.1)] | |
| ] | |
| for input_str, expected in zip(cases, target): | |
| with self.subTest(input=input_str): | |
| toks = word_tokenize(input_str) | |
| spans = list(self.tagger.tag_numbers(toks, strategy=st)) | |
| self.assertEqual(len(spans), len(expected)) | |
| for (start_idx, end_idx, text, number), span in zip(expected, spans): | |
| self.assertEqual(span.start_token_idx, start_idx) | |
| self.assertEqual(span.end_token_idx, end_idx) | |
| self.assertEqual(span.text, text) | |
| self.assertEqual(span.number, number) | |
| st = DisambiguationStrategy.ASSUME_ORDINALS | |
| target = [ | |
| [(0, 1, 'vigésimo primeiro', 21)], | |
| [(0, 0, 'duodécimo', 12), (1, 1, 'primeiro', 1)], | |
| [(0, 0, 'vigésimo', 20), (2, 2, 'primeiro', 1)], | |
| [(0, 2, 'vigésimo e um', 21)], | |
| [(0, 0, 'vigésimo', 20)], | |
| [(0, 0, 'décimo', 10)] | |
| ] | |
| for input_str, expected in zip(cases, target): | |
| with self.subTest(input=input_str): | |
| toks = word_tokenize(input_str) | |
| spans = list(self.tagger.tag_numbers(toks, strategy=st)) | |
| self.assertEqual(len(spans), len(expected)) | |
| for (start_idx, end_idx, text, number), span in zip(expected, spans): | |
| self.assertEqual(span.start_token_idx, start_idx) | |
| self.assertEqual(span.end_token_idx, end_idx) | |
| self.assertEqual(span.text, text) | |
| self.assertEqual(span.number, number) | |
| st = DisambiguationStrategy.ASSUME_FRACTIONS | |
| target = [ | |
| [(0, 1, 'vigésimo primeiro', 21)], | |
| [(0, 0, 'duodécimo', 0.08333333333333333), (1, 1, 'primeiro', 1)], | |
| [(0, 0, 'vigésimo', 0.05), (2, 2, 'primeiro', 1)], | |
| [(0, 0, 'vigésimo', 0.05), (2, 2, 'um', 1)], | |
| [(0, 0, 'vigésimo', 0.05)], | |
| [(0, 0, 'décimo', 0.1)] | |
| ] | |
| for input_str, expected in zip(cases, target): | |
| with self.subTest(input=input_str): | |
| toks = word_tokenize(input_str) | |
| spans = list(self.tagger.tag_numbers(toks, strategy=st)) | |
| self.assertEqual(len(spans), len(expected)) | |
| for (start_idx, end_idx, text, number), span in zip(expected, spans): | |
| self.assertEqual(span.start_token_idx, start_idx) | |
| self.assertEqual(span.end_token_idx, end_idx) | |
| self.assertEqual(span.text, text) | |
| self.assertEqual(span.number, number) | |
| def test_edge_cases(self): | |
| test_cases = [ | |
| # Fractions and ordinals | |
| ("um e meio", [(0, 2, 'um e meio', 1.5)]), | |
| ("um meio", [(0, 1, 'um meio', 0.5)]), | |
| ("primeiro e meio", [(0, 0, 'primeiro', 1), (2, 2, 'meio', 0.5)]), | |
| ("um vigésimo", [(0, 1, 'um vigésimo', 1 / 20)]), | |
| ("meio vigésimo", [(0, 1, 'meio vigésimo', 1 / 40)]), | |
| ("um terço de um", [(0, 3, 'um terço de um', 1 / 3)]), | |
| ("um terço de dois", [(0, 3, 'um terço de dois', 2 / 3)]), | |
| ("duas oitavas partes", [(0, 1, 'duas oitavas', 2 / 8)]), | |
| ("um menos um terço", [(0, 0, 'um', 1), (1, 3, 'menos um terço', -1 / 3)]), | |
| # Large numbers and scale handling | |
| ("mil milhões", [(0, 1, 'mil milhões', 1000000000)]), | |
| ("mil biliões", [(0, 1, 'mil biliões', 1e15)]), # for long scale | |
| ("trilião", [(0, 0, 'trilião', 1e18)]), # for long scale | |
| ("um trilião", [(0, 1, 'um trilião', 1e18)]), # for long scale | |
| #("dois triliões", [(0, 1, 'dois triliões', 1e18)]), # for long scale | |
| ("um milhar", [(0, 1, 'um milhar', 1000)]), | |
| ("mil milhares", [(0, 1, 'mil milhares', 1000000)]), | |
| ("um milésimo", [(0, 1, 'um milésimo', 1 / 1000)]), | |
| # Decimals | |
| ("um ponto zero zero um", [(0, 4, 'um ponto zero zero um', 1.001)]), | |
| ("três vírgula zero zero cinco", [(0, 4, 'três vírgula zero zero cinco', 3.005)]), | |
| ("zero ponto um", [(0, 2, 'zero ponto um', 0.1)]), | |
| ("zero ponto zero zero um", [(0, 4, 'zero ponto zero zero um', 0.001)]), | |
| # Negative numbers | |
| ("menos meio milhão", [(0, 2, 'menos meio milhão', -500000)]), | |
| ("menos um ponto cinco", [(0, 3, 'menos um ponto cinco', -1.5)]), | |
| ("menos zero", [(0, 1, 'menos zero', 0)]), # check if zero remains 0 | |
| # Ordinals | |
| ("milésimo primeiro", [(0, 1, 'milésimo primeiro', 1001)]), | |
| #("décimo menos um", [(0, 0, 'décimo', 10), (1, 2, 'menos um', -1)]), | |
| #("vigésimo um menos dois", [(0, 0, 'vigésimo', 20), (1, 1, 'um', 1), (2, 3, 'menos dois', -2)]), | |
| # Gender handling | |
| ("uma centena e meia", [(0, 3, 'uma centena e meia', 150)]), | |
| ("duas centenas", [(0, 1, 'duas centenas', 200)]), | |
| ("milhar e meio", [(0, 2, 'milhar e meio', 1500)]), | |
| ("duas dúzias e meia", [(0, 3, 'duas dúzias e meia', 30)]) | |
| ] | |
| for input_str, expected in test_cases: | |
| with self.subTest(input=input_str): | |
| toks = word_tokenize(input_str) | |
| spans = list(self.tagger.tag_numbers(toks)) | |
| print(spans) | |
| self.assertEqual(len(spans), len(expected)) | |
| for (start_idx, end_idx, text, number), span in zip(expected, spans): | |
| self.assertEqual(span.start_token_idx, start_idx) | |
| self.assertEqual(span.end_token_idx, end_idx) | |
| self.assertEqual(span.text, text) | |
| self.assertEqual(span.number, number) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment