Created
May 16, 2025 15:30
-
-
Save undefined06855/963a3a23db0b9c58e5a7c9a4aa76e2f8 to your computer and use it in GitHub Desktop.
slow af python json parser
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from typing import Union | |
| class JSON: | |
| Number = Union[int, float] | |
| class UnexpectedTokenError(Exception): | |
| pass | |
| class UnexpectedEOFError(Exception): | |
| pass | |
| class Config(): | |
| def __init__(self): | |
| self.allow_comments:bool = False | |
| self.allow_invalid_numbers:bool = False | |
| self.allow_invalid_literals:bool = False | |
| self.no_throw:bool = False | |
| def __init__(self): | |
| self.string_data:str = "" | |
| self.head:int = 0 | |
| self.res:any = {} | |
| self.config:Config = self.Config() | |
| # first int is line, second int is column (though i think this is wrong?) | |
| # used in exceptions | |
| def count_lines(self, index:int) -> [int, int]: | |
| slice = self.string_data[0:index] | |
| line = 1 | |
| for char in slice: | |
| if self.is_newline(char): | |
| line += 1 | |
| return [line, index % (line-1)] | |
| # used in exceptions | |
| def formatted_head(self) -> str: | |
| line_data = self.count_lines(self.head) | |
| return f"line {line_data[0]} column {line_data[1]} (char {self.head})" | |
| def parse(self, data:str) -> any: | |
| self.string_data = data | |
| self.head = 0 | |
| self.res = {} | |
| if len(data) < 2: | |
| raise self.UnexpectedEOFError(f"Length of data is too short to contain valid JSON data!") | |
| self.consume_whitespace() | |
| try: | |
| # do the consuming | |
| self.res = self.consume_any() | |
| except IndexError as err: | |
| if not self.config.no_throw: | |
| if self.head == len(self.string_data): | |
| raise self.UnexpectedEOFError(f"Found EOF while parsing character at {self.formatted_head()}!") from err | |
| else: | |
| raise err | |
| except Exception as err: | |
| if self.config.no_throw: | |
| print(f"Error: {err}") | |
| else: | |
| raise err | |
| # any extra data is ignored | |
| return self.res | |
| def expect(self, string:str) -> bool: | |
| start = self.head | |
| end = self.head + len(string) | |
| slice = self.string_data[start:end] | |
| if slice != string: | |
| raise self.UnexpectedTokenError(f"Expected {string}, found {slice} at {self.formatted_head()}!") | |
| return False | |
| self.head += len(string) | |
| return True | |
| def expect_perhaps(self, string:str) -> bool: | |
| if self.upcoming_is(string): | |
| self.expect(string) | |
| return True | |
| return False | |
| def upcoming_is(self, string:str) -> bool: | |
| start = self.head | |
| end = self.head + len(string) | |
| slice = self.string_data[start:end] | |
| return slice == string | |
| def is_whitespace(self, string:str) -> bool: | |
| return string == " " or string == "\t" or self.is_newline(string) | |
| def is_newline(self, string:str) -> bool: | |
| return string == "\n" or string == "\r" | |
| def is_number(self, string:str) -> bool: | |
| return ord(string) >= ord("0") and ord(string) <= ord("9") | |
| def is_potential_number(self, string:str) -> bool: | |
| return self.is_number(string) or string == "-" | |
| def is_number_part(self, string:str) -> bool: | |
| return self.is_number(string) or string in [ ".", "E", "e", "+", "-" ] | |
| def consume_any(self) -> any: | |
| if self.upcoming_is("\""): | |
| return self.consume_string() | |
| elif self.upcoming_is("["): | |
| return self.consume_array() | |
| elif self.upcoming_is("{"): | |
| return self.consume_object() | |
| elif self.is_potential_number(self.string_data[self.head]): | |
| return self.consume_number() | |
| else: | |
| return self.consume_literal() | |
| def consume_whitespace(self) -> None: | |
| while self.is_whitespace(self.string_data[self.head]): | |
| self.head += 1 | |
| if self.upcoming_is("//") and self.config.allow_comments: | |
| self.consume_comment() | |
| self.consume_whitespace() | |
| # consume commets starting with // up to a newline | |
| def consume_comment(self) -> None: | |
| self.expect("//") | |
| while not self.is_newline(self.string_data[self.head]): | |
| self.head += 1 | |
| # consume numbers | |
| def consume_number(self) -> Number: | |
| sign = 1 | |
| # negative sign | |
| if self.expect_perhaps("-"): | |
| sign = -sign | |
| if not self.is_number(self.string_data[self.head]): | |
| raise self.UnexpectedTokenError(f"Expected number, found {self.string_data[self.head]} at {self.formatted_head()}") | |
| # main number parsing | |
| out = 0 | |
| found_decimal_point = False | |
| found_exponent = False | |
| found_any_numbers = False | |
| decimal_position = 0 | |
| exponent_part = 0 | |
| exponent_sign = 1 | |
| just_found_decimal = False | |
| just_found_exponent = False | |
| starts_with_zero = False | |
| while self.is_number_part(self.string_data[self.head]): | |
| # normal number char | |
| if self.is_number(self.string_data[self.head]): | |
| if starts_with_zero and not found_decimal_point and not found_exponent and not self.config.allow_invalid_numbers: | |
| # invalid! | |
| raise self.UnexpectedTokenError(f"Expected no more numbers following after a zero, found {self.string_data[self.head]} at {self.formatted_head()}!") | |
| if self.string_data[self.head] == "0" and not found_any_numbers: | |
| starts_with_zero = True | |
| if found_exponent: | |
| exponent_part *= 10 | |
| exponent_part += int(self.string_data[self.head]) | |
| else: | |
| out *= 10 | |
| out += int(self.string_data[self.head]) | |
| if found_decimal_point: | |
| decimal_position += 1 | |
| just_found_decimal = False | |
| just_found_exponent = False | |
| found_any_numbers = True | |
| # should have a number here | |
| elif just_found_decimal and not self.config.allow_invalid_numbers: | |
| raise self.UnexpectedTokenError(f"Expected number after decimal, found non-number at {self.formatted_head()}") | |
| # should have a number here but might be + or - | |
| elif just_found_exponent: | |
| if self.string_data[self.head] == "-": | |
| # negative exponent | |
| exponent_sign = -exponent_sign | |
| elif self.string_data[self.head] == "+": | |
| # ignore | |
| pass | |
| elif not self.config.allow_invalid_numbers: | |
| raise self.UnexpectedTokenError(f"Expected number after exponent, found non-number at {self.formatted_head()}") | |
| # decimal point | |
| elif self.string_data[self.head] == ".": | |
| if not found_any_numbers and not self.config.allow_invalid_numbers: | |
| # numbers cant start with . | |
| raise self.UnexpectedTokenError(f"Expected number, found . at {self.formatted_head()}") | |
| if found_decimal_point and not self.config.allow_invalid_numbers: | |
| # second . in this number | |
| raise self.UnexpectedTokenError(f"Expected number, found (second) . at {self.formatted_head()}") | |
| if found_exponent and not self.config.allow_invalid_numbers: | |
| # what | |
| raise self.UnexpectedTokenError(f"Expected number, found . after finding exponent at {self.formatted_head()}") | |
| found_decimal_point = True | |
| just_found_decimal = True | |
| # exponent | |
| elif self.string_data[self.head] == "e" or self.string_data[self.head] == "E": | |
| if not found_any_numbers and not self.config.allow_invalid_numbers: | |
| # numbers cant start with exponent | |
| raise self.UnexpectedTokenError(f"Expected number, found exponent at {self.formatted_head()}") | |
| if found_exponent and not self.config.allow_invalid_numbers: | |
| # second e in this number | |
| raise self.UnexpectedTokenError(f"Expected number, found (second) exponent at {self.formatted_head()}") | |
| found_exponent = True | |
| just_found_exponent = True | |
| self.head += 1 | |
| if (just_found_decimal or just_found_exponent) and not self.config.allow_invalid_numbers: | |
| raise self.UnexpectedTokenError(f"Expected number, found end of number at {self.formatted_head()}") | |
| out /= 10 ** decimal_position | |
| out *= 10 ** (exponent_part * exponent_sign) | |
| out *= sign | |
| # check if this is an integer | |
| if not found_decimal_point and not found_exponent: | |
| out = int(out) | |
| return out | |
| # consume things that aren't wrapped in a special character | |
| def consume_literal(self) -> Union[bool, None]: | |
| if self.expect_perhaps("true"): | |
| return True | |
| if self.expect_perhaps("false"): | |
| return False | |
| if self.expect_perhaps("null"): | |
| return None | |
| # attempt to parse invalid literal as string | |
| if self.config.allow_invalid_literals: | |
| ret = "" | |
| while self.string_data[self.head] not in [ ",", "]", "}" ] and not self.is_whitespace(self.string_data[self.head]): | |
| ret += self.string_data[self.head] | |
| self.head += 1 | |
| return ret | |
| raise self.UnexpectedTokenError(f"Expected literal, found {self.string_data[self.head]} at {self.formatted_head()}!") | |
| # consume arrays, starting with [ | |
| def consume_array(self) -> list[any]: | |
| self.expect("[") | |
| self.consume_whitespace() | |
| # empty! | |
| if self.expect_perhaps("]"): | |
| return [] | |
| out = [] | |
| while True: | |
| if self.expect_perhaps("]"): | |
| # potential trailing comma | |
| return out | |
| # consume something | |
| value = self.consume_any() | |
| out.append(value) | |
| self.consume_whitespace() | |
| has_comma = self.expect_perhaps(",") | |
| self.consume_whitespace() | |
| # if there isnt a comma, end of array | |
| if not has_comma: | |
| self.expect("]") | |
| return out | |
| # consume objects starting with { | |
| def consume_object(self) -> dict[str, any]: | |
| self.expect("{") | |
| self.consume_whitespace() | |
| # empty! | |
| if self.expect_perhaps("}"): | |
| return {} | |
| out = {} | |
| while True: | |
| if self.expect_perhaps("}"): | |
| # potential trailing comma | |
| return out | |
| # consume key | |
| key = self.consume_string() | |
| self.expect(":") | |
| self.consume_whitespace() | |
| # consume value | |
| value = self.consume_any() | |
| out[key] = value | |
| self.consume_whitespace() | |
| has_comma = self.expect_perhaps(",") | |
| self.consume_whitespace() | |
| # if there isnt a comma, end of object | |
| if not has_comma: | |
| self.expect("}") | |
| return out | |
| # consume strings starting with " | |
| def consume_string(self) -> str: | |
| self.expect("\"") | |
| # empty! | |
| if self.expect_perhaps("\""): | |
| return "" | |
| out = "" | |
| while True: | |
| out += self.string_data[self.head] | |
| # skip over next char for now | |
| # TODO: parse \n and such | |
| if self.expect_perhaps("\\"): | |
| self.head += 1 | |
| # if there is end quote, end string | |
| if self.expect_perhaps("\""): | |
| return out | |
| self.head += 1 | |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
(it is faster than json.loads if your input json is
""!!)