some more refactoring to make the code more readable and less error prone

Author: mangiucugna

src/json_repair/json_parser.py

@@ -133,23 +133,29 @@ def get_char_at(self, count: int = 0) -> str | None:
         except IndexError:
             return None
 
-    def skip_whitespaces_at(self, idx: int = 0, move_main_index: bool = True) -> int:
+    def skip_whitespaces(self) -> None:
         """
-        This function quickly iterates on whitespaces, syntactic sugar to make the code more concise
+        This function quickly iterates on whitespaces, moving the self.index forward
         """
         try:
-            char = self.json_str[self.index + idx]
-        except IndexError:
-            return idx
-        while char.isspace():
-            if move_main_index:
+            char = self.json_str[self.index]
+            while char.isspace():
                 self.index += 1
-            else:
+                char = self.json_str[self.index]
+        except IndexError:
+            pass
+
+    def scroll_whitespaces(self, idx: int = 0) -> int:
+        """
+        This function quickly iterates on whitespaces. Doesn't move the self.index and returns the offset from self.index
+        """
+        try:
+            char = self.json_str[self.index + idx]
+            while char.isspace():
                 idx += 1
-            try:
                 char = self.json_str[self.index + idx]
-            except IndexError:
-                return idx
+        except IndexError:
+            pass
         return idx
 
     def skip_to_character(self, character: str | list[str], idx: int = 0) -> int:

src/json_repair/parse_array.py

@@ -15,15 +15,15 @@ def parse_array(self: "JSONParser") -> list[JSONReturnType]:
     # Stop when you either find the closing parentheses or you have iterated over the entire string
     char = self.get_char_at()
     while char and char not in ["]", "}"]:
-        self.skip_whitespaces_at()
+        self.skip_whitespaces()
         value: JSONReturnType = ""
         if char in STRING_DELIMITERS:
             # Sometimes it can happen that LLMs forget to start an object and then you think it's a string in an array
             # So we are going to check if this string is followed by a : or not
             # And either parse the string or parse the object
             i = 1
             i = self.skip_to_character(char, i)
-            i = self.skip_whitespaces_at(idx=i + 1, move_main_index=False)
+            i = self.scroll_whitespaces(idx=i + 1)
             value = self.parse_object() if self.get_char_at(i) == ":" else self.parse_string()
         else:
             value = self.parse_json()

src/json_repair/parse_object.py

@@ -17,7 +17,7 @@ def parse_object(self: "JSONParser") -> JSONReturnType:
         # <member> ::= <string> ': ' <json>
 
         # Skip filler whitespaces
-        self.skip_whitespaces_at()
+        self.skip_whitespaces()
 
         # Sometimes LLMs do weird things, if we find a ":" so early, we'll change it to "," and move on
         if self.get_char_at() == ":":
@@ -53,14 +53,14 @@ def parse_object(self: "JSONParser") -> JSONReturnType:
                             prev_value.extend(
                                 new_array[0] if len(new_array) == 1 and isinstance(new_array[0], list) else new_array
                             )
-                        self.skip_whitespaces_at()
+                        self.skip_whitespaces()
                         if self.get_char_at() == ",":
                             self.index += 1
-                        self.skip_whitespaces_at()
+                        self.skip_whitespaces()
                         continue
             key = str(self.parse_string())
             if key == "":
-                self.skip_whitespaces_at()
+                self.skip_whitespaces()
             if key != "" or (key == "" and self.get_char_at() in [":", "}"]):
                 # If the string is empty but there is a object divider, we are done here
                 break
@@ -74,13 +74,13 @@ def parse_object(self: "JSONParser") -> JSONReturnType:
             break
 
         # Skip filler whitespaces
-        self.skip_whitespaces_at()
+        self.skip_whitespaces()
 
         # We reached the end here
         if (self.get_char_at() or "}") == "}":
             continue
 
-        self.skip_whitespaces_at()
+        self.skip_whitespaces()
 
         # An extreme case of missing ":" after a key
         if self.get_char_at() != ":":
@@ -92,7 +92,7 @@ def parse_object(self: "JSONParser") -> JSONReturnType:
         self.context.reset()
         self.context.set(ContextValues.OBJECT_VALUE)
         # The value can be any valid json
-        self.skip_whitespaces_at()
+        self.skip_whitespaces()
         # Corner case, a lone comma
         value: JSONReturnType = ""
         if self.get_char_at() in [",", "}"]:
@@ -110,7 +110,7 @@ def parse_object(self: "JSONParser") -> JSONReturnType:
             self.index += 1
 
         # Remove trailing spaces
-        self.skip_whitespaces_at()
+        self.skip_whitespaces()
 
     self.index += 1
 
@@ -126,11 +126,11 @@ def parse_object(self: "JSONParser") -> JSONReturnType:
     if not self.context.empty:
         return obj
 
-    self.skip_whitespaces_at()
+    self.skip_whitespaces()
     if self.get_char_at() != ",":
         return obj
     self.index += 1
-    self.skip_whitespaces_at()
+    self.skip_whitespaces()
     if self.get_char_at() not in STRING_DELIMITERS:
         return obj
     self.log(

src/json_repair/parse_string.py

@@ -93,7 +93,7 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
             self.index += 1
         else:
             # Ok this is not a doubled quote, check if this is an empty string or not
-            i = self.skip_whitespaces_at(idx=1, move_main_index=False)
+            i = self.scroll_whitespaces(idx=1)
             next_c = self.get_char_at(i)
             if next_c in STRING_DELIMITERS + ["{", "["]:
                 # something fishy is going on here
@@ -143,7 +143,7 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
         ):
             rstring_delimiter_missing = True
             # check if this is a case in which the closing comma is NOT missing instead
-            self.skip_whitespaces_at()
+            self.skip_whitespaces()
             if self.get_char_at(1) == "\\":
                 # Ok this is a quoted string, skip
                 rstring_delimiter_missing = False
@@ -153,7 +153,7 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
                 i += 1
                 # found a delimiter, now we need to check that is followed strictly by a comma or brace
                 # or the string ended
-                i = self.skip_whitespaces_at(idx=i, move_main_index=False)
+                i = self.scroll_whitespaces(idx=i)
                 next_c = self.get_char_at(i)
                 if not next_c or next_c in [",", "}"]:
                     rstring_delimiter_missing = False
@@ -168,7 +168,7 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
                     else:
                         # But again, this could just be something a bit stupid like "lorem, "ipsum" sic"
                         # Check if we find a : afterwards (skipping space)
-                        i = self.skip_whitespaces_at(idx=i + 1, move_main_index=False)
+                        i = self.scroll_whitespaces(idx=i + 1)
                         next_c = self.get_char_at(i)
                         if next_c and next_c != ":":
                             rstring_delimiter_missing = False
@@ -183,7 +183,7 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
                     break
                 else:
                     # skip any whitespace first
-                    i = self.skip_whitespaces_at(idx=1, move_main_index=False)
+                    i = self.scroll_whitespaces(idx=1)
                     # We couldn't find any rstring_delimeter before the end of the string
                     # check if this is the last string of an object and therefore we can keep going
                     # make an exception if this is the last char before the closing brace
@@ -220,7 +220,7 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
         if self.context.current == ContextValues.OBJECT_VALUE and char == "}":
             # We found the end of an object while parsing a value
             # Check if the object is really over, to avoid doubling the closing brace
-            i = self.skip_whitespaces_at(idx=1, move_main_index=False)
+            i = self.scroll_whitespaces(idx=1)
             next_c = self.get_char_at(i)
             if next_c == "`" and self.get_char_at(i + 1) == "`" and self.get_char_at(i + 2) == "`":
                 # This could be a special case in which the LLM added code fences after the object
@@ -286,7 +286,7 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
                     # found a second delimiter
                     i += 1
                     # Skip spaces
-                    i = self.skip_whitespaces_at(idx=i, move_main_index=False)
+                    i = self.scroll_whitespaces(idx=i)
                     if self.get_char_at(i) in [",", "}"]:
                         # Ok then this is a missing right quote
                         self.log(
@@ -319,7 +319,7 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
                     # We found a quote, now let's make sure there's a ":" following
                     i += 1
                     # found a delimiter, now we need to check that is followed strictly by a comma or brace
-                    i = self.skip_whitespaces_at(idx=i, move_main_index=False)
+                    i = self.scroll_whitespaces(idx=i)
                     if self.get_char_at(i) == ":":
                         # Reset the cursor
                         self.index -= 1
@@ -365,7 +365,7 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
                     next_c = self.get_char_at(i)
                     # Ok now I found a delimiter, let's skip whitespaces and see if next we find a } or a ,
                     i += 1
-                    i = self.skip_whitespaces_at(idx=i, move_main_index=False)
+                    i = self.scroll_whitespaces(idx=i)
                     next_c = self.get_char_at(i)
                     if next_c in ["}", ","]:
                         self.log(
@@ -378,15 +378,15 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
                     if all(str(self.get_char_at(j)).isspace() for j in range(1, i) if self.get_char_at(j)):
                         break
                     if self.context.current == ContextValues.OBJECT_VALUE:
-                        i = self.skip_whitespaces_at(idx=i + 1, move_main_index=False)
+                        i = self.scroll_whitespaces(idx=i + 1)
                         if self.get_char_at(i) == ",":
                             # So we found a comma, this could be a case of a single quote like "va"lue",
                             # Search if it's followed by another key, starting with the first delimeter
                             i = self.skip_to_character(character=lstring_delimiter, idx=i + 1)
                             i += 1
                             i = self.skip_to_character(character=rstring_delimiter, idx=i + 1)
                             i += 1
-                            i = self.skip_whitespaces_at(idx=i, move_main_index=False)
+                            i = self.scroll_whitespaces(idx=i)
                             next_c = self.get_char_at(i)
                             if next_c == ":":
                                 self.log(
@@ -449,7 +449,7 @@ def _append_literal_char(acc: str, current_char: str | None) -> tuple[str, str |
         self.log(
             "While parsing a string, handling an extreme corner case in which the LLM added a comment instead of valid string, invalidate the string and return an empty value",
         )
-        self.skip_whitespaces_at()
+        self.skip_whitespaces()
         if self.get_char_at() not in [":", ","]:
             return ""