Add mypy (#128)

Check the code using mypy --strict
Fix mypy warnings
Fix some of the actual bugs 
Test on Python 3.13

Author: insolor

.github/workflows/test.yml

@@ -16,6 +16,7 @@ jobs:
           - "3.10"
           - "3.11"
           - "3.12"
+          - "3.13"
 
     steps:
     - uses: actions/checkout@v4
@@ -36,9 +37,15 @@ jobs:
     - name: Lint with ruff
       run: |
         poetry run ruff check . --statistics
+
+    - name: Check with mypy
+      run: |
+        poetry run mypy . --strict
+
     - name: Test with pytest
       run: |
         poetry run pytest --cov=./
+
     - name: Upload coverage data to coveralls.io
       run: |
         pip install coveralls

df_translation_toolkit/convert/objects_po_to_csv.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
 from collections.abc import Iterable, Iterator
 from pathlib import Path
-from typing import TextIO
+from typing import IO, TextIO
 
 import typer
 from loguru import logger
@@ -21,10 +21,9 @@ def get_translations_from_tag_parts(
 ) -> Iterator[tuple[str, str]]:
     tag_translations = defaultdict(list)
 
-    prev_original = None
-    prev_translation = None
+    prev_original = ""
+    prev_translation = ""
     for original, translation in zip(original_parts, translation_parts, strict=False):
-        original: str
         if all_caps(original) or original.isdecimal():
             if original == "STP" and translation != original and not all_caps(translation):
                 tag_translations[prev_original + "s"].append(translation)
@@ -53,7 +52,7 @@ def get_translations_from_tag(original_tag: str, translation_tag: str) -> Iterat
         raise ValidationException(validation_problems)  # pass warnings
 
 
-def prepare_dictionary(dictionary: Iterable[tuple[str, str]], errors_file: TextIO) -> Iterable[tuple[str, str]]:
+def prepare_dictionary(dictionary: Iterable[tuple[str, str]], errors_file: TextIO | None) -> Iterable[tuple[str, str]]:
     for original_string_tag, translation_tag in dictionary:
         if original_string_tag and translation_tag and translation_tag != original_string_tag:
             try:
@@ -66,7 +65,7 @@ def prepare_dictionary(dictionary: Iterable[tuple[str, str]], errors_file: TextI
                     print(error_text, end="\n\n", file=errors_file)
 
 
-def convert(po_file: TextIO, csv_file: TextIO, error_file: TextIO | None = None) -> None:
+def convert(po_file: TextIO, csv_file: IO[str], error_file: TextIO | None = None) -> None:
     dictionary = simple_read_po(po_file)
     csv_writer = csv_utils.writer(csv_file)
 

df_translation_toolkit/create_mod/from_template.py

@@ -97,22 +97,22 @@ def create_info(info_file: Path, destination_encoding: str, language: str) -> No
 
 def get_dictionaries(translation_path: Path, language: str) -> Dictionaries:
     po_files: dict[str, Path] = {}
-    for po_file in ["objects", "text_set"]:
+    for po_file_type in ["objects", "text_set"]:
         mtime = 0
-        for file in translation_path.glob(f"*{po_file}*{language}.po"):
+        for file in translation_path.glob(f"*{po_file_type}*{language}.po"):
             if file.is_file() and file.stat().st_mtime > mtime:
-                po_files[po_file] = file
+                po_files[po_file_type] = file
 
-        if po_file not in po_files:
-            msg = f"Unable to find {po_file} po file for language {language}"
+        if po_file_type not in po_files:
+            msg = f"Unable to find {po_file_type} po file for language {language}"
             raise ValueError(msg)
 
-    with open(po_files["objects"], encoding="utf-8") as pofile:
+    with open(po_files["objects"], encoding="utf-8") as po_file:
         dictionary_object: Mapping[tuple[str, str | None], str] = {
-            (item.id, item.context): item.string for item in read_po(pofile)
+            (str(item.id), str(item.context) if item.context else None): str(item.string) for item in read_po(po_file)
         }
     with open(po_files["text_set"], encoding="utf-8") as po_file:
-        dictionary_textset: Mapping[str, str] = {item.id: item.string for item in read_po(po_file) if item.id}
+        dictionary_textset: Mapping[str, str] = {str(item.id): str(item.string) for item in read_po(po_file) if item.id}
     return Dictionaries(language.lower(), dictionary_object, dictionary_textset)
 
 

df_translation_toolkit/create_pot/from_steam_text.py

@@ -1,14 +1,15 @@
 from collections import defaultdict
 from collections.abc import Iterable
 from pathlib import Path
+from typing import Any
 
 import typer
 from loguru import logger
 
 from df_translation_toolkit.create_pot.from_raw_objects import extract_from_raw_file
 from df_translation_toolkit.parse.parse_raws import split_tag, tokenize_raw_file
 from df_translation_toolkit.parse.parse_text_set import extract_from_vanilla_text
-from df_translation_toolkit.utils.po_utils import save_pot
+from df_translation_toolkit.utils.po_utils import TranslationItem, save_pot
 
 
 def traverse_vanilla_directories(vanilla_path: Path) -> Iterable[Path]:
@@ -19,7 +20,7 @@ def traverse_vanilla_directories(vanilla_path: Path) -> Iterable[Path]:
                 yield objects
 
 
-def get_raw_object_type(file_name: Path, source_encoding: str) -> str:
+def get_raw_object_type(file_name: Path, source_encoding: str) -> str | None:
     with file_name.open(encoding=source_encoding) as file:
         for item in tokenize_raw_file(file):
             if item.is_tag:
@@ -33,7 +34,7 @@ def get_raw_object_type(file_name: Path, source_encoding: str) -> str:
         return None
 
 
-def get_translatable_strings(file_path: Path, source_encoding: str) -> tuple[str, Iterable[str]] | None:
+def get_translatable_strings(file_path: Path, source_encoding: str) -> tuple[str, Iterable[TranslationItem]] | None:
     object_type = get_raw_object_type(file_path, source_encoding)
     if object_type in dont_translate:
         return None
@@ -48,7 +49,7 @@ def get_translatable_strings(file_path: Path, source_encoding: str) -> tuple[str
     return key, data
 
 
-def iterable_is_empty(iterable: Iterable) -> bool:
+def iterable_is_empty(iterable: Iterable[Any]) -> bool:
     iterator = iter(iterable)
     try:
         next(iterator)
@@ -75,7 +76,7 @@ def main(vanilla_path: Path, destination_path: Path, source_encoding: str = "cp4
         msg = "Destination path doesn't exist"
         raise ValueError(msg)
 
-    results = defaultdict(list)
+    results: dict[str, list[TranslationItem]] = defaultdict(list)
 
     for directory in traverse_vanilla_directories(vanilla_path):
         logger.info(directory.relative_to(vanilla_path))
@@ -91,8 +92,8 @@ def main(vanilla_path: Path, destination_path: Path, source_encoding: str = "cp4
         for group, data in results.items():
             if data:
                 pot_path = destination_path / (group.lower() + ".pot")
-                with pot_path.open("wb") as file_path:
-                    save_pot(file_path, data)
+                with pot_path.open("wb") as pot_file:
+                    save_pot(pot_file, data)
 
 
 if __name__ == "__main__":

df_translation_toolkit/parse/parse_raws.py

@@ -64,9 +64,9 @@ class FilePartInfo(NamedTuple):
     line_number: int
     translatable: bool
     context: str | None
-    text: str | None = None
-    tag: str | None = None
-    tag_parts: Sequence[str] | None = None
+    text: str = ""
+    tag: str = ""
+    tag_parts: Sequence[str] = ()
 
 
 def parse_raw_file(file: Iterable[str]) -> Iterator[FilePartInfo]:
@@ -75,47 +75,50 @@ def parse_raw_file(file: Iterable[str]) -> Iterator[FilePartInfo]:
     for token in tokenize_raw_file(file):
         if not token.is_tag:
             yield FilePartInfo(line_number=token.line_number, translatable=False, context=context, text=token.text)
+            continue
+
+        tag = token.text
+        tag_parts = split_tag(tag)
+
+        if tag_parts[0] == "OBJECT":
+            object_name = tag_parts[1]
+            yield FilePartInfo(line_number=token.line_number, translatable=False, context=context, tag=tag)
+        elif object_name and (
+            tag_parts[0] == object_name
+            or (object_name in {"ITEM", "BUILDING"} and tag_parts[0].startswith(object_name))
+            or object_name.endswith("_" + tag_parts[0])
+        ):
+            context = ":".join(tag_parts)
+            yield FilePartInfo(token.line_number, translatable=False, context=context, tag=tag)
         else:
-            tag = token.text
-            tag_parts = split_tag(tag)
-
-            if tag_parts[0] == "OBJECT":
-                object_name = tag_parts[1]
-                yield FilePartInfo(line_number=token.line_number, translatable=False, context=context, tag=tag)
-            elif object_name and (
-                tag_parts[0] == object_name
-                or (object_name in {"ITEM", "BUILDING"} and tag_parts[0].startswith(object_name))
-                or object_name.endswith("_" + tag_parts[0])
-            ):
-                context = ":".join(tag_parts)
-                yield FilePartInfo(token.line_number, translatable=False, context=context, tag=tag)
-            else:
-                yield FilePartInfo(token.line_number, translatable=True, context=context, tag=tag, tag_parts=tag_parts)
+            yield FilePartInfo(token.line_number, translatable=True, context=context, tag=tag, tag_parts=tag_parts)
 
 
 def extract_translatables_from_raws(file: Iterable[str]) -> Iterator[TranslationItem]:
-    translation_keys: set[tuple[str, tuple[str, ...]]] = set()
+    translation_keys: set[tuple[str | None, tuple[str, ...]]] = set()
 
     for item in parse_raw_file(file):
-        if item.translatable:
-            tag_parts = item.tag_parts
-            if (
-                "TILE" not in tag_parts[0]
-                and any(is_translatable(s) for s in tag_parts[1:])
-                and (item.context, tuple(tag_parts)) not in translation_keys  # Don't add duplicate items to translate
-            ):
-                if not is_translatable(tag_parts[-1]):
-                    last = last_suitable(tag_parts, is_translatable)
-                    tag_parts = tag_parts[:last]
-                    tag_parts.append("")  # Add an empty element to the tag to mark the tag as not completed
-                translation_keys.add((item.context, tuple(tag_parts)))
-                yield TranslationItem(context=item.context, text=join_tag(tag_parts), line_number=item.line_number)
+        if not item.translatable:
+            continue
+
+        tag_parts = item.tag_parts
+        if (
+            "TILE" not in tag_parts[0]
+            and any(is_translatable(s) for s in tag_parts[1:])
+            and (item.context, tuple(tag_parts)) not in translation_keys  # Don't add duplicate items to translate
+        ):
+            if not is_translatable(tag_parts[-1]):
+                last = last_suitable(tag_parts, is_translatable)
+                tag_parts = list(tag_parts[:last])
+                tag_parts.append("")  # Add an empty element to the tag to mark the tag as not completed
+            translation_keys.add((item.context, tuple(tag_parts)))
+            yield TranslationItem(context=item.context, text=join_tag(tag_parts), line_number=item.line_number)
 
 
 def get_from_dict_with_context(
     dictionary: Mapping[tuple[str, str | None], str],
     key: str,
-    context: str,
+    context: str | None,
 ) -> str | None:
     if (key, context) in dictionary:
         return dictionary[(key, context)]
@@ -128,7 +131,7 @@ def get_from_dict_with_context(
 
 def get_tag_translation(dictionary: Mapping[tuple[str, str | None], str], item: FilePartInfo) -> str:
     tag = item.tag
-    tag_parts = item.tag_parts
+    tag_parts = list(item.tag_parts)
     context = item.context
     new_tag = get_from_dict_with_context(dictionary, tag, context)
 
@@ -160,13 +163,13 @@ def translate_raw_file(file: Iterable[str], dictionary: Mapping[tuple[str, str |
             modified_line_parts = []
             prev_line_number = item.line_number
 
-        if item.text is not None:
+        if item.text:
             modified_line_parts.append(item.text)
-        elif not item.translatable or not any(is_translatable(s) for s in item.tag_parts[1:]):
-            modified_line_parts.append(item.tag)
-        else:
+        elif item.translatable and any(is_translatable(s) for s in item.tag_parts[1:]):
             translation = get_tag_translation(dictionary, item)
             modified_line_parts.append(translation)
+        else:
+            modified_line_parts.append(item.tag)
 
     if modified_line_parts:
         yield "".join(modified_line_parts)

df_translation_toolkit/translate/translate_plain_text.py

@@ -19,7 +19,7 @@ def translate_plain_text_file(
 ) -> Iterator[str]:
     with source_file_path.open() as source_file, destination_file_path.open("w", encoding=encoding) as destination_file:
         yield destination_file_path.name
-        for text_block, _, _ in parse_plain_text_file(source_file, join_paragraphs):
+        for text_block, _, _ in parse_plain_text_file(source_file, join_paragraphs=join_paragraphs):
             clean_text_block = text_block.rstrip("\n")
             if clean_text_block in dictionary:
                 translation = dictionary[clean_text_block]
@@ -38,12 +38,18 @@ def translate_plain_text(
     join_paragraphs: bool = True,
 ) -> Iterator[str]:
     with po_filename.open("r", encoding="utf-8") as po_file:
-        dictionary = {item.id: item.string for item in read_po(po_file) if item.id}
+        dictionary = {str(item.id): str(item.string) for item in read_po(po_file) if item.id}
 
     for item in Path(path).rglob("*.txt"):
         if item.is_file():
             with backup(item) as backup_file:
-                yield from translate_plain_text_file(backup_file, item, dictionary, encoding, join_paragraphs)
+                yield from translate_plain_text_file(
+                    backup_file,
+                    item,
+                    dictionary,
+                    encoding,
+                    join_paragraphs=join_paragraphs,
+                )
 
 
 def main(
@@ -53,7 +59,7 @@ def main(
     split: bool = False,  # noqa: FBT001, FBT002
 ) -> None:
     join_paragraphs = not split
-    for filename in translate_plain_text(po_filename, path, encoding, join_paragraphs):
+    for filename in translate_plain_text(po_filename, path, encoding, join_paragraphs=join_paragraphs):
         print(filename, file=sys.stderr)
 
 

df_translation_toolkit/translate/translate_raws.py

@@ -35,15 +35,17 @@ def translate_single_raw_file(
 
 def translate_raws(po_filename: Path, path: Path, encoding: str) -> Iterator[str]:
     with po_filename.open("r", encoding="utf-8") as pofile:
-        dictionary = {(item.id, item.context): item.string for item in read_po(pofile)}
+        dictionary = {
+            (str(item.id), str(item.context) if item.context else None): str(item.string) for item in read_po(pofile)
+        }
 
     for file_path in path.glob("*.txt"):
         if file_path.is_file() and not file_path.name.startswith("language_"):
             with backup(file_path) as bak_name:
                 yield from translate_single_raw_file(bak_name, file_path, dictionary, encoding)
 
 
-def main(po_filename: Path, path: Path, encoding: str) -> str:
+def main(po_filename: Path, path: Path, encoding: str) -> None:
     for filename in translate_raws(po_filename, path, encoding):
         print(filename, file=sys.stderr)
 

df_translation_toolkit/utils/csv_utils.py

@@ -1,14 +1,23 @@
 import csv
-from collections.abc import Iterator
+from collections.abc import Iterable, Iterator
 from pathlib import Path
-from typing import TextIO
+from typing import IO, Protocol, TextIO
 
 
-def writer(file: TextIO, **kwargs):  # noqa: ANN201
+class CSVWriter(Protocol):
+    def writerow(self, row: list[str]) -> None: ...
+    def writerows(self, rows: Iterable[list[str]]) -> None: ...
+
+
+class CSVReader(Protocol):
+    def __iter__(self) -> Iterator[list[str]]: ...
+
+
+def writer(file: IO[str], **kwargs) -> CSVWriter:  # type: ignore[no-untyped-def]
     return csv.writer(file, dialect="unix", lineterminator="\r\n", **kwargs)
 
 
-def reader(file: TextIO, **kwargs):  # noqa: ANN201
+def reader(file: TextIO, **kwargs) -> CSVReader:  # type: ignore[no-untyped-def]
     return csv.reader(file, dialect="unix", lineterminator="\r\n", **kwargs)
 
 

df_translation_toolkit/utils/df_ignore_string_rules.py

@@ -10,7 +10,7 @@
 class IgnoringRuleRegistry:
     all_rules: dict[str, Callable[[str], bool]]
 
-    def __init__(self) -> str:
+    def __init__(self) -> None:
         self.all_rules = {}
 
     def register(self, function: Callable[[str], bool]) -> Callable[[str], bool]:
@@ -79,7 +79,11 @@ def ignore_paths(string: str) -> bool:
 
 @rules.register
 def ignore_tags(string: str) -> bool:
-    return len(string) >= MINMAL_TAG_LENGTH and string not in ignore_tags_exceptions and re.fullmatch("[A-Z_]+", string)
+    return (
+        len(string) >= MINMAL_TAG_LENGTH
+        and string not in ignore_tags_exceptions
+        and bool(re.fullmatch("[A-Z_]+", string))
+    )
 
 
 @rules.register
@@ -458,7 +462,7 @@ def ignore_by_blacklisted_substrings(string: str) -> bool:
     return any(substring in string for substring in blacklisted_substrings)
 
 
-def all_ignore_rules(string: str) -> str:
+def all_ignore_rules(string: str) -> str | None:
     return rules.check_ignore(string)