[Fixes #13800] Refact metadata i18n mechanism

Author: etj

geonode/base/apps.py

@@ -42,3 +42,11 @@ class BaseAppConfig(NotificationsAppConfigBase, AppConfig):
             _("Resource is created"),
         ),
     )
+
+    def ready(self):
+        """Finalize setup"""
+        from geonode.base.signals import connect_signals
+
+        connect_signals()
+
+        super(BaseAppConfig, self).ready()

geonode/metadata/i18n.py geonode/base/i18n.pygeonode/metadata/i18n.py renamed to geonode/base/i18n.py

@@ -1,12 +1,10 @@
 import logging
-from datetime import datetime
 
-from cachetools import FIFOCache
 from django.db import connection
+from django.utils.translation import get_language, gettext as _
 
 from geonode.base.models import ThesaurusKeywordLabel, Thesaurus
 
-
 logger = logging.getLogger(__name__)
 
 I18N_THESAURUS_IDENTIFIER = "labels-i18n"
@@ -47,6 +45,7 @@ def get_localized_tkeywords(lang, thesaurus_identifier: str):
     return sorted(ret.values(), key=lambda i: i["about"].lower())
 
 
+# TODO: deprecate and use LabelResolver.gettext()
 def get_localized_label(lang, about):
     # look for override
     ovr_qs = ThesaurusKeywordLabel.objects.filter(
@@ -65,32 +64,37 @@ def get_localized_label(lang, about):
     )
 
 
-class I18nCache:
+class I18nCacheEntry:
+    def __init__(self):
+        # the date field of the thesaurus when it was last loaded, it's used for the expiration check
+        self.date: str | None = None
+        self.caches: dict = {}  # the caches for this language
+
 
-    DATA_KEY_SCHEMA = "schema"
-    DATA_KEY_LABELS = "labels"
+class I18nCache:
+    """
+    Caches language related data.
+    Synch is performed via date field in the "labels-i18n" thesaurus.
+    """
 
     def __init__(self):
-        # the cache has the lang as key, and various info in the dict value:
-        # - date: the date field of the thesaurus when it was last loaded, it's used for the expiration check
-        # - labels: the keyword labels from the i18n thesaurus
-        # - schema: the localized json schema
-        # FIFO bc we want to renew the data once in a while
-        self.cache = FIFOCache(16)
+        # the cache has the lang as key, and I18nCacheEntry as a value:
+        self.lang_cache = {}
 
     def get_entry(self, lang, data_key):
         """
         returns date:str, data
         date is needed for checking the entry freshness when setting info
         data may be None if not cached or expired
         """
-        cached_entry = self.cache.get(lang, None)
+        cached_entry = self.lang_cache.get(lang, None)
 
+        # TODO: thesaurus date check should be done only after a given time interval from last check
         thesaurus_date = (  # may be none if thesaurus does not exist
             Thesaurus.objects.filter(identifier=I18N_THESAURUS_IDENTIFIER).values_list("date", flat=True).first()
         )
         if cached_entry:
-            if thesaurus_date == cached_entry["date"]:
+            if thesaurus_date == cached_entry.date:
                 # only return cached data if thesaurus has not been modified
                 return thesaurus_date, cached_entry.get(data_key, None)
             else:
@@ -99,7 +103,7 @@ def get_entry(self, lang, data_key):
         return thesaurus_date, None
 
     def set(self, lang: str, data_key: str, data: dict, request_date: str):
-        cached_entry: dict = self.cache.setdefault(lang, {})
+        cached_entry: I18nCacheEntry = self.lang_cache.setdefault(lang, I18nCacheEntry())
 
         latest_date = (
             Thesaurus.objects.filter(identifier=I18N_THESAURUS_IDENTIFIER).values_list("date", flat=True).first()
@@ -108,60 +112,43 @@ def set(self, lang: str, data_key: str, data: dict, request_date: str):
         if request_date == latest_date:
             # no changes after processing, set the info right away
             logger.debug(f"Caching lang:{lang} key:{data_key} date:{request_date}")
-            cached_entry.update({"date": latest_date, data_key: data})
+            cached_entry.date = latest_date
+            cached_entry.caches[data_key] = data
         else:
             logger.warning(
                 f"Cache will not be updated for lang:{lang} key:{data_key} reqdate:{request_date} latest:{latest_date}"
             )
 
-    def get_labels(self, lang):
-        date, labels = self.get_entry(lang, self.DATA_KEY_LABELS)
-        if labels is None:
-            labels = {}
-            for i in get_localized_tkeywords(lang, I18N_THESAURUS_IDENTIFIER):
-                about = i["about"]
-                if about.endswith(OVR_SUFFIX) and not i["label"]:
-                    # we don't want default values for override entries
-                    continue
-                labels[about] = i["label"] or i["default"]
-            self.set(lang, self.DATA_KEY_LABELS, labels, date)
-        return labels
-
     def clear_schema_cache(self):
         logger.info("Clearing schema cache")
-        while True:
-            try:
-                self.cache.popitem()
-            except KeyError:
-                return
-
+        self.lang_cache.clear()
 
-def thesaurus_changed(sender, instance, **kwargs):
-    if instance.identifier == I18N_THESAURUS_IDENTIFIER:
-        if hasattr(instance, "_signal_handled"):  # avoid signal recursion
-            return
-        logger.debug(f"Thesaurus changed: {instance.identifier}")
-        _update_thesaurus_date()
 
+class LabelResolver:
+    CACHE_KEY_LABELS = "labels"
 
-def thesaurusk_changed(sender, instance, **kwargs):
-    if instance.thesaurus.identifier == I18N_THESAURUS_IDENTIFIER:
-        logger.debug(f"ThesaurusKeyword changed: {instance.about} ALT:{instance.alt_label}")
-        _update_thesaurus_date()
+    def gettext(self, key, lang=None, fallback=True):
+        lang = lang or get_language()
+        return self.get_labels(lang).get(key, None) or (_(key) if fallback else None)
 
+    def get_labels(self, lang):
+        date, labels = i18nCache.get_entry(lang, self.CACHE_KEY_LABELS)
+        if labels is None:
+            labels = self._create_labels_cache(lang)
+            i18nCache.set(lang, self.CACHE_KEY_LABELS, labels, date)
+        return labels
 
-def thesauruskl_changed(sender, instance, **kwargs):
-    if instance.keyword.thesaurus.identifier == I18N_THESAURUS_IDENTIFIER:
-        logger.debug(
-            f"ThesaurusKeywordLabel changed: {instance.keyword.about} ALT:{instance.keyword.alt_label} L:{instance.lang}"
-        )
-        _update_thesaurus_date()
+    def _create_labels_cache(self, lang):
+        labels = {}
+        for i in get_localized_tkeywords(lang, I18N_THESAURUS_IDENTIFIER):
+            about = i["about"]
+            if about.endswith(OVR_SUFFIX) and not i["label"]:
+                # we don't want default values for override entries
+                continue
+            labels[about] = i["label"] or i["default"]
+        return labels
 
 
-def _update_thesaurus_date():
-    logger.debug("Updating label thesaurus date")
-    # update timestamp to invalidate other processes also
-    i18n_thesaurus = Thesaurus.objects.get(identifier=I18N_THESAURUS_IDENTIFIER)
-    i18n_thesaurus.date = datetime.now().replace(microsecond=0).isoformat()
-    i18n_thesaurus._signal_handled = True
-    i18n_thesaurus.save()
+i18nCache = I18nCache()
+labelResolver = LabelResolver()
+gettext = labelResolver.gettext

geonode/base/signals.py

@@ -0,0 +1,48 @@
+import logging
+from datetime import datetime
+
+from django.db.models.signals import post_save
+
+from geonode.base.i18n import I18N_THESAURUS_IDENTIFIER
+from geonode.base.models import Thesaurus, ThesaurusKeyword, ThesaurusKeywordLabel
+
+logger = logging.getLogger(__name__)
+
+
+def connect_signals():
+    logger.debug("Setting up signal connections...")
+    post_save.connect(thesaurus_changed, sender=Thesaurus, weak=False, dispatch_uid="metadata_reset_t")
+    post_save.connect(thesaurusk_changed, sender=ThesaurusKeyword, weak=False, dispatch_uid="metadata_reset_tk")
+    post_save.connect(thesauruskl_changed, sender=ThesaurusKeywordLabel, weak=False, dispatch_uid="metadata_reset_tkl")
+    logger.debug("Signal connections set")
+
+
+def thesaurus_changed(sender, instance, **kwargs):
+    if instance.identifier == I18N_THESAURUS_IDENTIFIER:
+        if hasattr(instance, "_signal_handled"):  # avoid signal recursion
+            return
+        logger.debug(f"Thesaurus changed: {instance.identifier}")
+        _update_thesaurus_date()
+
+
+def thesaurusk_changed(sender, instance, **kwargs):
+    if instance.thesaurus.identifier == I18N_THESAURUS_IDENTIFIER:
+        logger.debug(f"ThesaurusKeyword changed: {instance.about} ALT:{instance.alt_label}")
+        _update_thesaurus_date()
+
+
+def thesauruskl_changed(sender, instance, **kwargs):
+    if instance.keyword.thesaurus.identifier == I18N_THESAURUS_IDENTIFIER:
+        logger.debug(
+            f"ThesaurusKeywordLabel changed: {instance.keyword.about} ALT:{instance.keyword.alt_label} L:{instance.lang}"
+        )
+        _update_thesaurus_date()
+
+
+def _update_thesaurus_date():
+    logger.debug("Updating label thesaurus date")
+    # update timestamp to invalidate other processes also
+    i18n_thesaurus = Thesaurus.objects.get(identifier=I18N_THESAURUS_IDENTIFIER)
+    i18n_thesaurus.date = datetime.now().replace(microsecond=0).isoformat()
+    i18n_thesaurus._signal_handled = True
+    i18n_thesaurus.save()

geonode/metadata/api/views.py

@@ -37,7 +37,7 @@
 from geonode.base.views import LinkedResourcesAutocomplete, RegionAutocomplete, HierarchicalKeywordAutocomplete
 from geonode.groups.models import GroupProfile
 from geonode.metadata.handlers.abstract import MetadataHandler
-from geonode.metadata.i18n import get_localized_label
+from geonode.base.i18n import get_localized_label
 from geonode.metadata.manager import metadata_manager
 from geonode.metadata.multilang import utils as multi
 from geonode.people.utils import get_available_users

geonode/metadata/apps.py

@@ -19,10 +19,6 @@ def ready(self):
 def run_setup_hooks(*args, **kwargs):
     setup_metadata_handlers()
 
-    from geonode.metadata.signals import connect_signals
-
-    connect_signals()
-
 
 def setup_metadata_handlers():
     from geonode.metadata.manager import metadata_manager

geonode/metadata/handlers/abstract.py

@@ -26,7 +26,8 @@
 from django.utils.translation import gettext as _
 
 from geonode.base.models import ResourceBase
-from geonode.metadata.i18n import OVR_SUFFIX
+from geonode.base.i18n import OVR_SUFFIX
+from geonode.metadata.manager import CONTEXT_KEY_LABELS
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +52,7 @@ def update_schema(self, jsonschema: dict, context: dict, lang=None):
         It adds the subschema handled by the handler, and returns the
         augmented instance of the JSON Schema.
         Context is populated by the manager with some common info:
-         - key "labels": contains the localized label loaded from the db as a dict, where key is the ThesaurusKeyword about
+         - key CONTEXT_KEY_LABELS: contains the localized label loaded from the db as a dict, where key is the ThesaurusKeyword about
            and value is the localized ThesaurusKeywordLabel, or the AltLabel if the localized label does not exist.
         """
         pass
@@ -72,7 +73,7 @@ def update_resource(
     ):
         """
         Called when persisting data, updates the field `field_name` of the resource
-        with the content content, where json_instance is  the full JSON Schema instance,
+        with the content, where json_instance is the full JSON Schema instance,
         in case the handler needs some cross related data contained in the resource.
         """
         pass
@@ -172,12 +173,14 @@ def localize_message(context: dict, msg_code: str, msg_info: dict):
 
     @staticmethod
     def _localize_label(context, lang: str, text: str):
+        # TODO: deprecate and use LabelResolver.gettext(...fallback=true)
         label = MetadataHandler._get_tkl_labels(context, lang, text)
         return label or _(text)
 
     @staticmethod
     def _get_tkl_labels(context, lang: str | None, text: str):
-        return context.get("labels", {}).get(text, None)
+        # TODO: deprecate and use LabelResolver.gettext(...fallback=false)
+        return context.get(CONTEXT_KEY_LABELS, {}).get(text, None)
 
     @staticmethod
     def _localize_subschema_labels(context, subschema: dict, lang: str, property_name: str = None):

geonode/metadata/handlers/sparse.py

@@ -24,11 +24,11 @@
 
 from geonode.metadata.handlers.abstract import MetadataHandler
 from geonode.metadata.exceptions import UnsetFieldException
-from geonode.metadata.i18n import get_localized_tkeywords
 from geonode.metadata.models import SparseField
 
-logger = logging.getLogger(__name__)
+from geonode.base.i18n import get_localized_tkeywords
 
+logger = logging.getLogger(__name__)
 
 CONTEXT_ID = "sparse"
 

geonode/metadata/manager.py

@@ -27,16 +27,19 @@
 from geonode.indexing.manager import index_manager
 from geonode.metadata.handlers.abstract import MetadataHandler
 from geonode.metadata.exceptions import UnsetFieldException
-from geonode.metadata.i18n import I18nCache
+from geonode.base.i18n import I18nCache
 from geonode.metadata.settings import MODEL_SCHEMA
 
 logger = logging.getLogger(__name__)
 
+CACHE_KEY_SCHEMA = "schema"
+CONTEXT_KEY_LABELS = "labels"
+
 
 class MetadataManager:
     """
     The metadata manager is the bridge between the API and the geonode model.
-    The metadata manager will loop over all of the registered metadata handlers,
+    The metadata manager will loop over all the registered metadata handlers,
     calling their update_schema(jsonschema) which will add the subschemas of the
     fields handled by each handler. At the end of the loop, the schema will be ready
     to be delivered to the caller.
@@ -58,7 +61,7 @@ def post_init(self):
             handler.post_init()
 
     def _init_schema_context(self, lang):
-        return {"labels": self._i18n_cache.get_labels(lang)}
+        return {CONTEXT_KEY_LABELS: self._i18n_cache.get_labels(lang)}
 
     def build_schema(self, lang=None):
         logger.debug(f"build_schema {lang}")
@@ -83,12 +86,12 @@ def build_schema(self, lang=None):
 
     def get_schema(self, lang=None):
         lang = str(lang)
-        thesaurus_date, schema = self._i18n_cache.get_entry(lang, I18nCache.DATA_KEY_SCHEMA)
+        thesaurus_date, schema = self._i18n_cache.get_entry(lang, CACHE_KEY_SCHEMA)
         if schema is None:
             logger.info(f"Building schema for {lang}")
             schema = self.build_schema(lang)
             logger.debug("Schema built")
-            self._i18n_cache.set(lang, I18nCache.DATA_KEY_SCHEMA, schema, thesaurus_date)
+            self._i18n_cache.set(lang, CACHE_KEY_SCHEMA, schema, thesaurus_date)
         return schema
 
     def build_schema_instance(self, resource, lang=None):