feat: Case-insensitive search for non-ASCII chat and contact names (#7477)

This makes `Contact::get_all()` and `Chatlist::try_load()` case-insensitive for non-ASCII chat and
contact names as well. The same approach as in f6f4ccc "feat:
Case-insensitive search for non-ASCII messages (#5052)" is used: `chats.name_normalized` and
`contacts.name_normalized` colums are added which store lowercased/normalized names (for a contact,
if the name is unset, it's a normalized authname). If a normalized name is the same as the
chat/contact name, it's not stored to reduce the db size. A db migration is added for 10000 random
chats and the same number of the most recently seen contacts, for users it will probably migrate all
chats/contacts and for bots which may have more data it's not important.

Author: iequidoo

src/calls.rs

@@ -10,11 +10,11 @@ use crate::context::{Context, WeakContext};
 use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::log::warn;
-use crate::message::{self, Message, MsgId, Viewtype};
+use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::net::dns::lookup_host_with_cache;
 use crate::param::Param;
-use crate::tools::time;
+use crate::tools::{normalize_text, time};
 use anyhow::{Context as _, Result, ensure};
 use sdp::SessionDescription;
 use serde::Serialize;
@@ -86,7 +86,7 @@ impl CallInfo {
             .sql
             .execute(
                 "UPDATE msgs SET txt=?, txt_normalized=? WHERE id=?",
-                (text, message::normalize_text(text), self.msg.id),
+                (text, normalize_text(text), self.msg.id),
             )
             .await?;
         Ok(())

src/chat.rs

@@ -45,7 +45,7 @@ use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
     IsNoneOrEmpty, SystemTime, buf_compress, create_broadcast_secret, create_id,
     create_outgoing_rfc724_mid, create_smeared_timestamp, create_smeared_timestamps, get_abs_path,
-    gm2local_offset, smeared_time, time, truncate_msg_text,
+    gm2local_offset, normalize_text, smeared_time, time, truncate_msg_text,
 };
 use crate::webxdc::StatusUpdateSerial;
 use crate::{chatlist_events, imap};
@@ -286,10 +286,11 @@ impl ChatId {
         let timestamp = cmp::min(timestamp, smeared_time(context));
         let row_id =
             context.sql.insert(
-                "INSERT INTO chats (type, name, grpid, blocked, created_timestamp, protected, param) VALUES(?, ?, ?, ?, ?, 0, ?);",
+                "INSERT INTO chats (type, name, name_normalized, grpid, blocked, created_timestamp, protected, param) VALUES(?, ?, ?, ?, ?, ?, 0, ?)",
                 (
                     chattype,
                     &grpname,
+                    normalize_text(&grpname),
                     grpid,
                     create_blocked,
                     timestamp,
@@ -782,7 +783,7 @@ impl ChatId {
                                     time(),
                                     msg.viewtype,
                                     &msg.text,
-                                    message::normalize_text(&msg.text),
+                                    normalize_text(&msg.text),
                                     msg.param.to_string(),
                                     msg.in_reply_to.as_deref().unwrap_or_default(),
                                     msg.id,
@@ -823,7 +824,7 @@ impl ChatId {
                         msg.viewtype,
                         MessageState::OutDraft,
                         &msg.text,
-                        message::normalize_text(&msg.text),
+                        normalize_text(&msg.text),
                         msg.param.to_string(),
                         1,
                         msg.in_reply_to.as_deref().unwrap_or_default(),
@@ -1919,7 +1920,7 @@ impl Chat {
                         msg.viewtype,
                         msg.state,
                         msg_text,
-                        message::normalize_text(&msg_text),
+                        normalize_text(&msg_text),
                         &msg.subject,
                         msg.param.to_string(),
                         msg.hidden,
@@ -1970,7 +1971,7 @@ impl Chat {
                         msg.viewtype,
                         msg.state,
                         msg_text,
-                        message::normalize_text(&msg_text),
+                        normalize_text(&msg_text),
                         &msg.subject,
                         msg.param.to_string(),
                         msg.hidden,
@@ -2274,8 +2275,8 @@ async fn update_special_chat_name(
         context
             .sql
             .execute(
-                "UPDATE chats SET name=? WHERE id=? AND name!=?",
-                (&name, chat_id, &name),
+                "UPDATE chats SET name=?, name_normalized=? WHERE id=? AND name!=?",
+                (&name, normalize_text(&name), chat_id, &name),
             )
             .await?;
     }
@@ -2388,11 +2389,12 @@ impl ChatIdBlocked {
             .transaction(move |transaction| {
                 transaction.execute(
                     "INSERT INTO chats
-                     (type, name, param, blocked, created_timestamp)
-                     VALUES(?, ?, ?, ?, ?)",
+                     (type, name, name_normalized, param, blocked, created_timestamp)
+                     VALUES(?, ?, ?, ?, ?, ?)",
                     (
                         Chattype::Single,
-                        chat_name,
+                        &chat_name,
+                        normalize_text(&chat_name),
                         params.to_string(),
                         create_blocked as u8,
                         smeared_time,
@@ -2944,7 +2946,7 @@ pub(crate) async fn save_text_edit_to_db(
             "UPDATE msgs SET txt=?, txt_normalized=?, param=? WHERE id=?",
             (
                 new_text,
-                message::normalize_text(new_text),
+                normalize_text(new_text),
                 original_msg.param.to_string(),
                 original_msg.id,
             ),
@@ -3433,9 +3435,15 @@ pub(crate) async fn create_group_ex(
         .sql
         .insert(
             "INSERT INTO chats
-        (type, name, grpid, param, created_timestamp)
-        VALUES(?, ?, ?, \'U=1\', ?);",
-            (Chattype::Group, &chat_name, &grpid, timestamp),
+        (type, name, name_normalized, grpid, param, created_timestamp)
+        VALUES(?, ?, ?, ?, \'U=1\', ?)",
+            (
+                Chattype::Group,
+                &chat_name,
+                normalize_text(&chat_name),
+                &grpid,
+                timestamp,
+            ),
         )
         .await?;
 
@@ -3519,9 +3527,15 @@ pub(crate) async fn create_out_broadcast_ex(
 
         t.execute(
             "INSERT INTO chats
-            (type, name, grpid, created_timestamp)
-            VALUES(?, ?, ?, ?);",
-            (Chattype::OutBroadcast, &chat_name, &grpid, timestamp),
+            (type, name, name_normalized, grpid, created_timestamp)
+            VALUES(?, ?, ?, ?, ?)",
+            (
+                Chattype::OutBroadcast,
+                &chat_name,
+                normalize_text(&chat_name),
+                &grpid,
+                timestamp,
+            ),
         )?;
         let chat_id = ChatId::new(t.last_insert_rowid().try_into()?);
 
@@ -4094,8 +4108,8 @@ async fn rename_ex(
             context
                 .sql
                 .execute(
-                    "UPDATE chats SET name=? WHERE id=?;",
-                    (new_name.to_string(), chat_id),
+                    "UPDATE chats SET name=?, name_normalized=? WHERE id=?",
+                    (&new_name, normalize_text(&new_name), chat_id),
                 )
                 .await?;
             if chat.is_promoted()
@@ -4529,7 +4543,7 @@ pub async fn add_device_msg_with_importance(
                     msg.viewtype,
                     state,
                     &msg.text,
-                    message::normalize_text(&msg.text),
+                    normalize_text(&msg.text),
                     msg.param.to_string(),
                     rfc724_mid,
                 ),
@@ -4668,7 +4682,7 @@ pub(crate) async fn add_info_msg_with_cmd(
             Viewtype::Text,
             MessageState::InNoticed,
             text,
-            message::normalize_text(text),
+            normalize_text(text),
             rfc724_mid,
             ephemeral_timer,
             param.to_string(),
@@ -4710,7 +4724,7 @@ pub(crate) async fn update_msg_text_and_timestamp(
         .sql
         .execute(
             "UPDATE msgs SET txt=?, txt_normalized=?, timestamp=? WHERE id=?;",
-            (text, message::normalize_text(text), timestamp, msg_id),
+            (text, normalize_text(text), timestamp, msg_id),
         )
         .await?;
     context.emit_msgs_changed(chat_id, msg_id);

src/chatlist.rs

@@ -185,7 +185,7 @@ impl Chatlist {
                 warn!(context, "Cannot update special chat names: {err:#}.")
             }
 
-            let str_like_cmd = format!("%{query}%");
+            let str_like_cmd = format!("%{}%", query.to_lowercase());
             context
                 .sql
                 .query_map_vec(
@@ -201,7 +201,7 @@ impl Chatlist {
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked!=1
-                   AND c.name LIKE ?3
+                   AND IFNULL(c.name_normalized,c.name) LIKE ?3
                    AND (NOT ?4 OR EXISTS (SELECT 1 FROM msgs m WHERE m.chat_id = c.id AND m.state == ?5 AND hidden=0))
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
@@ -472,7 +472,7 @@ mod tests {
     use crate::chat::save_msgs;
     use crate::chat::{
         add_contact_to_chat, create_group, get_chat_contacts, remove_contact_from_chat,
-        send_text_msg,
+        send_text_msg, set_chat_name,
     };
     use crate::receive_imf::receive_imf;
     use crate::stock_str::StockMessage;
@@ -482,7 +482,7 @@ mod tests {
     use std::time::Duration;
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_try_load() {
+    async fn test_try_load() -> Result<()> {
         let mut tcm = TestContextManager::new();
         let bob = &tcm.bob().await;
         let chat_id1 = create_group(bob, "a chat").await.unwrap();
@@ -552,6 +552,15 @@ mod tests {
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
+
+        let chat_id = create_group(bob, "Δ-chat").await.unwrap();
+        let chats = Chatlist::try_load(bob, 0, Some("δ"), None).await?;
+        assert_eq!(chats.len(), 1);
+        assert_eq!(chats.ids[0].0, chat_id);
+        set_chat_name(bob, chat_id, "abcδe").await?;
+        let chats = Chatlist::try_load(bob, 0, Some("Δ"), None).await?;
+        assert_eq!(chats.len(), 1);
+        Ok(())
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]