From df3c2ea1471c0f911ba1048b2470d753d49b51b2 Mon Sep 17 00:00:00 2001
From: M66B <M66B@users.noreply.github.com>
Date: Thu, 29 Sep 2022 14:34:17 +0200
Subject: [PATCH] Remove diacrits from index/query

---
 .../email/BoundaryCallbackMessages.java       | 28 ++++++++++++-------
 .../java/eu/faircode/email/Fts4DbHelper.java  | 11 ++++++--
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java b/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java
index f218258124..74b9519859 100644
--- a/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java
+++ b/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java
@@ -345,23 +345,21 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
                     }
 
                     if (!matched && criteria.in_subject) {
-                        if (message.subject != null &&
-                                message.subject.toLowerCase().contains(query))
+                        if (contains(message.subject, query))
                             matched = true;
                     }
 
                     if (!matched && criteria.in_keywords) {
                         if (message.keywords != null)
                             for (String keyword : message.keywords)
-                                if (keyword.toLowerCase().contains(query)) {
+                                if (contains(keyword, query)) {
                                     matched = true;
                                     break;
                                 }
                     }
 
                     if (!matched && criteria.in_notes) {
-                        if (message.notes != null &&
-                                message.notes.toLowerCase().contains(query))
+                        if (contains(message.notes, query))
                             matched = true;
                     }
 
@@ -370,10 +368,9 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
                             File file = EntityMessage.getFile(context, id);
                             if (file.exists()) {
                                 String html = Helper.readText(file);
-                                if (html.toLowerCase().contains(query)) {
+                                if (contains(html, query)) {
                                     String text = HtmlHelper.getFullText(html);
-                                    if (text != null &&
-                                            text.toLowerCase().contains(query))
+                                    if (contains(text, query))
                                         matched = true;
                                 }
                             }
@@ -782,15 +779,26 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
         return imessages;
     }
 
-    private static boolean contains(Address[] addresses, String text) {
+    private static boolean contains(Address[] addresses, String query) {
         if (addresses == null)
             return false;
         for (Address address : addresses)
-            if (address.toString().toLowerCase().contains(text))
+            if (contains(address.toString(), query))
                 return true;
         return false;
     }
 
+    private static boolean contains(String text, String query) {
+        if (TextUtils.isEmpty(text))
+            return false;
+        text = text.toLowerCase();
+        if (text.contains(query))
+            return true;
+        String normalized = Normalizer.normalize(text, Normalizer.Form.NFKD)
+                .replaceAll("[\\p{InCombiningDiacriticalMarks}]", "");
+        return normalized.contains(query);
+    }
+
     State getState() {
         return this.state;
     }
diff --git a/app/src/main/java/eu/faircode/email/Fts4DbHelper.java b/app/src/main/java/eu/faircode/email/Fts4DbHelper.java
index 7c7eed410b..e00dd4cc44 100644
--- a/app/src/main/java/eu/faircode/email/Fts4DbHelper.java
+++ b/app/src/main/java/eu/faircode/email/Fts4DbHelper.java
@@ -29,6 +29,7 @@ import android.os.Build;
 import android.text.TextUtils;
 
 import java.io.File;
+import java.text.Normalizer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -145,8 +146,10 @@ public class Fts4DbHelper extends SQLiteOpenHelper {
         boundary.setText(text);
         int start = boundary.first();
         for (int end = boundary.next(); end != android.icu.text.BreakIterator.DONE; end = boundary.next()) {
-            String word = text.substring(start, end).trim();
+            String word = text.substring(start, end).trim().toLowerCase();
             if (!TextUtils.isEmpty(word)) {
+                word = Normalizer.normalize(word, Normalizer.Form.NFKD)
+                        .replaceAll("[\\p{InCombiningDiacriticalMarks}]", "");
                 if (sb.length() > 0)
                     sb.append(' ');
                 sb.append(word);
@@ -179,12 +182,16 @@ public class Fts4DbHelper extends SQLiteOpenHelper {
             Long account, Long folder, long[] exclude,
             BoundaryCallbackMessages.SearchCriteria criteria) {
 
+        String query = criteria.query.trim();
+        query = Normalizer.normalize(query, Normalizer.Form.NFKD)
+                .replaceAll("[\\p{InCombiningDiacriticalMarks}]", "");
+
         List<String> word = new ArrayList<>();
         List<String> plus = new ArrayList<>();
         List<String> minus = new ArrayList<>();
         List<String> opt = new ArrayList<>();
         StringBuilder all = new StringBuilder();
-        for (String w : criteria.query.trim().split("\\s+")) {
+        for (String w : query.split("\\s+")) {
             if (all.length() > 0)
                 all.append(' ');