Mirrors/FairEmail
1
0
Fork 0
mirror of https://github.com/M66B/FairEmail.git synced 2026-05-19 13:34:00 +02:00
Code Issues Packages Projects Releases Wiki Activity

Experiment: search attachment content

Browse Source
This commit is contained in:
M66B
2023-08-09 18:54:37 +02:00
parent 9276018795
commit b03f0ebe6b
10 changed files with 199 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ATTRIBUTION.md
View File

@@ -47,3 +47,4 @@ FairEmail uses:
* [MaterialDings](https://github.com/Accusoft/MaterialDings). Copyright (c) 2018 Accusoft Corporation. [MIT License](https://github.com/Accusoft/MaterialDings/blob/master/LICENSE.md).
* [Send](https://github.com/timvisee/send). [Mozilla Public License 2.0](https://github.com/timvisee/send/blob/master/LICENSE).
* [DetectHtml](https://github.com/dbennett455/DetectHtml). [The MIT License](https://github.com/dbennett455/DetectHtml/blob/master/LICENSE).
* [Elephant Bird](https://github.com/twitter/elephant-bird). [Apache License Version 2.0](https://github.com/twitter/elephant-bird/blob/master/LICENSE).

1
app/src/main/assets/ATTRIBUTION.md
View File

@@ -47,3 +47,4 @@ FairEmail uses:
* [MaterialDings](https://github.com/Accusoft/MaterialDings). Copyright (c) 2018 Accusoft Corporation. [MIT License](https://github.com/Accusoft/MaterialDings/blob/master/LICENSE.md).
* [Send](https://github.com/timvisee/send). [Mozilla Public License 2.0](https://github.com/timvisee/send/blob/master/LICENSE).
* [DetectHtml](https://github.com/dbennett455/DetectHtml). [The MIT License](https://github.com/dbennett455/DetectHtml/blob/master/LICENSE).
* [Elephant Bird](https://github.com/twitter/elephant-bird). [Apache License Version 2.0](https://github.com/twitter/elephant-bird/blob/master/LICENSE).

87
app/src/main/java/com/twitter/elephantbird/util/StreamSearcher.java Normal file
View File

@@ -0,0 +1,87 @@
package com.twitter.elephantbird.util;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
/**
* An efficient stream searching class based on the Knuth-Morris-Pratt algorithm.
* For more on the algorithm works see: https://www.inf.fh-flensburg.de/lang/algorithmen/pattern/kmpen.htm.
*/
public class StreamSearcher {
protected byte[] pattern_;
protected int[] borders_;
// An upper bound on pattern length for searching. Results are undefined for longer patterns.
public static final int MAX_PATTERN_LENGTH = 1024;
public StreamSearcher(byte[] pattern) {
setPattern(pattern);
}
/**
* Sets a new pattern for this StreamSearcher to use.
* @param pattern
* the pattern the StreamSearcher will look for in future calls to search(...)
*/
public void setPattern(byte[] pattern) {
pattern_ = Arrays.copyOf(pattern, pattern.length);
borders_ = new int[pattern_.length + 1];
preProcess();
}
/**
* Searches for the next occurrence of the pattern in the stream, starting from the current stream position. Note
* that the position of the stream is changed. If a match is found, the stream points to the end of the match -- i.e. the
* byte AFTER the pattern. Else, the stream is entirely consumed. The latter is because InputStream semantics make it difficult to have
* another reasonable default, i.e. leave the stream unchanged.
*
* @return bytes consumed if found, -1 otherwise.
* @throws IOException
*/
public long search(InputStream stream) throws IOException {
long bytesRead = 0;
int b;
int j = 0;
while ((b = stream.read()) != -1) {
bytesRead++;
while (j >= 0 && (byte)b != pattern_[j]) {
j = borders_[j];
}
// Move to the next character in the pattern.
++j;
// If we've matched up to the full pattern length, we found it. Return,
// which will automatically save our position in the InputStream at the point immediately
// following the pattern match.
if (j == pattern_.length) {
return bytesRead;
}
}
// No dice, Note that the stream is now completely consumed.
return -1;
}
/**
* Builds up a table of longest "borders" for each prefix of the pattern to find. This table is stored internally
* and aids in implementation of the Knuth-Moore-Pratt string search.
* <p>
* For more information, see: https://www.inf.fh-flensburg.de/lang/algorithmen/pattern/kmpen.htm.
*/
protected void preProcess() {
int i = 0;
int j = -1;
borders_[i] = j;
while (i < pattern_.length) {
while (j >= 0 && pattern_[i] != pattern_[j]) {
j = borders_[j];
}
borders_[++i] = ++j;
}
}
}

1
app/src/main/java/eu/faircode/email/ActivityView.java
View File

@@ -2427,6 +2427,7 @@ public class ActivityView extends ActivityBilling implements FragmentManager.OnB
criteria.in_subject = false;
criteria.in_keywords = false;
criteria.in_message = false;
criteria.in_attachments = false;
criteria.in_notes = false;
}

2
app/src/main/java/eu/faircode/email/AdapterFolder.java
View File

@@ -525,6 +525,7 @@ public class AdapterFolder extends RecyclerView.Adapter<AdapterFolder.ViewHolder
criteria.in_subject = false;
criteria.in_keywords = false;
criteria.in_message = false;
criteria.in_attachments = false;
criteria.in_notes = false;
criteria.with_unseen = true;
FragmentMessages.search(
@@ -539,6 +540,7 @@ public class AdapterFolder extends RecyclerView.Adapter<AdapterFolder.ViewHolder
criteria.in_subject = false;
criteria.in_keywords = false;
criteria.in_message = false;
criteria.in_attachments = false;
criteria.in_notes = false;
criteria.with_flagged = true;
FragmentMessages.search(

41
app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java
View File

@@ -39,6 +39,7 @@ import com.sun.mail.imap.protocol.IMAPProtocol;
import com.sun.mail.imap.protocol.IMAPResponse;
import com.sun.mail.imap.protocol.SearchSequence;
import com.sun.mail.util.MessageRemovedIOException;
import com.twitter.elephantbird.util.StreamSearcher;
import org.json.JSONArray;
import org.json.JSONException;
@@ -46,9 +47,12 @@ import org.json.JSONObject;
import org.jsoup.nodes.Document;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.lang.reflect.Constructor;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.text.Normalizer;
import java.util.ArrayList;
@@ -341,6 +345,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
//criteria.in_subject,
//criteria.in_keywords,
//criteria.in_message,
//criteria.in_attachments,
//criteria.in_notes,
//criteria.in_headers,
criteria.with_unseen,
@@ -866,6 +871,37 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
Log.e(ex);
}
if (criteria.in_attachments && !TextUtils.isEmpty(criteria.query))
try {
DB db = DB.getInstance(context);
List<EntityAttachment> attachments = db.attachment().getAttachments(message.id);
if (attachments != null)
for (EntityAttachment attachment : attachments) {
File file = attachment.getFile(context);
if (file.exists() && file.length() > 0) {
byte[] sample = new byte[(int) Math.min(4096, file.length())];
try (InputStream is = new FileInputStream(file)) {
Helper.readBuffer(is, sample);
}
Charset detected = CharsetHelper.detect(sample, null);
if (detected == null)
detected = StandardCharsets.ISO_8859_1;
Log.i("Searching for " + criteria.query +
" as " + detected +
" in " + file.getName() + ":" + file.length());
try (InputStream is = new FileInputStream(file)) {
StreamSearcher searcher = new StreamSearcher(criteria.query.getBytes(detected));
if (searcher.search(is) > 0)
return true;
}
}
}
} catch (Throwable ex) {
Log.e(ex);
}
return false;
}
@@ -987,6 +1023,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
boolean in_subject = true;
boolean in_keywords = true;
boolean in_message = true;
boolean in_attachments = false;
boolean in_notes = true;
boolean in_headers = false;
boolean in_html = false;
@@ -1272,6 +1309,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
this.in_subject == other.in_subject &&
this.in_keywords == other.in_keywords &&
this.in_message == other.in_message &&
this.in_attachments == other.in_attachments &&
this.in_notes == other.in_notes &&
this.in_headers == other.in_headers &&
this.in_html == other.in_html &&
@@ -1300,6 +1338,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
json.put("in_subject", in_subject);
json.put("in_keywords", in_keywords);
json.put("in_message", in_message);
json.put("in_attachments", in_attachments);
json.put("in_notes", in_notes);
json.put("in_headers", in_headers);
json.put("in_html", in_html);
@@ -1347,6 +1386,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
criteria.in_subject = json.optBoolean("in_subject");
criteria.in_keywords = json.optBoolean("in_keywords");
criteria.in_message = json.optBoolean("in_message");
criteria.in_attachments = json.optBoolean("in_attachments");
criteria.in_notes = json.optBoolean("in_notes");
criteria.in_headers = json.optBoolean("in_headers");
criteria.in_html = json.optBoolean("in_html");
@@ -1395,6 +1435,7 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
" subject=" + in_subject +
" keywords=" + in_keywords +
" message=" + in_message +
" attachment=" + in_attachments +
" notes=" + in_notes +
" headers=" + in_headers +
" html=" + in_html +

43
app/src/main/java/eu/faircode/email/CharsetHelper.java
View File

@@ -225,19 +225,7 @@ public class CharsetHelper {
try {
byte[] octets = text.getBytes(StandardCharsets.ISO_8859_1);
byte[] sample;
if (octets.length < MAX_SAMPLE_SIZE)
sample = octets;
else {
sample = new byte[MAX_SAMPLE_SIZE];
System.arraycopy(octets, 0, sample, 0, MAX_SAMPLE_SIZE);
}
Log.i("compact_enc_det sample=" + sample.length);
DetectResult detected = jni_detect_charset(sample,
ref == null ? StandardCharsets.ISO_8859_1.name() : ref.name(),
Locale.getDefault().getLanguage());
DetectResult detected = _detect(octets, ref);
if (TextUtils.isEmpty(detected.charset)) {
Log.e("compact_enc_det result=" + detected);
@@ -262,7 +250,34 @@ public class CharsetHelper {
}
}
private static class DetectResult {
public static Charset detect(byte[] octets, Charset ref) {
try {
DetectResult detected = _detect(octets, ref);
if (TextUtils.isEmpty(detected.charset))
return null;
return Charset.forName(detected.charset);
} catch (Throwable ex) {
Log.w(ex);
return null;
}
}
private static DetectResult _detect(byte[] octets, Charset ref) {
byte[] sample;
if (octets.length < MAX_SAMPLE_SIZE)
sample = octets;
else {
sample = new byte[MAX_SAMPLE_SIZE];
System.arraycopy(octets, 0, sample, 0, MAX_SAMPLE_SIZE);
}
Log.i("compact_enc_det sample=" + sample.length);
return jni_detect_charset(sample,
ref == null ? StandardCharsets.ISO_8859_1.name() : ref.name(),
Locale.getDefault().getLanguage());
}
public static class DetectResult {
String charset;
int sample_size;
int bytes_consumed;

6
app/src/main/java/eu/faircode/email/FragmentDialogSearch.java
View File

@@ -114,6 +114,7 @@ public class FragmentDialogSearch extends FragmentDialogBase {
CheckBox cbKeywords = dview.findViewById(R.id.cbKeywords);
CheckBox cbMessage = dview.findViewById(R.id.cbMessage);
TextView tvSearchTextUnsupported = dview.findViewById(R.id.tvSearchTextUnsupported);
CheckBox cbAttachments = dview.findViewById(R.id.cbAttachments);
CheckBox cbNotes = dview.findViewById(R.id.cbNotes);
CheckBox cbHeaders = dview.findViewById(R.id.cbHeaders);
CheckBox cbHtml = dview.findViewById(R.id.cbHtml);
@@ -123,7 +124,7 @@ public class FragmentDialogSearch extends FragmentDialogBase {
CheckBox cbFlagged = dview.findViewById(R.id.cbFlagged);
CheckBox cbHidden = dview.findViewById(R.id.cbHidden);
CheckBox cbEncrypted = dview.findViewById(R.id.cbEncrypted);
CheckBox cbAttachments = dview.findViewById(R.id.cbAttachments);
CheckBox cbWithAttachments = dview.findViewById(R.id.cbWithAttachments);
Spinner spMessageSize = dview.findViewById(R.id.spMessageSize);
Button btnBefore = dview.findViewById(R.id.btnBefore);
Button btnAfter = dview.findViewById(R.id.btnAfter);
@@ -430,6 +431,7 @@ public class FragmentDialogSearch extends FragmentDialogBase {
criteria.in_subject = cbSubject.isChecked();
criteria.in_keywords = cbKeywords.isChecked();
criteria.in_message = cbMessage.isChecked();
criteria.in_attachments = cbAttachments.isChecked();
criteria.in_notes = cbNotes.isChecked();
criteria.in_headers = cbHeaders.isChecked();
criteria.in_html = cbHtml.isChecked();
@@ -437,7 +439,7 @@ public class FragmentDialogSearch extends FragmentDialogBase {
criteria.with_flagged = cbFlagged.isChecked();
criteria.with_hidden = cbHidden.isChecked();
criteria.with_encrypted = cbEncrypted.isChecked();
criteria.with_attachments = cbAttachments.isChecked();
criteria.with_attachments = cbWithAttachments.isChecked();
if (!criteria.fts) {
int pos = spMessageSize.getSelectedItemPosition();

36
app/src/main/res/layout/dialog_search.xml
View File

@@ -337,6 +337,29 @@
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/tvSearchTextHint" />
<CheckBox
android:id="@+id/cbAttachments"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_marginTop="12dp"
android:text="@string/title_search_in_attachments"
android:textAppearance="@style/TextAppearance.AppCompat.Medium"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/tvSearchTextUnsupported" />
<TextView
android:id="@+id/tvSearchAttachmentsHint"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_marginTop="6dp"
android:text="@string/title_search_attachments_hint"
android:textAppearance="@style/TextAppearance.AppCompat.Small"
android:textStyle="italic"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/cbAttachments" />
<CheckBox
android:id="@+id/cbNotes"
android:layout_width="0dp"
@@ -347,7 +370,7 @@
android:textAppearance="@style/TextAppearance.AppCompat.Medium"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/tvSearchTextUnsupported" />
app:layout_constraintTop_toBottomOf="@id/tvSearchAttachmentsHint" />
<CheckBox
android:id="@+id/cbHeaders"
@@ -450,7 +473,7 @@
app:layout_constraintTop_toBottomOf="@id/cbHidden" />
<CheckBox
android:id="@+id/cbAttachments"
android:id="@+id/cbWithAttachments"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_marginTop="12dp"
@@ -470,7 +493,7 @@
android:visibility="gone"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@id/cbAttachments" />
app:layout_constraintTop_toBottomOf="@id/cbWithAttachments" />
<Spinner
android:id="@+id/spMessageSize"
@@ -569,8 +592,11 @@
android:layout_width="0dp"
android:layout_height="0dp"
app:constraint_referenced_ids="
cbSearchIndex,tvSearchIndexHint,cbSenders,cbRecipients,cbSubject,cbKeywords,cbMessage,tvSearchTextHint,tvSearchTextUnsupported,cbNotes,
tvAnd,cbUnseen,cbFlagged,cbHidden,cbEncrypted,cbAttachments,
cbSearchIndex,tvSearchIndexHint,
cbSenders,cbRecipients,cbSubject,cbKeywords,
cbMessage,tvSearchTextHint,tvSearchTextUnsupported,
cbAttachments,tvSearchAttachmentsHint,cbNotes,
tvAnd,cbUnseen,cbFlagged,cbHidden,cbEncrypted,cbWithAttachments,
tvDate,btnAfter,btnBefore,tvBefore,tvAfter,cbSearchDevice" />
</androidx.constraintlayout.widget.ConstraintLayout>
</eu.faircode.email.ScrollViewEx>

2
app/src/main/res/values/strings.xml
View File

@@ -1763,6 +1763,7 @@
<string name="title_search_index_hint">Searching via the search index is fast, but only finds whole words.</string>
<string name="title_search_text_hint">Searching for text in messages, when there are a large number of messages, might not work on some servers</string>
<string name="title_search_text_unsupported">\'%s\' means that the mail server doesn\'t support searching in message texts</string>
<string name="title_search_attachments_hint">This will be slow when there are many or large attachments</string>
<string name="title_search_size_hint">Searching for messages by size, when there are a large number of messages, might not work on some servers</string>
<string name="title_search_more">More options</string>
<string name="title_search_use_index">Use search index</string>
@@ -1771,6 +1772,7 @@
<string name="title_search_in_subject">In subject</string>
<string name="title_search_in_keywords">In keywords (if supported)</string>
<string name="title_search_in_message">In message text</string>
<string name="title_search_in_attachments">In attachments (on device only)</string>
<string name="title_search_in_notes">In local notes</string>
<string name="title_search_in_headers" translatable="false">In headers</string>
<string name="title_search_in_html" translatable="false">In HTML</string>