From db91a316c00e8db192a66a9805846dd798b44f0d Mon Sep 17 00:00:00 2001 From: M66B Date: Sun, 12 Sep 2021 13:18:16 +0200 Subject: [PATCH] Refactoring --- .../email/FragmentDialogOpenLink.java | 174 +----------------- .../java/eu/faircode/email/UriHelper.java | 169 +++++++++++++++++ 2 files changed, 175 insertions(+), 168 deletions(-) diff --git a/app/src/main/java/eu/faircode/email/FragmentDialogOpenLink.java b/app/src/main/java/eu/faircode/email/FragmentDialogOpenLink.java index e96f0f4464..5b219f34b4 100644 --- a/app/src/main/java/eu/faircode/email/FragmentDialogOpenLink.java +++ b/app/src/main/java/eu/faircode/email/FragmentDialogOpenLink.java @@ -41,7 +41,6 @@ import android.text.TextWatcher; import android.text.method.LinkMovementMethod; import android.text.style.ForegroundColorSpan; import android.text.style.StyleSpan; -import android.util.Base64; import android.util.Pair; import android.view.KeyEvent; import android.view.LayoutInflater; @@ -66,50 +65,13 @@ import androidx.preference.PreferenceManager; import java.net.IDN; import java.net.InetAddress; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; public class FragmentDialogOpenLink extends FragmentDialogBase { private static final String URI_RESET_OPEN = "https://support.google.com/pixelphone/answer/6271667"; - // https://github.com/newhouse/url-tracking-stripper - private static final List PARANOID_QUERY = Collections.unmodifiableList(Arrays.asList( - // https://en.wikipedia.org/wiki/UTM_parameters - "awt_a", // AWeber - "awt_l", // AWeber - "awt_m", // AWeber - - "icid", // Adobe - "gclid", // Google - "gclsrc", // Google ads - "dclid", // DoubleClick (Google) - "fbclid", // Facebook - "igshid", // Instagram - - "mc_cid", // MailChimp - "mc_eid", // MailChimp - - "zanpid", // Zanox (Awin) - - "kclickid" // https://support.freespee.com/hc/en-us/articles/202577831-Kenshoo-integration - )); - - // https://github.com/snarfed/granary/blob/master/granary/facebook.py#L1789 - - private static final List FACEBOOK_WHITELIST_PATH = Collections.unmodifiableList(Arrays.asList( - "/nd/", "/n/", "/story.php" - )); - - private static final List FACEBOOK_WHITELIST_QUERY = Collections.unmodifiableList(Arrays.asList( - "story_fbid", "fbid", "id", "comment_id" - )); - private ImageButton ibMore; private TextView tvMore; private Button btnOwner; @@ -148,7 +110,7 @@ public class FragmentDialogOpenLink extends FragmentDialogBase { if (uri.isOpaque()) sanitized = uri; else { - Uri s = sanitize(uri); + Uri s = UriHelper.sanitize(uri); sanitized = (s == null ? uri : s); } @@ -222,10 +184,8 @@ public class FragmentDialogOpenLink extends FragmentDialogBase { public void afterTextChanged(Editable editable) { Uri uri = Uri.parse(editable.toString()); - boolean secure = (!uri.isOpaque() && - "https".equals(uri.getScheme())); - boolean hyperlink = (!uri.isOpaque() && - ("http".equals(uri.getScheme()) || "https".equals(uri.getScheme()))); + boolean secure = UriHelper.isSecure(uri); + boolean hyperlink = UriHelper.isHyperLink(uri); cbSecure.setTag(secure); cbSecure.setChecked(secure); @@ -312,7 +272,7 @@ public class FragmentDialogOpenLink extends FragmentDialogBase { return; Uri uri = Uri.parse(etLink.getText().toString()); - etLink.setText(format(secure(uri, checked), context)); + etLink.setText(format(UriHelper.secure(uri, checked), context)); } }); @@ -322,10 +282,10 @@ public class FragmentDialogOpenLink extends FragmentDialogBase { @Override public void onCheckedChanged(CompoundButton compoundButton, boolean checked) { Uri link = (checked ? sanitized : uri); - boolean secure = (!link.isOpaque() && "https".equals(link.getScheme())); + boolean secure = UriHelper.isSecure(link); cbSecure.setTag(secure); cbSecure.setChecked(secure); - etLink.setText(format(secure(link, secure), context)); + etLink.setText(format(UriHelper.secure(link, secure), context)); } }); @@ -538,128 +498,6 @@ public class FragmentDialogOpenLink extends FragmentDialogBase { tvReset.setVisibility(show ? View.VISIBLE : View.GONE); } - private static Uri sanitize(Uri uri) { - boolean changed = false; - - Uri url; - Uri.Builder builder; - if (uri.getHost() != null && - uri.getHost().endsWith("safelinks.protection.outlook.com") && - !TextUtils.isEmpty(uri.getQueryParameter("url"))) { - changed = true; - url = Uri.parse(uri.getQueryParameter("url")); - } else if ("https".equals(uri.getScheme()) && - "smex-ctp.trendmicro.com".equals(uri.getHost()) && - "/wis/clicktime/v1/query".equals(uri.getPath()) && - !TextUtils.isEmpty(uri.getQueryParameter("url"))) { - changed = true; - url = Uri.parse(uri.getQueryParameter("url")); - } else if ("https".equals(uri.getScheme()) && - "www.google.com".equals(uri.getHost()) && - uri.getPath() != null && - uri.getPath().startsWith("/amp/")) { - // https://blog.amp.dev/2017/02/06/whats-in-an-amp-url/ - Uri result = null; - - String u = uri.toString(); - u = u.replace("https://www.google.com/amp/", ""); - - int p = u.indexOf("/"); - while (p > 0) { - String segment = u.substring(0, p); - if (segment.contains(".")) { - result = Uri.parse("https://" + u); - break; - } - - u = u.substring(p + 1); - p = u.indexOf("/"); - } - - changed = (result != null); - url = (result == null ? uri : result); - } else if (uri.getQueryParameterNames().size() == 1) { - // Sophos Email Appliance - Uri result = null; - String key = uri.getQueryParameterNames().iterator().next(); - if (TextUtils.isEmpty(uri.getQueryParameter(key))) - try { - String data = new String(Base64.decode(key, Base64.DEFAULT)); - int u = data.indexOf("&&url="); - if (u > 0) - result = Uri.parse(URLDecoder.decode(data.substring(u + 6), StandardCharsets.UTF_8.name())); - } catch (Throwable ex) { - Log.w(ex); - } - - changed = (result != null); - url = (result == null ? uri : result); - } else - url = uri; - - if (url.isOpaque()) - return uri; - - builder = url.buildUpon(); - - builder.clearQuery(); - String host = uri.getHost(); - String path = uri.getPath(); - if (host != null) - host = host.toLowerCase(Locale.ROOT); - if (path != null) - path = path.toLowerCase(Locale.ROOT); - boolean first = "www.facebook.com".equals(host); - for (String key : url.getQueryParameterNames()) { - // https://en.wikipedia.org/wiki/UTM_parameters - // https://docs.oracle.com/en/cloud/saas/marketing/eloqua-user/Help/EloquaAsynchronousTrackingScripts/EloquaTrackingParameters.htm - String lkey = key.toLowerCase(Locale.ROOT); - if (PARANOID_QUERY.contains(lkey) || - lkey.startsWith("utm_") || - lkey.startsWith("elq") || - ((host != null && host.endsWith("facebook.com")) && - !first && - FACEBOOK_WHITELIST_PATH.contains(path) && - !FACEBOOK_WHITELIST_QUERY.contains(lkey)) || - ("store.steampowered.com".equals(host) && - "snr".equals(lkey))) - changed = true; - else if (!TextUtils.isEmpty(key)) - for (String value : url.getQueryParameters(key)) { - Log.i("Query " + key + "=" + value); - Uri suri = Uri.parse(value); - if ("http".equals(suri.getScheme()) || "https".equals(suri.getScheme())) { - Uri s = sanitize(suri); - if (s != null) { - changed = true; - value = s.toString(); - } - } - builder.appendQueryParameter(key, value); - } - first = false; - } - - return (changed ? builder.build() : null); - } - - private static Uri secure(Uri uri, boolean https) { - String scheme = uri.getScheme(); - if (https ? "http".equals(scheme) : "https".equals(scheme)) { - Uri.Builder builder = uri.buildUpon(); - builder.scheme(https ? "https" : "http"); - - String authority = uri.getEncodedAuthority(); - if (authority != null) { - authority = authority.replace(https ? ":80" : ":443", https ? ":443" : ":80"); - builder.encodedAuthority(authority); - } - - return builder.build(); - } else - return uri; - } - private Spanned format(Uri uri, Context context) { String scheme = uri.getScheme(); String host = uri.getHost(); diff --git a/app/src/main/java/eu/faircode/email/UriHelper.java b/app/src/main/java/eu/faircode/email/UriHelper.java index c46777e4b7..fd8a6fc977 100644 --- a/app/src/main/java/eu/faircode/email/UriHelper.java +++ b/app/src/main/java/eu/faircode/email/UriHelper.java @@ -22,6 +22,7 @@ package eu.faircode.email; import android.content.Context; import android.net.Uri; import android.text.TextUtils; +import android.util.Base64; import android.webkit.URLUtil; import androidx.annotation.NonNull; @@ -30,7 +31,12 @@ import androidx.core.util.PatternsCompat; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Locale; public class UriHelper { @@ -40,6 +46,38 @@ public class UriHelper { // https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat private static final String SUFFIX_LIST_NAME = "public_suffix_list.dat"; + // https://github.com/newhouse/url-tracking-stripper + private static final List PARANOID_QUERY = Collections.unmodifiableList(Arrays.asList( + // https://en.wikipedia.org/wiki/UTM_parameters + "awt_a", // AWeber + "awt_l", // AWeber + "awt_m", // AWeber + + "icid", // Adobe + "gclid", // Google + "gclsrc", // Google ads + "dclid", // DoubleClick (Google) + "fbclid", // Facebook + "igshid", // Instagram + + "mc_cid", // MailChimp + "mc_eid", // MailChimp + + "zanpid", // Zanox (Awin) + + "kclickid" // https://support.freespee.com/hc/en-us/articles/202577831-Kenshoo-integration + )); + + // https://github.com/snarfed/granary/blob/master/granary/facebook.py#L1789 + + private static final List FACEBOOK_WHITELIST_PATH = Collections.unmodifiableList(Arrays.asList( + "/nd/", "/n/", "/story.php" + )); + + private static final List FACEBOOK_WHITELIST_QUERY = Collections.unmodifiableList(Arrays.asList( + "story_fbid", "fbid", "id", "comment_id" + )); + static String getParentDomain(Context context, String host) { if (host == null) return null; @@ -147,4 +185,135 @@ public class UriHelper { } } } + + static Uri sanitize(Uri uri) { + boolean changed = false; + + Uri url; + Uri.Builder builder; + if (uri.getHost() != null && + uri.getHost().endsWith("safelinks.protection.outlook.com") && + !TextUtils.isEmpty(uri.getQueryParameter("url"))) { + changed = true; + url = Uri.parse(uri.getQueryParameter("url")); + } else if ("https".equals(uri.getScheme()) && + "smex-ctp.trendmicro.com".equals(uri.getHost()) && + "/wis/clicktime/v1/query".equals(uri.getPath()) && + !TextUtils.isEmpty(uri.getQueryParameter("url"))) { + changed = true; + url = Uri.parse(uri.getQueryParameter("url")); + } else if ("https".equals(uri.getScheme()) && + "www.google.com".equals(uri.getHost()) && + uri.getPath() != null && + uri.getPath().startsWith("/amp/")) { + // https://blog.amp.dev/2017/02/06/whats-in-an-amp-url/ + Uri result = null; + + String u = uri.toString(); + u = u.replace("https://www.google.com/amp/", ""); + + int p = u.indexOf("/"); + while (p > 0) { + String segment = u.substring(0, p); + if (segment.contains(".")) { + result = Uri.parse("https://" + u); + break; + } + + u = u.substring(p + 1); + p = u.indexOf("/"); + } + + changed = (result != null); + url = (result == null ? uri : result); + } else if (uri.getQueryParameterNames().size() == 1) { + // Sophos Email Appliance + Uri result = null; + String key = uri.getQueryParameterNames().iterator().next(); + if (TextUtils.isEmpty(uri.getQueryParameter(key))) + try { + String data = new String(Base64.decode(key, Base64.DEFAULT)); + int u = data.indexOf("&&url="); + if (u > 0) + result = Uri.parse(URLDecoder.decode(data.substring(u + 6), StandardCharsets.UTF_8.name())); + } catch (Throwable ex) { + Log.w(ex); + } + + changed = (result != null); + url = (result == null ? uri : result); + } else + url = uri; + + if (url.isOpaque()) + return uri; + + builder = url.buildUpon(); + + builder.clearQuery(); + String host = uri.getHost(); + String path = uri.getPath(); + if (host != null) + host = host.toLowerCase(Locale.ROOT); + if (path != null) + path = path.toLowerCase(Locale.ROOT); + boolean first = "www.facebook.com".equals(host); + for (String key : url.getQueryParameterNames()) { + // https://en.wikipedia.org/wiki/UTM_parameters + // https://docs.oracle.com/en/cloud/saas/marketing/eloqua-user/Help/EloquaAsynchronousTrackingScripts/EloquaTrackingParameters.htm + String lkey = key.toLowerCase(Locale.ROOT); + if (PARANOID_QUERY.contains(lkey) || + lkey.startsWith("utm_") || + lkey.startsWith("elq") || + ((host != null && host.endsWith("facebook.com")) && + !first && + FACEBOOK_WHITELIST_PATH.contains(path) && + !FACEBOOK_WHITELIST_QUERY.contains(lkey)) || + ("store.steampowered.com".equals(host) && + "snr".equals(lkey))) + changed = true; + else if (!TextUtils.isEmpty(key)) + for (String value : url.getQueryParameters(key)) { + Log.i("Query " + key + "=" + value); + Uri suri = Uri.parse(value); + if ("http".equals(suri.getScheme()) || "https".equals(suri.getScheme())) { + Uri s = sanitize(suri); + if (s != null) { + changed = true; + value = s.toString(); + } + } + builder.appendQueryParameter(key, value); + } + first = false; + } + + return (changed ? builder.build() : null); + } + + static Uri secure(Uri uri, boolean https) { + String scheme = uri.getScheme(); + if (https ? "http".equals(scheme) : "https".equals(scheme)) { + Uri.Builder builder = uri.buildUpon(); + builder.scheme(https ? "https" : "http"); + + String authority = uri.getEncodedAuthority(); + if (authority != null) { + authority = authority.replace(https ? ":80" : ":443", https ? ":443" : ":80"); + builder.encodedAuthority(authority); + } + + return builder.build(); + } else + return uri; + } + + static boolean isSecure(Uri uri) { + return (!uri.isOpaque() && "https".equals(uri.getScheme())); + } + + static boolean isHyperLink(Uri uri) { + return (!uri.isOpaque() && + ("http".equals(uri.getScheme()) || "https".equals(uri.getScheme()))); + } }