Add MozillaPreloadHSTSAdapter

Closes #11

Author: jayvdb

README.md

@@ -52,6 +52,7 @@ Adapters can be mounted on 'http://', or a narrower mount point.
 * HTTPRedirectBlockAdapter - Mount on 'https://' to block HTTPS responses redirecting to HTTP
 * HTTPSEverywhereOnlyAdapter - Apply HTTPS Everywhere rules
 * ChromePreloadHSTSAdapter - Upgrade to HTTPS for sites on Chrome preload list
+* MozillaPreloadHSTSAdapter - Upgrade to HTTPS for sites on Mozilla preload list
 * HTTPSEverywhereAdapter - Chrome preload hsts and https everywhere rules combined
 * ForceHTTPSAdapter - Just use HTTPS, always, everywhere
 * PreferHTTPSAdapter - Check HTTP if there are any redirects, before switching to HTTPS.

https_everywhere/_chrome_preload_hsts.py

@@ -1,5 +1,4 @@
 import json
-import logging
 import os.path
 
 import requests
@@ -15,11 +14,12 @@
 
 
 def _fetch_preload():
-    filename = _storage_location("transport_security_state_static.json")
+    filename = _storage_location(_github_url)
     if os.path.exists(filename):
         return filename
 
     r = requests.get(_github_url)
+    r.raise_for_status()
 
     with open(filename, "w") as f:
         f.write(r.text)

https_everywhere/_fetch.py

@@ -21,10 +21,13 @@ def _storage_location(filename=None, timestamp=None):
         except (IOError, OSError):
             pass
 
-    if timestamp:
-        filename = "default.rulesets.{}".format(timestamp)
-
     if filename:
+        if "/" in filename:
+            filename = os.path.basename(filename)
+
+        if timestamp:
+            filename = "{}.{}".format(filename, timestamp)
+
         return os.path.join(cache_dir, filename)
 
     return cache_dir
@@ -48,7 +51,7 @@ def _get_local_ts():
 def _get_local(timestamp=None):
     if not timestamp:
         timestamp = _get_local_ts()  # pragma: no cover
-    location = _storage_location(timestamp=timestamp)
+    location = _storage_location("default.rulesets", timestamp)
     if os.path.exists(location):
         with open(location) as f:
             return json.load(f)
@@ -68,7 +71,7 @@ def fetch_update(timestamp=None):
         ruleset_url, headers={"Accept-Encoding": "gzip, deflate, br"}, stream=True
     )
     r.raise_for_status()
-    location = _storage_location(timestamp=timestamp)
+    location = _storage_location("default.rulesets", timestamp)
     try:
         data = gzip.GzipFile(fileobj=r.raw).read()
     except Exception:

https_everywhere/_mozilla_preload_hsts.py

@@ -0,0 +1,78 @@
+import os.path
+
+import requests
+
+from logging_helper import setup_logging
+
+from ._fetch import _storage_location
+from ._util import _check_in, _reverse_host
+
+logger = setup_logging()
+
+_hg_url = "https://hg.mozilla.org/releases/mozilla-{version}/raw-file/tip/security/manager/ssl/nsSTSPreloadList.inc"
+_VERSIONS = ["beta", "release"]
+
+
+def _fetch_preload(version="release"):
+    filename = _storage_location(_hg_url, version)
+    if os.path.exists(filename):
+        return filename
+
+    r = requests.get(_hg_url.format(version=version))
+    r.raise_for_status()
+
+    with open(filename, "w") as f:
+        f.write(r.text)
+
+    return filename
+
+
+def _load_preload_data(filename):
+    with open(filename) as f:
+        positive = set()
+        negative = set()
+        lines = [line.strip() for line in f.readlines()]
+        start = lines.index("%%")
+        lines = lines[start + 1 :]
+        end = lines.index("%%")
+        lines = lines[:end]
+        for line in lines:
+            name, flag = line.split(",")
+            name = name.strip()
+            if flag.strip() == "1":
+                positive.add(name)
+            else:
+                negative.add(name)
+        return positive, negative
+
+
+def _preload_remove_negative(remove_overlap=False):
+    filename = _fetch_preload()
+    domains, negative = _load_preload_data(filename)
+
+    for name in negative:
+        rv = _check_in(domains, name)
+        if rv:
+            logger.warning("Removing {} because of negative {}".format(rv, name))
+            domains.remove(rv)
+
+    if remove_overlap:
+        entries = {}
+        for name in domains:
+            reversed_name = _reverse_host(name)
+            assert reversed_name not in entries
+            entries[reversed_name] = name
+
+        previous = ""
+        for item in sorted(entries.keys()):
+            entry = entries[item]
+            if not previous or previous not in item:
+                previous = item
+                continue
+
+            domains.remove(entry)
+            logger.warning(
+                "Removing {} because of base domain {}".format(entry, entries[previous])
+            )
+
+    return domains

https_everywhere/adapter.py

@@ -11,6 +11,7 @@
 
 from ._rules import https_url_rewrite, _get_rulesets
 from ._chrome_preload_hsts import _preload_including_subdomains
+from ._mozilla_preload_hsts import _preload_remove_negative
 from ._util import _check_in
 
 PY2 = str != "".__class__
@@ -146,11 +147,11 @@ def get_redirect(self, url):
         return super(HTTPSEverywhereOnlyAdapter, self).get_redirect(url)
 
 
-class ChromePreloadHSTSAdapter(RedirectAdapter):
+class PreloadHSTSAdapter(RedirectAdapter):
     def __init__(self, *args, **kwargs):
-        super(ChromePreloadHSTSAdapter, self).__init__(*args, **kwargs)
+        super(PreloadHSTSAdapter, self).__init__(*args, **kwargs)
         # prime cache
-        self._domains = _preload_including_subdomains()
+        self._domains = self._get_preload()
 
     def get_redirect(self, url):
         if url.startswith("http://"):
@@ -159,7 +160,15 @@ def get_redirect(self, url):
                 new_url = "https:" + url[5:]
                 return new_url
 
-        return super(ChromePreloadHSTSAdapter, self).get_redirect(url)
+        return super(PreloadHSTSAdapter, self).get_redirect(url)
+
+
+class ChromePreloadHSTSAdapter(PreloadHSTSAdapter):
+    _get_preload = _preload_including_subdomains
+
+
+class MozillaPreloadHSTSAdapter(PreloadHSTSAdapter):
+    _get_preload = _preload_remove_negative
 
 
 class HTTPSEverywhereAdapter(ChromePreloadHSTSAdapter, HTTPSEverywhereOnlyAdapter):

tests/test_adapter.py

@@ -9,6 +9,7 @@
     HTTPRedirectBlockAdapter,
     HTTPSEverywhereOnlyAdapter,
     ChromePreloadHSTSAdapter,
+    MozillaPreloadHSTSAdapter,
     HTTPSEverywhereAdapter,
     ForceHTTPSAdapter,
     PreferHTTPSAdapter,
@@ -261,6 +262,12 @@ def test_python_org_packages(self):
         self.assertEqual(r.history[1].reason, "Moved Permanently")
 
 
+class TestMozillaPreloadAdapter(TestChromePreloadAdapter):
+    cls = MozillaPreloadHSTSAdapter
+
+    test_medbank_mt = TestEverywhereOnlyAdapter.test_medbank_mt
+
+
 class TestEverywhereAdapter(TestChromePreloadAdapter):
 
     cls = HTTPSEverywhereAdapter