Merge pull request #162 from dhs-ncats/develop

Pulling develop into master

.travis.yml

@@ -3,7 +3,6 @@ language: python
 sudo: false
 
 python:
-  - '2.7'
   - '3.4'
   - '3.5'
   - '3.6'

README.md

@@ -1,7 +1,7 @@
 ## Pushing HTTPS :lock:
 
+[![Latest Version](https://img.shields.io/pypi/v/pshtt.svg)](https://pypi.python.org/pypi/pshtt/)
 [![Coverage Status](https://coveralls.io/repos/github/dhs-ncats/pshtt/badge.svg)](https://coveralls.io/github/dhs-ncats/pshtt)
-
 [![Build Status](https://travis-ci.org/dhs-ncats/pshtt.svg?branch=master)](https://travis-ci.org/dhs-ncats/pshtt)
 
 `pshtt` (_"pushed"_) is a tool to scan domains for HTTPS best practices. It saves its results to a CSV (or JSON) file.
@@ -12,6 +12,8 @@
 
 ## Getting Started
 
+`pshtt` requires **Python 3.4+**. Python 2 is not supported.
+
 `pshtt` can be installed as a module, or run directly from the repository.
 
 #### Installed as a module
@@ -124,9 +126,9 @@ The following values are returned in `results.csv`:
 * `HSTS Max Age` - A domain's HSTS max-age is its canonical endpoint's max-age.
 * `HSTS Entire Domain` - A domain has HSTS enabled for the entire domain if its **root HTTPS endpoint** (_not the canonical HTTPS endpoint_) has HSTS enabled and uses the HSTS `includeSubDomains` flag.
 * `HSTS Preload Ready` - A domain is HSTS "preload ready" if its **root HTTPS endpoint** (_not the canonical HTTPS endpoint_) has HSTS enabled, has a max-age of at least 18 weeks, and uses the `includeSubDomains` and `preload` flag.
-* `HSTS Preload Pending` - A domain is "preload pending" when it appears in the [Chrome preload pending list](https://hstspreload.org/api/v2/pending).
-* `HSTS Preloaded` - A domain is HSTS preloaded if its domain name appears in the [Chrome preload list](https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json), regardless of what header is present on any endpoint.
-* `Base Domain HSTS Preloaded` - A domain's base domain is HSTS preloaded. This is subtly different from `HSTS Entire Domain`, which inpects headers on the base domain to see if HSTS is set correctly to encompass the entire zone. This checks the preload list directly.
+* `HSTS Preload Pending` - A domain is "preload pending" when it appears in the [Chrome preload pending list](https://hstspreload.org/api/v2/pending) with the `include_subdomains` flag equal to `true`.  The intent of `pshtt` is to make sure that the user is _fully_ protected, so it only counts domains as HSTS preloaded if they are _fully_ HSTS preloaded (meaning that all subdomains are included as well).
+* `HSTS Preloaded` - A domain is HSTS preloaded if its domain name appears in the [Chrome preload list](https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json) with the `include_subdomains` flag equal to `true`, regardless of what header is present on any endpoint.  The intent of `pshtt` is to make sure that the user is _fully_ protected, so it only counts domains as HSTS preloaded if they are _fully_ HSTS preloaded (meaning that all subdomains are included as well).
+* `Base Domain HSTS Preloaded` - A domain's base domain is HSTS preloaded if its base domain appears in the [Chrome preload list](https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json) with the `include_subdomains` flag equal to `true`. This is subtly different from `HSTS Entire Domain`, which inpects headers on the base domain to see if HSTS is set correctly to encompass the entire zone.
 
 #### Scoring
 

bump_version.sh

@@ -0,0 +1,33 @@
+#/usr/bin/env bash
+
+# bump_version.sh (show|major|minor|patch|prerelease|build)
+
+VERSION_FILE=pshtt/__init__.py
+
+HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)"
+
+old_version=$(sed "s/__version__ = '\(.*\)'/\1/" $VERSION_FILE)
+
+if [[ $# -ne 1 ]]
+then
+   echo $HELP_INFORMATION
+else
+    case $1 in
+        major|minor|patch|prerelease|build)
+            new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))")
+            echo Changing version from $old_version to $new_version
+            sed -i "s/$old_version/$new_version/" $VERSION_FILE
+            ;;
+        finalize)
+            new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))")
+            echo Changing version from $old_version to $new_version
+            sed -i "s/$old_version/$new_version/" $VERSION_FILE
+            ;;
+        show)
+            echo $old_version
+            ;;
+        *)
+            echo $HELP_INFORMATION
+            ;;
+    esac
+fi

pshtt/__init__.py

@@ -1 +1 @@
-__version__ = '0.4.0-dev'
+__version__ = '0.4.1'

pshtt/pshtt.py

@@ -26,6 +26,7 @@
     from urllib2 import URLError
 
 import sslyze
+from sslyze.server_connectivity_tester import ServerConnectivityTester, ServerConnectivityError
 import sslyze.synchronous_scanner
 
 # We're going to be making requests with certificate validation disabled.
@@ -335,7 +336,6 @@ def basic_check(endpoint):
             base_immediate = parent_domain_for(subdomain_immediate)
 
             endpoint.redirect_immediately_to = immediate
-            endpoint.redirect_immediately_to_www = (re.match(r'^https?://www\.', immediate) is not None)
             endpoint.redirect_immediately_to_https = immediate.startswith("https://")
             endpoint.redirect_immediately_to_http = immediate.startswith("http://")
             endpoint.redirect_immediately_to_external = (base_original != base_immediate)
@@ -344,6 +344,13 @@ def basic_check(endpoint):
                 (subdomain_original != subdomain_immediate)
             )
 
+            # We're interested in whether an endpoint redirects to the www version
+            # of itself (not whether it redirects to www prepended to any other
+            # hostname, even within the same parent domain).
+            endpoint.redirect_immediately_to_www = (
+                subdomain_immediate == ("www.%s" % subdomain_original)
+            )
+
             if ultimate_req is not None:
                 # For ultimate destination, use the URL we arrived at,
                 # not Location header. Auto-resolves relative redirects.
@@ -442,17 +449,10 @@ def https_check(endpoint):
     # remove the https:// from prefix for sslyze
     try:
         hostname = endpoint.url[8:]
-        server_info = sslyze.server_connectivity.ServerConnectivityInfo(hostname=hostname, port=443)
-    except Exception as err:
-        endpoint.unknown_error = True
-        logging.warn("Unknown exception when checking server connectivity info with sslyze.")
-        utils.debug("{0}".format(err))
-        return
-
-    try:
-        server_info.test_connectivity_to_server()
-    except sslyze.server_connectivity.ServerConnectivityError as err:
-        logging.warn("Error in sslyze server connectivity check")
+        server_tester = ServerConnectivityTester(hostname=hostname, port=443)
+        server_info = server_tester.perform()
+    except ServerConnectivityError as err:
+        logging.warn("Error in sslyze server connectivity check when connecting to {}".format(err.server_info.hostname))
         utils.debug("{0}".format(err))
         return
     except Exception as err:
@@ -592,12 +592,6 @@ def root_down(endpoint):
             )
         )
 
-    def goes_to_www(endpoint):
-        return (
-            endpoint.redirect_immediately_to_www and
-            (not endpoint.redirect_immediately_to_external)
-        )
-
     all_roots_unused = root_unused(https) and root_unused(http)
 
     all_roots_down = root_down(https) and root_down(http)
@@ -606,8 +600,8 @@ def goes_to_www(endpoint):
         at_least_one_www_used and
         all_roots_unused and (
             all_roots_down or
-            goes_to_www(https) or
-            goes_to_www(http)
+            https.redirect_immediately_to_www or
+            http.redirect_immediately_to_www
         )
     )
 

setup.py

@@ -44,8 +44,6 @@
 
         # Specify the Python versions you support here. In particular, ensure
         # that you indicate whether you support Python 2, Python 3 or both.
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
@@ -59,7 +57,7 @@
 
     install_requires=[
         'requests>=2.18.4',
-        'sslyze>=1.1.0',
+        'sslyze>=1.4.1',
         'wget>=3.2',
         'docopt',
         'pytablereader',
@@ -70,9 +68,10 @@
 
     extras_require={
         # 'dev': ['check-manifest'],
-        'test': [
+        'dev': [
+            'pytest',
+            'semver',
             'tox',
-            'pytest'
         ],
     },
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,py34,py35,py36,flake8
+envlist = py34,py35,py36,flake8
 skip_missing_interpreters = true
 ; usedevelop = true