Update various stale documentation files etc. upgrade the docker file…

… to buster, update dependecies that are packages along with stapled.

.gitlab-ci.yml

@@ -7,8 +7,9 @@ stages:
 variables:
   DOCKER_DRIVER: overlay2
   DOCKER_FILES_URL: docker.greenhost.net/open/stapled
-  BUILD_COMPILE_CONTAINER: $DOCKER_FILES_URL/build-stretch
+  BUILD_COMPILE_CONTAINER: $DOCKER_FILES_URL/build-buster
   STRETCH_TEST_CONTAINER: $DOCKER_FILES_URL/test-stretch
+  BUSTER_TEST_CONTAINER: $DOCKER_FILES_URL/test-buster
   # GL bug causes recursive strategy to fail for LE certificates
   # GH recently started using LE certificates:
   # https://gitlab.com/gitlab-org/gitlab-runner/issues/2148
@@ -30,7 +31,7 @@ build:compile-container:
   - docker push $BUILD_COMPILE_CONTAINER:$CI_BUILD_REF
   - docker push $BUILD_COMPILE_CONTAINER:latest
 
-build:test-container:
+build:test-container-stretch:
   stage: build-containers
   image: docker:stable-git
   services:
@@ -45,6 +46,21 @@ build:test-container:
   - docker push $STRETCH_TEST_CONTAINER:$CI_BUILD_REF
   - docker push $STRETCH_TEST_CONTAINER:latest
 
+build:test-container-buster:
+  stage: build-containers
+  image: docker:stable-git
+  services:
+    #- docker:stable-dind
+    - docker:18-dind # Hotfix for https://gitlab.com/gitlab-org/gitlab-runner/issues/4501
+  variables:
+   GIT_STRATEGY: fetch
+  script:
+  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN docker.greenhost.net
+  - docker pull $BUSTER_TEST_CONTAINER || true
+  - docker build --cache-from $CONTAINER_IMAGE:latest --pull --cache-from $BUSTER_TEST_CONTAINER:latest -t $BUSTER_TEST_CONTAINER:$CI_BUILD_REF -t $BUSTER_TEST_CONTAINER:latest -f ./docker/test-buster/Dockerfile ./
+  - docker push $BUSTER_TEST_CONTAINER:$CI_BUILD_REF
+  - docker push $BUSTER_TEST_CONTAINER:latest
+
 build:package:
   stage: packaging
   image: $BUILD_COMPILE_CONTAINER
@@ -54,7 +70,7 @@ build:package:
   - which python3 && $(which python3) --version
   - openssl version
   # Hack to get python-daemon to install..
-  - pip3 install --user docutils==0.14
+  - pip3 install --user docutils==0.15.2
   - pip3 install --user -r requirements.txt
   - make clean
   - make
@@ -66,15 +82,27 @@ build:package:
     - dist/stapled*.tar.bz2
     - dist/stapled*.deb
 
-source:unit:
+source:unit-stretch:
   stage: source-tests
-  image: python:3.6-stretch
+  image: python:3.5-stretch
   script:
   - git submodule sync --recursive
   - git submodule update --init --recursive
   - which python3 && $(which python3) --version
   # Hack to get python-daemon to install..
-  - pip3 install --user docutils==0.14
+  - pip3 install --user docutils==0.15.2
+  - pip3 install -r requirements.txt
+  - pytest -v
+
+source:unit-buster:
+  stage: source-tests
+  image: python:3.7-buster
+  script:
+  - git submodule sync --recursive
+  - git submodule update --init --recursive
+  - which python3 && $(which python3) --version
+  # Hack to get python-daemon to install..
+  - pip3 install --user docutils==0.15.2
   - pip3 install -r requirements.txt
   - pytest -v
 
@@ -92,9 +120,23 @@ test:stretch:
   dependencies:
   - build:package
 
+test:buster:
+  stage: functional-tests
+  image: $BUSTER_TEST_CONTAINER
+  variables:
+    GIT_STRATEGY: none
+  script:
+  - which python3 && $(which python3) --version
+  - openssl version
+  - apt-get install -y -q ./dist/stapled_*all.deb
+  - /refresh_testdata.sh
+  - stapled -p /tmp/testdata/ --recursive --interactive --no-haproxy-sockets -vvvv --one-off
+  dependencies:
+  - build:package
+
 source:dev-setup:
   stage: source-tests
-  image: python:3.6-stretch
+  image: python:3.7-buster
   script:
   - git submodule sync --recursive
   - git submodule update --init --recursive

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:stretch
+FROM debian:buster
 RUN apt-get update -qq
 RUN apt-get upgrade -y
 RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
@@ -7,4 +7,5 @@ RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
 RUN pip3 install --user pip
 ADD . ./
 RUN pip3 install -r requirements.txt
+RUN pip3 install certvalidator ocspbuilder asn1crypto oscrypto
 CMD echo Ready for your commands. && /bin/bash

LICENSE

@@ -1,5 +1,5 @@
 Stapled (Staple Daemon)
-Copyright 2017 Greenhost BV
+Copyright 2019 Greenhost BV
 
 Licensed Apache Version 2.0
 

README.rst

@@ -52,12 +52,27 @@ From github (for developers)
     virtualenv -p python3 env/
     # Load the virtualenv
     source env/bin/activate
-    # Install the current directory with pip. This allows you to edit the code
-    pip install -e .
 
 Every time you want to run ``stapled`` you will need to run
-``source env/bin/activate`` to load the virtualenv first. Alternatively you can
-start the daemon by running ``stapled``
+``source env/bin/activate`` to load the virtualenv first. Then you run stapled
+as a module:
+
+.. code-block:: bash
+
+    pythom -m stapled [arguments]
+
+Alternatively you can start the daemon by running ``stapled`` without even
+activating the virtualenv if you install it like this:
+
+.. code-block:: bash
+    # Install dependencies..
+    pip3 install asn1crypto ocspbuilder oscrypto certvalidator
+    # Install the current directory with pip. This install the project dir as
+    # a console script allowing you to run `stapled`,
+    pip3 install -e .
+
+Note that this means you have to keep track of the installed dependencies
+yourself!
 
 Upgrading
 ---------
@@ -78,7 +93,7 @@ the following:
     # Clone submodules too
     git submodule upgrade --init --recursive
     # Install the current directory with pip. This allows you to edit the code
-    pip install -e . --upgrade
+    pip3 install -e . --upgrade
 
 Troubleshooting
 ===============
@@ -143,7 +158,8 @@ by running one or more of the following ``make`` commands.
     # All of the above
     make all
 
-Everything is tested under Debian Stretch, your mileage may vary.
+Everything is tested under Debian Stretch (Python 3.5 and Debian Buster
+(Python 3.7), on other distros :abbr:`YMMV (Your Mileage May Vary)`.
 
 Docker build
 ------------
@@ -159,7 +175,7 @@ build a package.
 
     make docker-all
 
-Remove any previous docker image and/or container named `stapled` then buil the
+Remove any previous docker image and/or container named `stapled` then build the
 image with the same dependencies we used. Then compile the packages, then
 place them in the `./docker-dist` dir.
 

config/stapled.conf

@@ -37,9 +37,14 @@ file-extensions=crt,pem,cer
 ;; ignore=[no-ocsp/*.crt, /etc/ssl/private/not_a_real_crt.pem]
 ; ignore=ssl-cert-snakeoil.key
 
+;; Recursively scan the paths specified by --cert-paths for certificates.
+; recursive
+
 [validity]
 
-;; Uncomment to update every staple at startup. Leave commented to try to
+;; Don't re-use existing ocsp files, refresh all staples regardless of their
+;; validity. By default existing staples are recycled if they are valid for
+;; longer than the minimum_validity setting. Leave commented to try to
 ;; re-use staples that are still valid long enough (See `minimum-validity`
 ;; directive)
 ; no-recycle
@@ -58,14 +63,22 @@ daemon
 
 ;; Amount of threads to use for the renewal process. Increasing this will only
 ;; help if the daemon is sitting idle, e.g. waiting for OCSP responses for
-;; longerperiods of time. It can help increase concurrency to a certain point
-;; but ifyou really need to fetch high volumes of staples, you should start
+;; longer periods of time. It can help increase concurrency to a certain point
+;; but if you really need to fetch high volumes of staples, you should start
 ;; more processes.
 renewal-threads=5
 
 ;; How long the scheduler should sleep between each scheduling attempt.
 refresh-interval=30
 
+;; Run only a one-off staple renewal and quit stapled when done. Note that this
+;; will still spawn the same amount of threads as a normal process would for
+;; performance reasons as well as consistency between one-off and normal runs.
+;; This setting overrides the --refresh-interval setting because a refresh is
+;; not scheduled during one-off runs. The --daemon and --no-daemon/--interactive
+;; arguments are also ignored.
+; one-off
+
 [logging]
 ;; Log to syslog, you can not set a `logdir` to only log to syslog, or
 ;; enable both at the same time. Uncomment to enable.
@@ -86,6 +99,15 @@ logdir=/var/log/stapled/
 ;; https://cbonte.github.io/haproxy-dconv/1.7/management.html#9.3-set%20ssl%20ocsp-response
 haproxy-sockets=[/var/run/haproxy/admin.sock]
 
+;; By default stapled will try to connect to the default socket path, which can
+;; be changed or set to an empty list by the --haproxy-sockets argument.
+;; The --no-haproxy-sockets argument explicitly disables the haproxy socket
+;; connection and overrides the --haproxy-sockets argument's paths if set.
+;; Note that this does NOT disable the --haproxy-config argument, i.e.: if a
+;; haproxy config is set, it will be parsed for certificate paths, without
+;; matching sockets.
+; no-haproxy-sockets
+
 ;; Use HAProxy config files as the source of cert-paths and socket mappings.
 ;; Setting this will merge your `cert-paths` with paths found in the specified
 ;; HAProxy config files. Sockets defined in `haproxy-sockets` will also be
@@ -103,8 +125,3 @@ haproxy-sockets=[/var/run/haproxy/admin.sock]
 ;; and/or `syslog` to prevent output on stdout while logging the set verbosity
 ;; level to a file or syslog. Uncomment to enable
 ; quiet
-
-;; Don't re-use existing ocsp files, refresh all staples regardless of their
-;; validity. By default existing staples are recycled if they are valid for
-;; longer than the minimum_validity setting.
-; no-recycle

debian/changelog

@@ -1,3 +1,14 @@
+stapled (1.4) buster; urgency=high
+
+ * Tested against buster
+ * Tested against Python 3.7
+ * Still tested against Python 3.5.
+ * Still tested against stretch
+ * Dependencies updated to latest versions
+ * Improved exception handling of haproxy adder module
+
+ -- Chris <[email protected]> Wed, 11 Sep 2019 14:44:23 +0200
+
 stapled (1.3) stretch; urgency=low
 
  * Improved excpetion handling to limit flooding of logs

docker/build-buster/Dockerfile

@@ -0,0 +1,10 @@
+FROM debian:buster
+RUN apt-get update -qq
+RUN apt-get upgrade -y
+RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
+    python-all python3-all python3-dev python3-setuptools python3-pip \
+    rpm tar gzip bzip2 git debhelper ca-certificates
+ADD ./requirements.txt ./requirements.txt
+RUN pip3 install --user pip
+RUN pip3 install --user docutils==0.15.2
+RUN pip3 install --user -r requirements.txt

docker/build-stretch/Dockerfile

@@ -6,5 +6,5 @@ RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
     rpm tar gzip bzip2 git debhelper ca-certificates
 ADD ./requirements.txt ./requirements.txt
 RUN pip3 install --user pip
-RUN pip3 install --user docutils==0.14
+RUN pip3 install --user docutils==0.15.2
 RUN pip3 install --user -r requirements.txt

docker/test-buster/Dockerfile

@@ -0,0 +1,6 @@
+FROM debian:buster
+RUN apt-get update -qq
+RUN apt-get upgrade -y
+RUN apt-get install -y openssl ca-certificates python3-cffi \
+    python3-configargparse python3-daemon
+COPY ./refresh_testdata.sh ./refresh_testdata.sh

docs/source/using.rst

@@ -18,15 +18,15 @@ OCSP server locally, quite a setup. So until now we didn't do so yet. Note that
 if you have experience with this kind of setup and you want to help this project
 move forward, you are welcome to help.
 
-Obviously we do test stapled, admittedly a little bit primitively. You can find a
-script in ``scripts/`` called ``refresh_testdata.sh``. It will delete any
+Obviously we do test stapled, admittedly a little bit primitively. You can find
+a script in ``scripts/`` called ``refresh_testdata.sh``. It will delete any
 directory named ``testdata`` in the root of the project and create a fresh one.
 Then it will download 3 certificate chains from live servers. These will be
 placed in subdirectories with the same name as the domain name.
 
-Next you can run ``python stapled -vvvv -d testdata/*`` to get output printed to
-your terminal. The ``testdata/[domain].[tld]`` directories will be populated
-with ``[domain].[tld].ocsp`` files.
+Next you can run ``python -m stapled -vvvv -d testdata/*`` to get output
+printed to your terminal. The ``testdata/[domain].[tld]`` directories will be
+populated with ``[domain].[tld].ocsp`` files.
 
 Caveats
 =======

refresh_testdata.sh

@@ -20,4 +20,4 @@ for DOMAIN in $DOMAINS; do
     sed  -n "/-----BEGIN/,/-----END/ w $TESTDATA_PATH$DOMAIN/chain.pem"
 done
 echo "[+] Start stapled with the desired verbosity and test directory e.g.:"
-echo "python -m stapled -vvvv -d /tmp/testdata/ --recursive"
+echo "stapled -p /tmp/testdata/ --recursive --interactive --no-haproxy-sockets -vvvv --one-off"

requirements.txt

@@ -1,35 +1,35 @@
 # Project deps
 ConfigArgParse==0.14.0
 future==0.17.1
-cffi==1.12.2
+cffi==1.12.3
 
 ## python-daemon deps
 lockfile==0.12.2
-docutils==0.14
+docutils==0.15.2
 python-daemon==2.2.3
 
 # Packaging and distribution
-twine==1.13.0
-tqdm==4.31.1
+twine==1.14.0
+tqdm==4.35.0
 idna==2.8
-certifi==2018.11.29
+certifi==2019.6.16
 chardet==3.0.4
 pkginfo==1.5.0.1
-requests==2.21.0
+requests==2.22.0
 requests-toolbelt==0.9.1
-urllib3==1.24.1
+urllib3==1.25.3
 stdeb==0.8.5
-wheel==0.33.1
+wheel==0.33.6
 
 # Docs
-Sphinx==1.8.4
+Sphinx==2.2.0
 sphinx-argparse==0.2.5
 sphinx-rtd-theme==0.4.3
 
 # Linting
-pydocstyle==3.0.0
+pydocstyle==4.0.1
 pycodestyle==2.5.0
-pylint==2.3.0
+pylint==2.3.1
 
 # Testing
-pytest==4.3.0
+pytest==5.1.2