diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d16c01f..9bf4942 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,8 +7,9 @@ stages: variables: DOCKER_DRIVER: overlay2 DOCKER_FILES_URL: docker.greenhost.net/open/stapled - BUILD_COMPILE_CONTAINER: $DOCKER_FILES_URL/build-stretch + BUILD_COMPILE_CONTAINER: $DOCKER_FILES_URL/build-buster STRETCH_TEST_CONTAINER: $DOCKER_FILES_URL/test-stretch + BUSTER_TEST_CONTAINER: $DOCKER_FILES_URL/test-buster # GL bug causes recursive strategy to fail for LE certificates # GH recently started using LE certificates: # https://gitlab.com/gitlab-org/gitlab-runner/issues/2148 @@ -30,7 +31,7 @@ build:compile-container: - docker push $BUILD_COMPILE_CONTAINER:$CI_BUILD_REF - docker push $BUILD_COMPILE_CONTAINER:latest -build:test-container: +build:test-container-stretch: stage: build-containers image: docker:stable-git services: @@ -45,6 +46,21 @@ build:test-container: - docker push $STRETCH_TEST_CONTAINER:$CI_BUILD_REF - docker push $STRETCH_TEST_CONTAINER:latest +build:test-container-buster: + stage: build-containers + image: docker:stable-git + services: + #- docker:stable-dind + - docker:18-dind # Hotfix for https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 + variables: + GIT_STRATEGY: fetch + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN docker.greenhost.net + - docker pull $BUSTER_TEST_CONTAINER || true + - docker build --cache-from $CONTAINER_IMAGE:latest --pull --cache-from $BUSTER_TEST_CONTAINER:latest -t $BUSTER_TEST_CONTAINER:$CI_BUILD_REF -t $BUSTER_TEST_CONTAINER:latest -f ./docker/test-buster/Dockerfile ./ + - docker push $BUSTER_TEST_CONTAINER:$CI_BUILD_REF + - docker push $BUSTER_TEST_CONTAINER:latest + build:package: stage: packaging image: $BUILD_COMPILE_CONTAINER @@ -54,7 +70,7 @@ build:package: - which python3 && $(which python3) --version - openssl version # Hack to get python-daemon to install.. - - pip3 install --user docutils==0.14 + - pip3 install --user docutils==0.15.2 - pip3 install --user -r requirements.txt - make clean - make @@ -66,15 +82,27 @@ build:package: - dist/stapled*.tar.bz2 - dist/stapled*.deb -source:unit: +source:unit-stretch: stage: source-tests - image: python:3.6-stretch + image: python:3.5-stretch script: - git submodule sync --recursive - git submodule update --init --recursive - which python3 && $(which python3) --version # Hack to get python-daemon to install.. - - pip3 install --user docutils==0.14 + - pip3 install --user docutils==0.15.2 + - pip3 install -r requirements.txt + - pytest -v + +source:unit-buster: + stage: source-tests + image: python:3.7-buster + script: + - git submodule sync --recursive + - git submodule update --init --recursive + - which python3 && $(which python3) --version + # Hack to get python-daemon to install.. + - pip3 install --user docutils==0.15.2 - pip3 install -r requirements.txt - pytest -v @@ -92,9 +120,23 @@ test:stretch: dependencies: - build:package +test:buster: + stage: functional-tests + image: $BUSTER_TEST_CONTAINER + variables: + GIT_STRATEGY: none + script: + - which python3 && $(which python3) --version + - openssl version + - apt-get install -y -q ./dist/stapled_*all.deb + - /refresh_testdata.sh + - stapled -p /tmp/testdata/ --recursive --interactive --no-haproxy-sockets -vvvv --one-off + dependencies: + - build:package + source:dev-setup: stage: source-tests - image: python:3.6-stretch + image: python:3.7-buster script: - git submodule sync --recursive - git submodule update --init --recursive diff --git a/Dockerfile b/Dockerfile index 0ac8997..c0b5143 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stretch +FROM debian:buster RUN apt-get update -qq RUN apt-get upgrade -y RUN apt-get install -q -y build-essential python3-cffi libffi-dev \ @@ -7,4 +7,5 @@ RUN apt-get install -q -y build-essential python3-cffi libffi-dev \ RUN pip3 install --user pip ADD . ./ RUN pip3 install -r requirements.txt +RUN pip3 install certvalidator ocspbuilder asn1crypto oscrypto CMD echo Ready for your commands. && /bin/bash diff --git a/LICENSE b/LICENSE index a023ae9..446f3d2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ Stapled (Staple Daemon) -Copyright 2017 Greenhost BV +Copyright 2019 Greenhost BV Licensed Apache Version 2.0 diff --git a/README.rst b/README.rst index 71b1185..6b48151 100644 --- a/README.rst +++ b/README.rst @@ -52,12 +52,27 @@ From github (for developers) virtualenv -p python3 env/ # Load the virtualenv source env/bin/activate - # Install the current directory with pip. This allows you to edit the code - pip install -e . Every time you want to run ``stapled`` you will need to run -``source env/bin/activate`` to load the virtualenv first. Alternatively you can -start the daemon by running ``stapled`` +``source env/bin/activate`` to load the virtualenv first. Then you run stapled +as a module: + +.. code-block:: bash + + pythom -m stapled [arguments] + +Alternatively you can start the daemon by running ``stapled`` without even +activating the virtualenv if you install it like this: + +.. code-block:: bash + # Install dependencies.. + pip3 install asn1crypto ocspbuilder oscrypto certvalidator + # Install the current directory with pip. This install the project dir as + # a console script allowing you to run `stapled`, + pip3 install -e . + +Note that this means you have to keep track of the installed dependencies +yourself! Upgrading --------- @@ -78,7 +93,7 @@ the following: # Clone submodules too git submodule upgrade --init --recursive # Install the current directory with pip. This allows you to edit the code - pip install -e . --upgrade + pip3 install -e . --upgrade Troubleshooting =============== @@ -143,7 +158,8 @@ by running one or more of the following ``make`` commands. # All of the above make all -Everything is tested under Debian Stretch, your mileage may vary. +Everything is tested under Debian Stretch (Python 3.5 and Debian Buster +(Python 3.7), on other distros :abbr:`YMMV (Your Mileage May Vary)`. Docker build ------------ @@ -159,7 +175,7 @@ build a package. make docker-all -Remove any previous docker image and/or container named `stapled` then buil the +Remove any previous docker image and/or container named `stapled` then build the image with the same dependencies we used. Then compile the packages, then place them in the `./docker-dist` dir. diff --git a/config/stapled.conf b/config/stapled.conf index bc9c1ac..709ae50 100644 --- a/config/stapled.conf +++ b/config/stapled.conf @@ -37,9 +37,14 @@ file-extensions=crt,pem,cer ;; ignore=[no-ocsp/*.crt, /etc/ssl/private/not_a_real_crt.pem] ; ignore=ssl-cert-snakeoil.key +;; Recursively scan the paths specified by --cert-paths for certificates. +; recursive + [validity] -;; Uncomment to update every staple at startup. Leave commented to try to +;; Don't re-use existing ocsp files, refresh all staples regardless of their +;; validity. By default existing staples are recycled if they are valid for +;; longer than the minimum_validity setting. Leave commented to try to ;; re-use staples that are still valid long enough (See `minimum-validity` ;; directive) ; no-recycle @@ -58,14 +63,22 @@ daemon ;; Amount of threads to use for the renewal process. Increasing this will only ;; help if the daemon is sitting idle, e.g. waiting for OCSP responses for -;; longerperiods of time. It can help increase concurrency to a certain point -;; but ifyou really need to fetch high volumes of staples, you should start +;; longer periods of time. It can help increase concurrency to a certain point +;; but if you really need to fetch high volumes of staples, you should start ;; more processes. renewal-threads=5 ;; How long the scheduler should sleep between each scheduling attempt. refresh-interval=30 +;; Run only a one-off staple renewal and quit stapled when done. Note that this +;; will still spawn the same amount of threads as a normal process would for +;; performance reasons as well as consistency between one-off and normal runs. +;; This setting overrides the --refresh-interval setting because a refresh is +;; not scheduled during one-off runs. The --daemon and --no-daemon/--interactive +;; arguments are also ignored. +; one-off + [logging] ;; Log to syslog, you can not set a `logdir` to only log to syslog, or ;; enable both at the same time. Uncomment to enable. @@ -86,6 +99,15 @@ logdir=/var/log/stapled/ ;; https://cbonte.github.io/haproxy-dconv/1.7/management.html#9.3-set%20ssl%20ocsp-response haproxy-sockets=[/var/run/haproxy/admin.sock] +;; By default stapled will try to connect to the default socket path, which can +;; be changed or set to an empty list by the --haproxy-sockets argument. +;; The --no-haproxy-sockets argument explicitly disables the haproxy socket +;; connection and overrides the --haproxy-sockets argument's paths if set. +;; Note that this does NOT disable the --haproxy-config argument, i.e.: if a +;; haproxy config is set, it will be parsed for certificate paths, without +;; matching sockets. +; no-haproxy-sockets + ;; Use HAProxy config files as the source of cert-paths and socket mappings. ;; Setting this will merge your `cert-paths` with paths found in the specified ;; HAProxy config files. Sockets defined in `haproxy-sockets` will also be @@ -103,8 +125,3 @@ haproxy-sockets=[/var/run/haproxy/admin.sock] ;; and/or `syslog` to prevent output on stdout while logging the set verbosity ;; level to a file or syslog. Uncomment to enable ; quiet - -;; Don't re-use existing ocsp files, refresh all staples regardless of their -;; validity. By default existing staples are recycled if they are valid for -;; longer than the minimum_validity setting. -; no-recycle diff --git a/debian/changelog b/debian/changelog index ce6fa89..0f38c42 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +stapled (1.4) buster; urgency=high + + * Tested against buster + * Tested against Python 3.7 + * Still tested against Python 3.5. + * Still tested against stretch + * Dependencies updated to latest versions + * Improved exception handling of haproxy adder module + + -- Chris Wed, 11 Sep 2019 14:44:23 +0200 + stapled (1.3) stretch; urgency=low * Improved excpetion handling to limit flooding of logs diff --git a/docker/build-buster/Dockerfile b/docker/build-buster/Dockerfile new file mode 100644 index 0000000..974bf8a --- /dev/null +++ b/docker/build-buster/Dockerfile @@ -0,0 +1,10 @@ +FROM debian:buster +RUN apt-get update -qq +RUN apt-get upgrade -y +RUN apt-get install -q -y build-essential python3-cffi libffi-dev \ + python-all python3-all python3-dev python3-setuptools python3-pip \ + rpm tar gzip bzip2 git debhelper ca-certificates +ADD ./requirements.txt ./requirements.txt +RUN pip3 install --user pip +RUN pip3 install --user docutils==0.15.2 +RUN pip3 install --user -r requirements.txt diff --git a/docker/build-stretch/Dockerfile b/docker/build-stretch/Dockerfile index 88887c7..a734195 100644 --- a/docker/build-stretch/Dockerfile +++ b/docker/build-stretch/Dockerfile @@ -6,5 +6,5 @@ RUN apt-get install -q -y build-essential python3-cffi libffi-dev \ rpm tar gzip bzip2 git debhelper ca-certificates ADD ./requirements.txt ./requirements.txt RUN pip3 install --user pip -RUN pip3 install --user docutils==0.14 +RUN pip3 install --user docutils==0.15.2 RUN pip3 install --user -r requirements.txt diff --git a/docker/test-buster/Dockerfile b/docker/test-buster/Dockerfile new file mode 100644 index 0000000..ae572d8 --- /dev/null +++ b/docker/test-buster/Dockerfile @@ -0,0 +1,6 @@ +FROM debian:buster +RUN apt-get update -qq +RUN apt-get upgrade -y +RUN apt-get install -y openssl ca-certificates python3-cffi \ + python3-configargparse python3-daemon +COPY ./refresh_testdata.sh ./refresh_testdata.sh diff --git a/docs/source/using.rst b/docs/source/using.rst index c32cb87..8e679cb 100644 --- a/docs/source/using.rst +++ b/docs/source/using.rst @@ -18,15 +18,15 @@ OCSP server locally, quite a setup. So until now we didn't do so yet. Note that if you have experience with this kind of setup and you want to help this project move forward, you are welcome to help. -Obviously we do test stapled, admittedly a little bit primitively. You can find a -script in ``scripts/`` called ``refresh_testdata.sh``. It will delete any +Obviously we do test stapled, admittedly a little bit primitively. You can find +a script in ``scripts/`` called ``refresh_testdata.sh``. It will delete any directory named ``testdata`` in the root of the project and create a fresh one. Then it will download 3 certificate chains from live servers. These will be placed in subdirectories with the same name as the domain name. -Next you can run ``python stapled -vvvv -d testdata/*`` to get output printed to -your terminal. The ``testdata/[domain].[tld]`` directories will be populated -with ``[domain].[tld].ocsp`` files. +Next you can run ``python -m stapled -vvvv -d testdata/*`` to get output +printed to your terminal. The ``testdata/[domain].[tld]`` directories will be +populated with ``[domain].[tld].ocsp`` files. Caveats ======= diff --git a/refresh_testdata.sh b/refresh_testdata.sh index b111cdf..46736ef 100755 --- a/refresh_testdata.sh +++ b/refresh_testdata.sh @@ -20,4 +20,4 @@ for DOMAIN in $DOMAINS; do sed -n "/-----BEGIN/,/-----END/ w $TESTDATA_PATH$DOMAIN/chain.pem" done echo "[+] Start stapled with the desired verbosity and test directory e.g.:" -echo "python -m stapled -vvvv -d /tmp/testdata/ --recursive" +echo "stapled -p /tmp/testdata/ --recursive --interactive --no-haproxy-sockets -vvvv --one-off" diff --git a/requirements.txt b/requirements.txt index 542e43e..0383dc0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,35 +1,35 @@ # Project deps ConfigArgParse==0.14.0 future==0.17.1 -cffi==1.12.2 +cffi==1.12.3 ## python-daemon deps lockfile==0.12.2 -docutils==0.14 +docutils==0.15.2 python-daemon==2.2.3 # Packaging and distribution -twine==1.13.0 -tqdm==4.31.1 +twine==1.14.0 +tqdm==4.35.0 idna==2.8 -certifi==2018.11.29 +certifi==2019.6.16 chardet==3.0.4 pkginfo==1.5.0.1 -requests==2.21.0 +requests==2.22.0 requests-toolbelt==0.9.1 -urllib3==1.24.1 +urllib3==1.25.3 stdeb==0.8.5 -wheel==0.33.1 +wheel==0.33.6 # Docs -Sphinx==1.8.4 +Sphinx==2.2.0 sphinx-argparse==0.2.5 sphinx-rtd-theme==0.4.3 # Linting -pydocstyle==3.0.0 +pydocstyle==4.0.1 pycodestyle==2.5.0 -pylint==2.3.0 +pylint==2.3.1 # Testing -pytest==4.3.0 +pytest==5.1.2 diff --git a/setup.py b/setup.py index f5ade4c..2dd8133 100644 --- a/setup.py +++ b/setup.py @@ -39,21 +39,21 @@ def all_packages(): url='https://github.com/greenhost/stapled', packages=all_packages(), package_dir=find_lib_path_dict(), - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, <4', + python_requires='!=3.0.*, !=3.1.*, !=3.2.*, <4', install_requires=[ - 'python-daemon>=2.2.0', - 'configargparse>=0.13.0', + 'python-daemon>=2.2.3', + 'configargparse>=0.14.0', # Required by deps in `stapled/libs` 'future>=0.17.1', - 'cffi>=1.11.5', + 'cffi>=1.12.3', # Required for python-daemon (package dep not specified) - 'docutils>=0.14', + 'docutils>=0.15.2', ], extras_require={ 'docs': [ - 'Sphinx>=1.8.2', + 'Sphinx>=2.2.0', 'sphinx-argparse>=0.2.5', - 'sphinx_rtd_theme>=0.4.2', + 'sphinx_rtd_theme>=0.4.3', ] }, license='Apache Version 2.0', @@ -64,8 +64,6 @@ def all_packages(): 'License :: OSI Approved :: Apache Version 2.0', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', diff --git a/stapled/__main__.py b/stapled/__main__.py index c276df4..eff8672 100755 --- a/stapled/__main__.py +++ b/stapled/__main__.py @@ -217,8 +217,7 @@ def get_cli_arg_parser(): ) parser.add( '--no-haproxy-sockets', - action='store_false', - dest='haproxy_sockets', + action='store_true', help=( "Disable HAProxy sockets, overrides ``--haproxy-sockets`` if " "specified in the config file." @@ -348,8 +347,8 @@ def init(): # config files, de-duplicated. cert_paths = haproxy_socket_mapping.keys() - # Determine if we need to start a stapleadder thread. - if not any(haproxy_socket_mapping.values()): + # Determine if we need to start a staple adder thread. + if args.no_haproxy_sockets or not any(haproxy_socket_mapping.values()): haproxy_socket_mapping = None daemon_kwargs = dict( @@ -499,15 +498,19 @@ def __get_haproxy_socket_mapping(args): # files in the sockets dictionary. for i, paths in enumerate(conf_cert_paths): for path in paths: - if path in mapping: + # When certificate path is already in the mapping add the socket + # file to the mapping if haproxy_sockets is not disabled. + if args.no_haproxy_sockets: + # haproxy_sockets are disabled, just ensure the path is in the + # mapping without sockets. + mapping[path] = [] + else: + # Make sure paths are mapped to all unique sockets. mapping[path] = unique( - mapping[path] + conf_haproxy_sockets[i], + mapping.get(path, []) + conf_haproxy_sockets[i], preserve_order=False ) - else: - mapping[path] = conf_haproxy_sockets[i] - - logger.debug("Paths to socket mapping: %s", str(mapping)) + logger.debug("Paths to socket mappings: %s", str(mapping)) return mapping @@ -537,47 +540,6 @@ def __get_validated_args(): return args -def __get_haproxy_socket_mapping(args): - """ - Get mapping of configured sockets and certificate directories. - - From: haproxy config, stapled config and command line arguments. - - :param Namespace args: Argparser argument list. - :return dict Of cert-paths and sockets for inform of changes. - """ - # Parse the cert_paths argument - arg_cert_paths = __get_arg_cert_paths(args) - # Parse haproxy_sockets argument. - arg_haproxy_sockets = __get_arg_haproxy_sockets(args) - # Make a mapping from certificate paths to sockets in a dict. - mapping = dict(zip(arg_cert_paths, arg_haproxy_sockets)) - - # Parse HAProxy config files. - try: - conf_cert_paths, conf_haproxy_sockets = parse_haproxy_config( - args.haproxy_config - ) - except (OSError) as exc: - logger.critical(exc) - exit(1) - - # Combine the socket and certificate paths of the arguments and config - # files in the sockets dictionary. - for i, paths in enumerate(conf_cert_paths): - for path in paths: - if path in mapping: - mapping[path] = unique( - mapping[path] + conf_haproxy_sockets[i], - preserve_order=False - ) - else: - mapping[path] = conf_haproxy_sockets[i] - - logger.debug("Paths to socket mapping: %s", str(mapping)) - return mapping - - if __name__ == '__main__': try: init() diff --git a/stapled/core/daemon.py b/stapled/core/daemon.py index 206fba7..6746129 100644 --- a/stapled/core/daemon.py +++ b/stapled/core/daemon.py @@ -285,7 +285,7 @@ def handle_one_off(self): ] # Check if enabled before adding stapleadder queue. if self.staple_adder: - stop_threads.append(('adder', [self.staple_adder])) + stop_threads.append(('proxy-add', [self.staple_adder])) def one_off_generator(): """Make generator to iteratively end the stapled process.""" @@ -297,7 +297,8 @@ def one_off_generator(): for queue, threads in stop_threads: # Queue must exist, if it doesn't it wasn't yet created. - if not self.scheduler.queue_exists(queue): + # Wait for it to be created. + while not self.scheduler.queue_exists(queue): yield False # Wait until queues are empty.. diff --git a/stapled/lib/asn1crypto b/stapled/lib/asn1crypto index 7f587d5..6060d29 160000 --- a/stapled/lib/asn1crypto +++ b/stapled/lib/asn1crypto @@ -1 +1 @@ -Subproject commit 7f587d56f2b5c4815bfed835cfc10abd9c49cdb2 +Subproject commit 6060d29ac5ef67abc9f4a8347813c0ca87f328bd diff --git a/stapled/lib/certvalidator b/stapled/lib/certvalidator index 4383a4b..b69d3b7 160000 --- a/stapled/lib/certvalidator +++ b/stapled/lib/certvalidator @@ -1 +1 @@ -Subproject commit 4383a4bfd5e769679bc4eedd1e4d334eb0c7d85a +Subproject commit b69d3b745b5af9e5ccd4b9781407ab7e82076d6b diff --git a/stapled/lib/ocspbuilder b/stapled/lib/ocspbuilder index 2db8721..64e7a1f 160000 --- a/stapled/lib/ocspbuilder +++ b/stapled/lib/ocspbuilder @@ -1 +1 @@ -Subproject commit 2db8721710d193107c7787f8c094440443b7fbfa +Subproject commit 64e7a1f02615e8ec1e7826a097883c480ff73a1c diff --git a/stapled/lib/oscrypto b/stapled/lib/oscrypto index ef86af4..5857594 160000 --- a/stapled/lib/oscrypto +++ b/stapled/lib/oscrypto @@ -1 +1 @@ -Subproject commit ef86af48af8e0362b163c9807adf389460debac1 +Subproject commit 585759457f60a0979274862db2e8dc6211b28a24 diff --git a/stapled/version.py b/stapled/version.py index a83e677..75be500 100644 --- a/stapled/version.py +++ b/stapled/version.py @@ -1,3 +1,3 @@ -__version__ = '1.3' +__version__ = '1.4' __app_name__ = 'stapled' __debian_version__ = 'stretch' diff --git a/version b/version index c667175..318c1b8 100755 --- a/version +++ b/version @@ -31,7 +31,7 @@ class GitVersion(object): CLEAN_COMMIT = 'nothing to commit, working tree clean' DEFAULT_VERSION = [0, 1] CHANGELOG_FILE = 'debian/changelog' - DEFAULT_TARGET_DEB_OS = "stretch" + DEFAULT_TARGET_DEB_OS = "buster" def __init__(self, **kwargs): """