Merge branch '66-make-new-release-for-ops' into 'master'

Resolve "Make new release for OPS"

Closes #66

See merge request !50

.gitlab-ci.yml

@@ -7,8 +7,9 @@ stages:
 variables:
   DOCKER_DRIVER: overlay2
   DOCKER_FILES_URL: docker.greenhost.net/open/stapled
-  BUILD_COMPILE_CONTAINER: $DOCKER_FILES_URL/build-stretch
+  BUILD_COMPILE_CONTAINER: $DOCKER_FILES_URL/build-buster
   STRETCH_TEST_CONTAINER: $DOCKER_FILES_URL/test-stretch
+  BUSTER_TEST_CONTAINER: $DOCKER_FILES_URL/test-buster
   # GL bug causes recursive strategy to fail for LE certificates
   # GH recently started using LE certificates:
   # https://gitlab.com/gitlab-org/gitlab-runner/issues/2148
@@ -30,7 +31,7 @@ build:compile-container:
   - docker push $BUILD_COMPILE_CONTAINER:$CI_BUILD_REF
   - docker push $BUILD_COMPILE_CONTAINER:latest
 
-build:test-container:
+build:test-container-stretch:
   stage: build-containers
   image: docker:stable-git
   services:
@@ -45,6 +46,21 @@ build:test-container:
   - docker push $STRETCH_TEST_CONTAINER:$CI_BUILD_REF
   - docker push $STRETCH_TEST_CONTAINER:latest
 
+build:test-container-buster:
+  stage: build-containers
+  image: docker:stable-git
+  services:
+    #- docker:stable-dind
+    - docker:18-dind # Hotfix for https://gitlab.com/gitlab-org/gitlab-runner/issues/4501
+  variables:
+   GIT_STRATEGY: fetch
+  script:
+  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN docker.greenhost.net
+  - docker pull $BUSTER_TEST_CONTAINER || true
+  - docker build --cache-from $CONTAINER_IMAGE:latest --pull --cache-from $BUSTER_TEST_CONTAINER:latest -t $BUSTER_TEST_CONTAINER:$CI_BUILD_REF -t $BUSTER_TEST_CONTAINER:latest -f ./docker/test-buster/Dockerfile ./
+  - docker push $BUSTER_TEST_CONTAINER:$CI_BUILD_REF
+  - docker push $BUSTER_TEST_CONTAINER:latest
+
 build:package:
   stage: packaging
   image: $BUILD_COMPILE_CONTAINER
@@ -54,7 +70,7 @@ build:package:
   - which python3 && $(which python3) --version
   - openssl version
   # Hack to get python-daemon to install..
-  - pip3 install --user docutils==0.14
+  - pip3 install --user docutils==0.15.2
   - pip3 install --user -r requirements.txt
   - make clean
   - make
@@ -66,15 +82,27 @@ build:package:
     - dist/stapled*.tar.bz2
     - dist/stapled*.deb
 
-source:unit:
+source:unit-stretch:
   stage: source-tests
-  image: python:3.6-stretch
+  image: python:3.5-stretch
   script:
   - git submodule sync --recursive
   - git submodule update --init --recursive
   - which python3 && $(which python3) --version
   # Hack to get python-daemon to install..
-  - pip3 install --user docutils==0.14
+  - pip3 install --user docutils==0.15.2
+  - pip3 install -r requirements.txt
+  - pytest -v
+
+source:unit-buster:
+  stage: source-tests
+  image: python:3.7-buster
+  script:
+  - git submodule sync --recursive
+  - git submodule update --init --recursive
+  - which python3 && $(which python3) --version
+  # Hack to get python-daemon to install..
+  - pip3 install --user docutils==0.15.2
   - pip3 install -r requirements.txt
   - pytest -v
 
@@ -92,9 +120,23 @@ test:stretch:
   dependencies:
   - build:package
 
+test:buster:
+  stage: functional-tests
+  image: $BUSTER_TEST_CONTAINER
+  variables:
+    GIT_STRATEGY: none
+  script:
+  - which python3 && $(which python3) --version
+  - openssl version
+  - apt-get install -y -q ./dist/stapled_*all.deb
+  - /refresh_testdata.sh
+  - stapled -p /tmp/testdata/ --recursive --interactive --no-haproxy-sockets -vvvv --one-off
+  dependencies:
+  - build:package
+
 source:dev-setup:
   stage: source-tests
-  image: python:3.6-stretch
+  image: python:3.7-buster
   script:
   - git submodule sync --recursive
   - git submodule update --init --recursive

Dockerfile

-FROM debian:stretch
+FROM debian:buster
 RUN apt-get update -qq
 RUN apt-get upgrade -y
 RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
@@ -7,4 +7,5 @@ RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
 RUN pip3 install --user pip
 ADD . ./
 RUN pip3 install -r requirements.txt
+RUN pip3 install certvalidator ocspbuilder asn1crypto oscrypto
 CMD echo Ready for your commands. && /bin/bash

LICENSE

 Stapled (Staple Daemon)
-Copyright 2017 Greenhost BV
+Copyright 2019 Greenhost BV
 
 Licensed Apache Version 2.0
 

README.rst

@@ -52,12 +52,27 @@ From github (for developers)
     virtualenv -p python3 env/
     # Load the virtualenv
     source env/bin/activate
-    # Install the current directory with pip. This allows you to edit the code
-    pip install -e .
 
 Every time you want to run ``stapled`` you will need to run
-``source env/bin/activate`` to load the virtualenv first. Alternatively you can
-start the daemon by running ``stapled``
+``source env/bin/activate`` to load the virtualenv first. Then you run stapled
+as a module:
+
+.. code-block:: bash
+
+    pythom -m stapled [arguments]
+
+Alternatively you can start the daemon by running ``stapled`` without even
+activating the virtualenv if you install it like this:
+
+.. code-block:: bash
+    # Install dependencies..
+    pip3 install asn1crypto ocspbuilder oscrypto certvalidator
+    # Install the current directory with pip. This install the project dir as
+    # a console script allowing you to run `stapled`,
+    pip3 install -e .
+
+Note that this means you have to keep track of the installed dependencies
+yourself!
 
 Upgrading
 ---------
@@ -78,7 +93,7 @@ the following:
     # Clone submodules too
     git submodule upgrade --init --recursive
     # Install the current directory with pip. This allows you to edit the code
-    pip install -e . --upgrade
+    pip3 install -e . --upgrade
 
 Troubleshooting
 ===============
@@ -143,7 +158,8 @@ by running one or more of the following ``make`` commands.
     # All of the above
     make all
 
-Everything is tested under Debian Stretch, your mileage may vary.
+Everything is tested under Debian Stretch (Python 3.5 and Debian Buster
+(Python 3.7), on other distros :abbr:`YMMV (Your Mileage May Vary)`.
 
 Docker build
 ------------
@@ -159,7 +175,7 @@ build a package.
 
     make docker-all
 
-Remove any previous docker image and/or container named `stapled` then buil the
+Remove any previous docker image and/or container named `stapled` then build the
 image with the same dependencies we used. Then compile the packages, then
 place them in the `./docker-dist` dir.
 

config/stapled.conf

@@ -37,9 +37,14 @@ file-extensions=crt,pem,cer
 ;; ignore=[no-ocsp/*.crt, /etc/ssl/private/not_a_real_crt.pem]
 ; ignore=ssl-cert-snakeoil.key
 
+;; Recursively scan the paths specified by --cert-paths for certificates.
+; recursive
+
 [validity]
 
-;; Uncomment to update every staple at startup. Leave commented to try to
+;; Don't re-use existing ocsp files, refresh all staples regardless of their
+;; validity. By default existing staples are recycled if they are valid for
+;; longer than the minimum_validity setting. Leave commented to try to
 ;; re-use staples that are still valid long enough (See `minimum-validity`
 ;; directive)
 ; no-recycle
@@ -58,14 +63,22 @@ daemon
 
 ;; Amount of threads to use for the renewal process. Increasing this will only
 ;; help if the daemon is sitting idle, e.g. waiting for OCSP responses for
-;; longerperiods of time. It can help increase concurrency to a certain point
-;; but ifyou really need to fetch high volumes of staples, you should start
+;; longer periods of time. It can help increase concurrency to a certain point
+;; but if you really need to fetch high volumes of staples, you should start
 ;; more processes.
 renewal-threads=5
 
 ;; How long the scheduler should sleep between each scheduling attempt.
 refresh-interval=30
 
+;; Run only a one-off staple renewal and quit stapled when done. Note that this
+;; will still spawn the same amount of threads as a normal process would for
+;; performance reasons as well as consistency between one-off and normal runs.
+;; This setting overrides the --refresh-interval setting because a refresh is
+;; not scheduled during one-off runs. The --daemon and --no-daemon/--interactive
+;; arguments are also ignored.
+; one-off
+
 [logging]
 ;; Log to syslog, you can not set a `logdir` to only log to syslog, or
 ;; enable both at the same time. Uncomment to enable.
@@ -86,6 +99,15 @@ logdir=/var/log/stapled/
 ;; https://cbonte.github.io/haproxy-dconv/1.7/management.html#9.3-set%20ssl%20ocsp-response
 haproxy-sockets=[/var/run/haproxy/admin.sock]
 
+;; By default stapled will try to connect to the default socket path, which can
+;; be changed or set to an empty list by the --haproxy-sockets argument.
+;; The --no-haproxy-sockets argument explicitly disables the haproxy socket
+;; connection and overrides the --haproxy-sockets argument's paths if set.
+;; Note that this does NOT disable the --haproxy-config argument, i.e.: if a
+;; haproxy config is set, it will be parsed for certificate paths, without
+;; matching sockets.
+; no-haproxy-sockets
+
 ;; Use HAProxy config files as the source of cert-paths and socket mappings.
 ;; Setting this will merge your `cert-paths` with paths found in the specified
 ;; HAProxy config files. Sockets defined in `haproxy-sockets` will also be
@@ -103,8 +125,3 @@ haproxy-sockets=[/var/run/haproxy/admin.sock]
 ;; and/or `syslog` to prevent output on stdout while logging the set verbosity
 ;; level to a file or syslog. Uncomment to enable
 ; quiet
-
-;; Don't re-use existing ocsp files, refresh all staples regardless of their
-;; validity. By default existing staples are recycled if they are valid for
-;; longer than the minimum_validity setting.
-; no-recycle

debian/changelog

+stapled (1.4) buster; urgency=high
+
+ * Tested against buster
+ * Tested against Python 3.7
+ * Still tested against Python 3.5.
+ * Still tested against stretch
+ * Dependencies updated to latest versions
+ * Improved exception handling of haproxy adder module
+
+ -- Chris <chris@chrissnijder.nl> Wed, 11 Sep 2019 14:44:23 +0200
+
 stapled (1.3) stretch; urgency=low
 
  * Improved excpetion handling to limit flooding of logs

docker/build-buster/Dockerfile

+FROM debian:buster
+RUN apt-get update -qq
+RUN apt-get upgrade -y
+RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
+    python-all python3-all python3-dev python3-setuptools python3-pip \
+    rpm tar gzip bzip2 git debhelper ca-certificates
+ADD ./requirements.txt ./requirements.txt
+RUN pip3 install --user pip
+RUN pip3 install --user docutils==0.15.2
+RUN pip3 install --user -r requirements.txt

docker/build-stretch/Dockerfile

@@ -6,5 +6,5 @@ RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
     rpm tar gzip bzip2 git debhelper ca-certificates
 ADD ./requirements.txt ./requirements.txt
 RUN pip3 install --user pip
-RUN pip3 install --user docutils==0.14
+RUN pip3 install --user docutils==0.15.2
 RUN pip3 install --user -r requirements.txt

docker/test-buster/Dockerfile

+FROM debian:buster
+RUN apt-get update -qq
+RUN apt-get upgrade -y
+RUN apt-get install -y openssl ca-certificates python3-cffi \
+    python3-configargparse python3-daemon
+COPY ./refresh_testdata.sh ./refresh_testdata.sh

docs/source/using.rst

@@ -18,15 +18,15 @@ OCSP server locally, quite a setup. So until now we didn't do so yet. Note that
 if you have experience with this kind of setup and you want to help this project
 move forward, you are welcome to help.
 
-Obviously we do test stapled, admittedly a little bit primitively. You can find a
-script in ``scripts/`` called ``refresh_testdata.sh``. It will delete any
+Obviously we do test stapled, admittedly a little bit primitively. You can find
+a script in ``scripts/`` called ``refresh_testdata.sh``. It will delete any
 directory named ``testdata`` in the root of the project and create a fresh one.
 Then it will download 3 certificate chains from live servers. These will be
 placed in subdirectories with the same name as the domain name.
 
-Next you can run ``python stapled -vvvv -d testdata/*`` to get output printed to
-your terminal. The ``testdata/[domain].[tld]`` directories will be populated
-with ``[domain].[tld].ocsp`` files.
+Next you can run ``python -m stapled -vvvv -d testdata/*`` to get output
+printed to your terminal. The ``testdata/[domain].[tld]`` directories will be
+populated with ``[domain].[tld].ocsp`` files.
 
 Caveats
 =======

refresh_testdata.sh

@@ -20,4 +20,4 @@ for DOMAIN in $DOMAINS; do
     sed  -n "/-----BEGIN/,/-----END/ w $TESTDATA_PATH$DOMAIN/chain.pem"
 done
 echo "[+] Start stapled with the desired verbosity and test directory e.g.:"
-echo "python -m stapled -vvvv -d /tmp/testdata/ --recursive"
+echo "stapled -p /tmp/testdata/ --recursive --interactive --no-haproxy-sockets -vvvv --one-off"

requirements.txt

 # Project deps
 ConfigArgParse==0.14.0
 future==0.17.1
-cffi==1.12.2
+cffi==1.12.3
 
 ## python-daemon deps
 lockfile==0.12.2
-docutils==0.14
+docutils==0.15.2
 python-daemon==2.2.3
 
 # Packaging and distribution
-twine==1.13.0
-tqdm==4.31.1
+twine==1.14.0
+tqdm==4.35.0
 idna==2.8
-certifi==2018.11.29
+certifi==2019.6.16
 chardet==3.0.4
 pkginfo==1.5.0.1
-requests==2.21.0
+requests==2.22.0
 requests-toolbelt==0.9.1
-urllib3==1.24.1
+urllib3==1.25.3
 stdeb==0.8.5
-wheel==0.33.1
+wheel==0.33.6
 
 # Docs
-Sphinx==1.8.4
+Sphinx==2.2.0
 sphinx-argparse==0.2.5
 sphinx-rtd-theme==0.4.3
 
 # Linting
-pydocstyle==3.0.0
+pydocstyle==4.0.1
 pycodestyle==2.5.0
-pylint==2.3.0
+pylint==2.3.1
 
 # Testing
-pytest==4.3.0
+pytest==5.1.2

setup.py

@@ -39,21 +39,21 @@ setup(
     url='https://github.com/greenhost/stapled',
     packages=all_packages(),
     package_dir=find_lib_path_dict(),
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, <4',
+    python_requires='!=3.0.*, !=3.1.*, !=3.2.*, <4',
     install_requires=[
-        'python-daemon>=2.2.0',
-        'configargparse>=0.13.0',
+        'python-daemon>=2.2.3',
+        'configargparse>=0.14.0',
         # Required by deps in `stapled/libs`
         'future>=0.17.1',
-        'cffi>=1.11.5',
+        'cffi>=1.12.3',
         # Required for python-daemon (package dep not specified)
-        'docutils>=0.14',
+        'docutils>=0.15.2',
     ],
     extras_require={
         'docs': [
-            'Sphinx>=1.8.2',
+            'Sphinx>=2.2.0',
             'sphinx-argparse>=0.2.5',
-            'sphinx_rtd_theme>=0.4.2',
+            'sphinx_rtd_theme>=0.4.3',
         ]
     },
     license='Apache Version 2.0',
@@ -64,8 +64,6 @@ setup(
         'License :: OSI Approved :: Apache Version 2.0',
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',

stapled/__main__.py

@@ -217,8 +217,7 @@ def get_cli_arg_parser():
     )
     parser.add(
         '--no-haproxy-sockets',
-        action='store_false',
-        dest='haproxy_sockets',
+        action='store_true',
         help=(
             "Disable HAProxy sockets, overrides ``--haproxy-sockets`` if "
             "specified in the config file."
@@ -348,8 +347,8 @@ def init():
     # config files, de-duplicated.
     cert_paths = haproxy_socket_mapping.keys()
 
-    # Determine if we need to start a stapleadder thread.
-    if not any(haproxy_socket_mapping.values()):
+    # Determine if we need to start a staple adder thread.
+    if args.no_haproxy_sockets or not any(haproxy_socket_mapping.values()):
         haproxy_socket_mapping = None
 
     daemon_kwargs = dict(
@@ -499,15 +498,19 @@ def __get_haproxy_socket_mapping(args):
     # files in the sockets dictionary.
     for i, paths in enumerate(conf_cert_paths):
         for path in paths:
-            if path in mapping:
+            # When certificate path is already in the mapping add the socket
+            # file to the mapping if haproxy_sockets is not disabled.
+            if args.no_haproxy_sockets:
+                # haproxy_sockets are disabled, just ensure the path is in the
+                # mapping without sockets.
+                mapping[path] = []
+            else:
+                # Make sure paths are mapped to all unique sockets.
                 mapping[path] = unique(
-                    mapping[path] + conf_haproxy_sockets[i],
+                    mapping.get(path, []) + conf_haproxy_sockets[i],
                     preserve_order=False
                 )
-            else:
-                mapping[path] = conf_haproxy_sockets[i]
-
-    logger.debug("Paths to socket mapping: %s", str(mapping))
+    logger.debug("Paths to socket mappings: %s", str(mapping))
     return mapping
 
 
@@ -537,47 +540,6 @@ def __get_validated_args():
     return args
 
 
-def __get_haproxy_socket_mapping(args):
-    """
-    Get mapping of configured sockets and certificate directories.
-
-    From: haproxy config, stapled config and command line arguments.
-
-    :param Namespace args: Argparser argument list.
-    :return dict Of cert-paths and sockets for inform of changes.
-    """
-    # Parse the cert_paths argument
-    arg_cert_paths = __get_arg_cert_paths(args)
-    # Parse haproxy_sockets argument.
-    arg_haproxy_sockets = __get_arg_haproxy_sockets(args)
-    # Make a mapping from certificate paths to sockets in a dict.
-    mapping = dict(zip(arg_cert_paths, arg_haproxy_sockets))
-
-    # Parse HAProxy config files.
-    try:
-        conf_cert_paths, conf_haproxy_sockets = parse_haproxy_config(
-            args.haproxy_config
-        )
-    except (OSError) as exc:
-        logger.critical(exc)
-        exit(1)
-
-    # Combine the socket and certificate paths of the arguments and config
-    # files in the sockets dictionary.
-    for i, paths in enumerate(conf_cert_paths):
-        for path in paths:
-            if path in mapping:
-                mapping[path] = unique(
-                    mapping[path] + conf_haproxy_sockets[i],
-                    preserve_order=False
-                )
-            else:
-                mapping[path] = conf_haproxy_sockets[i]
-
-    logger.debug("Paths to socket mapping: %s", str(mapping))
-    return mapping
-
-
 if __name__ == '__main__':
     try:
         init()

stapled/core/daemon.py

@@ -285,7 +285,7 @@ class Stapledaemon(object):
         ]
         # Check if enabled before adding stapleadder queue.
         if self.staple_adder:
-            stop_threads.append(('adder', [self.staple_adder]))
+            stop_threads.append(('proxy-add', [self.staple_adder]))
 
         def one_off_generator():
             """Make generator to iteratively end the stapled process."""
@@ -297,7 +297,8 @@ class Stapledaemon(object):
 
             for queue, threads in stop_threads:
                 # Queue must exist, if it doesn't it wasn't yet created.
-                if not self.scheduler.queue_exists(queue):
+                # Wait for it to be created.
+                while not self.scheduler.queue_exists(queue):
                     yield False
 
                 # Wait until queues are empty..

asn1crypto @ 6060d29a

-Subproject commit 7f587d56f2b5c4815bfed835cfc10abd9c49cdb2
+Subproject commit 6060d29ac5ef67abc9f4a8347813c0ca87f328bd

certvalidator @ b69d3b74

-Subproject commit 4383a4bfd5e769679bc4eedd1e4d334eb0c7d85a
+Subproject commit b69d3b745b5af9e5ccd4b9781407ab7e82076d6b

ocspbuilder @ 64e7a1f0

-Subproject commit 2db8721710d193107c7787f8c094440443b7fbfa
+Subproject commit 64e7a1f02615e8ec1e7826a097883c480ff73a1c

oscrypto @ 58575945

-Subproject commit ef86af48af8e0362b163c9807adf389460debac1
+Subproject commit 585759457f60a0979274862db2e8dc6211b28a24

stapled/version.py

-__version__ = '1.3'
+__version__ = '1.4'
 __app_name__ = 'stapled'
 __debian_version__ = 'stretch'