Commit 86b9272a authored by Chris Snijder's avatar Chris Snijder 🏅
Browse files

Merge branch '66-make-new-release-for-ops' into 'master'

Resolve "Make new release for OPS"

Closes #66

See merge request !50
parents 844c0c81 3cfde704
Pipeline #8466 passed with stages
in 9 minutes and 12 seconds
......@@ -7,8 +7,9 @@ stages:
variables:
DOCKER_DRIVER: overlay2
DOCKER_FILES_URL: docker.greenhost.net/open/stapled
BUILD_COMPILE_CONTAINER: $DOCKER_FILES_URL/build-stretch
BUILD_COMPILE_CONTAINER: $DOCKER_FILES_URL/build-buster
STRETCH_TEST_CONTAINER: $DOCKER_FILES_URL/test-stretch
BUSTER_TEST_CONTAINER: $DOCKER_FILES_URL/test-buster
# GL bug causes recursive strategy to fail for LE certificates
# GH recently started using LE certificates:
# https://gitlab.com/gitlab-org/gitlab-runner/issues/2148
......@@ -30,7 +31,7 @@ build:compile-container:
- docker push $BUILD_COMPILE_CONTAINER:$CI_BUILD_REF
- docker push $BUILD_COMPILE_CONTAINER:latest
build:test-container:
build:test-container-stretch:
stage: build-containers
image: docker:stable-git
services:
......@@ -45,6 +46,21 @@ build:test-container:
- docker push $STRETCH_TEST_CONTAINER:$CI_BUILD_REF
- docker push $STRETCH_TEST_CONTAINER:latest
build:test-container-buster:
stage: build-containers
image: docker:stable-git
services:
#- docker:stable-dind
- docker:18-dind # Hotfix for https://gitlab.com/gitlab-org/gitlab-runner/issues/4501
variables:
GIT_STRATEGY: fetch
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN docker.greenhost.net
- docker pull $BUSTER_TEST_CONTAINER || true
- docker build --cache-from $CONTAINER_IMAGE:latest --pull --cache-from $BUSTER_TEST_CONTAINER:latest -t $BUSTER_TEST_CONTAINER:$CI_BUILD_REF -t $BUSTER_TEST_CONTAINER:latest -f ./docker/test-buster/Dockerfile ./
- docker push $BUSTER_TEST_CONTAINER:$CI_BUILD_REF
- docker push $BUSTER_TEST_CONTAINER:latest
build:package:
stage: packaging
image: $BUILD_COMPILE_CONTAINER
......@@ -54,7 +70,7 @@ build:package:
- which python3 && $(which python3) --version
- openssl version
# Hack to get python-daemon to install..
- pip3 install --user docutils==0.14
- pip3 install --user docutils==0.15.2
- pip3 install --user -r requirements.txt
- make clean
- make
......@@ -66,15 +82,27 @@ build:package:
- dist/stapled*.tar.bz2
- dist/stapled*.deb
source:unit:
source:unit-stretch:
stage: source-tests
image: python:3.6-stretch
image: python:3.5-stretch
script:
- git submodule sync --recursive
- git submodule update --init --recursive
- which python3 && $(which python3) --version
# Hack to get python-daemon to install..
- pip3 install --user docutils==0.14
- pip3 install --user docutils==0.15.2
- pip3 install -r requirements.txt
- pytest -v
source:unit-buster:
stage: source-tests
image: python:3.7-buster
script:
- git submodule sync --recursive
- git submodule update --init --recursive
- which python3 && $(which python3) --version
# Hack to get python-daemon to install..
- pip3 install --user docutils==0.15.2
- pip3 install -r requirements.txt
- pytest -v
......@@ -92,9 +120,23 @@ test:stretch:
dependencies:
- build:package
test:buster:
stage: functional-tests
image: $BUSTER_TEST_CONTAINER
variables:
GIT_STRATEGY: none
script:
- which python3 && $(which python3) --version
- openssl version
- apt-get install -y -q ./dist/stapled_*all.deb
- /refresh_testdata.sh
- stapled -p /tmp/testdata/ --recursive --interactive --no-haproxy-sockets -vvvv --one-off
dependencies:
- build:package
source:dev-setup:
stage: source-tests
image: python:3.6-stretch
image: python:3.7-buster
script:
- git submodule sync --recursive
- git submodule update --init --recursive
......
FROM debian:stretch
FROM debian:buster
RUN apt-get update -qq
RUN apt-get upgrade -y
RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
......@@ -7,4 +7,5 @@ RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
RUN pip3 install --user pip
ADD . ./
RUN pip3 install -r requirements.txt
RUN pip3 install certvalidator ocspbuilder asn1crypto oscrypto
CMD echo Ready for your commands. && /bin/bash
Stapled (Staple Daemon)
Copyright 2017 Greenhost BV
Copyright 2019 Greenhost BV
Licensed Apache Version 2.0
......
......@@ -52,12 +52,27 @@ From github (for developers)
virtualenv -p python3 env/
# Load the virtualenv
source env/bin/activate
# Install the current directory with pip. This allows you to edit the code
pip install -e .
Every time you want to run ``stapled`` you will need to run
``source env/bin/activate`` to load the virtualenv first. Alternatively you can
start the daemon by running ``stapled``
``source env/bin/activate`` to load the virtualenv first. Then you run stapled
as a module:
.. code-block:: bash
pythom -m stapled [arguments]
Alternatively you can start the daemon by running ``stapled`` without even
activating the virtualenv if you install it like this:
.. code-block:: bash
# Install dependencies..
pip3 install asn1crypto ocspbuilder oscrypto certvalidator
# Install the current directory with pip. This install the project dir as
# a console script allowing you to run `stapled`,
pip3 install -e .
Note that this means you have to keep track of the installed dependencies
yourself!
Upgrading
---------
......@@ -78,7 +93,7 @@ the following:
# Clone submodules too
git submodule upgrade --init --recursive
# Install the current directory with pip. This allows you to edit the code
pip install -e . --upgrade
pip3 install -e . --upgrade
Troubleshooting
===============
......@@ -143,7 +158,8 @@ by running one or more of the following ``make`` commands.
# All of the above
make all
Everything is tested under Debian Stretch, your mileage may vary.
Everything is tested under Debian Stretch (Python 3.5 and Debian Buster
(Python 3.7), on other distros :abbr:`YMMV (Your Mileage May Vary)`.
Docker build
------------
......@@ -159,7 +175,7 @@ build a package.
make docker-all
Remove any previous docker image and/or container named `stapled` then buil the
Remove any previous docker image and/or container named `stapled` then build the
image with the same dependencies we used. Then compile the packages, then
place them in the `./docker-dist` dir.
......
......@@ -37,9 +37,14 @@ file-extensions=crt,pem,cer
;; ignore=[no-ocsp/*.crt, /etc/ssl/private/not_a_real_crt.pem]
; ignore=ssl-cert-snakeoil.key
;; Recursively scan the paths specified by --cert-paths for certificates.
; recursive
[validity]
;; Uncomment to update every staple at startup. Leave commented to try to
;; Don't re-use existing ocsp files, refresh all staples regardless of their
;; validity. By default existing staples are recycled if they are valid for
;; longer than the minimum_validity setting. Leave commented to try to
;; re-use staples that are still valid long enough (See `minimum-validity`
;; directive)
; no-recycle
......@@ -58,14 +63,22 @@ daemon
;; Amount of threads to use for the renewal process. Increasing this will only
;; help if the daemon is sitting idle, e.g. waiting for OCSP responses for
;; longerperiods of time. It can help increase concurrency to a certain point
;; but ifyou really need to fetch high volumes of staples, you should start
;; longer periods of time. It can help increase concurrency to a certain point
;; but if you really need to fetch high volumes of staples, you should start
;; more processes.
renewal-threads=5
;; How long the scheduler should sleep between each scheduling attempt.
refresh-interval=30
;; Run only a one-off staple renewal and quit stapled when done. Note that this
;; will still spawn the same amount of threads as a normal process would for
;; performance reasons as well as consistency between one-off and normal runs.
;; This setting overrides the --refresh-interval setting because a refresh is
;; not scheduled during one-off runs. The --daemon and --no-daemon/--interactive
;; arguments are also ignored.
; one-off
[logging]
;; Log to syslog, you can not set a `logdir` to only log to syslog, or
;; enable both at the same time. Uncomment to enable.
......@@ -86,6 +99,15 @@ logdir=/var/log/stapled/
;; https://cbonte.github.io/haproxy-dconv/1.7/management.html#9.3-set%20ssl%20ocsp-response
haproxy-sockets=[/var/run/haproxy/admin.sock]
;; By default stapled will try to connect to the default socket path, which can
;; be changed or set to an empty list by the --haproxy-sockets argument.
;; The --no-haproxy-sockets argument explicitly disables the haproxy socket
;; connection and overrides the --haproxy-sockets argument's paths if set.
;; Note that this does NOT disable the --haproxy-config argument, i.e.: if a
;; haproxy config is set, it will be parsed for certificate paths, without
;; matching sockets.
; no-haproxy-sockets
;; Use HAProxy config files as the source of cert-paths and socket mappings.
;; Setting this will merge your `cert-paths` with paths found in the specified
;; HAProxy config files. Sockets defined in `haproxy-sockets` will also be
......@@ -103,8 +125,3 @@ haproxy-sockets=[/var/run/haproxy/admin.sock]
;; and/or `syslog` to prevent output on stdout while logging the set verbosity
;; level to a file or syslog. Uncomment to enable
; quiet
;; Don't re-use existing ocsp files, refresh all staples regardless of their
;; validity. By default existing staples are recycled if they are valid for
;; longer than the minimum_validity setting.
; no-recycle
stapled (1.4) buster; urgency=high
* Tested against buster
* Tested against Python 3.7
* Still tested against Python 3.5.
* Still tested against stretch
* Dependencies updated to latest versions
* Improved exception handling of haproxy adder module
-- Chris <chris@chrissnijder.nl> Wed, 11 Sep 2019 14:44:23 +0200
stapled (1.3) stretch; urgency=low
* Improved excpetion handling to limit flooding of logs
......
FROM debian:buster
RUN apt-get update -qq
RUN apt-get upgrade -y
RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
python-all python3-all python3-dev python3-setuptools python3-pip \
rpm tar gzip bzip2 git debhelper ca-certificates
ADD ./requirements.txt ./requirements.txt
RUN pip3 install --user pip
RUN pip3 install --user docutils==0.15.2
RUN pip3 install --user -r requirements.txt
......@@ -6,5 +6,5 @@ RUN apt-get install -q -y build-essential python3-cffi libffi-dev \
rpm tar gzip bzip2 git debhelper ca-certificates
ADD ./requirements.txt ./requirements.txt
RUN pip3 install --user pip
RUN pip3 install --user docutils==0.14
RUN pip3 install --user docutils==0.15.2
RUN pip3 install --user -r requirements.txt
FROM debian:buster
RUN apt-get update -qq
RUN apt-get upgrade -y
RUN apt-get install -y openssl ca-certificates python3-cffi \
python3-configargparse python3-daemon
COPY ./refresh_testdata.sh ./refresh_testdata.sh
......@@ -18,15 +18,15 @@ OCSP server locally, quite a setup. So until now we didn't do so yet. Note that
if you have experience with this kind of setup and you want to help this project
move forward, you are welcome to help.
Obviously we do test stapled, admittedly a little bit primitively. You can find a
script in ``scripts/`` called ``refresh_testdata.sh``. It will delete any
Obviously we do test stapled, admittedly a little bit primitively. You can find
a script in ``scripts/`` called ``refresh_testdata.sh``. It will delete any
directory named ``testdata`` in the root of the project and create a fresh one.
Then it will download 3 certificate chains from live servers. These will be
placed in subdirectories with the same name as the domain name.
Next you can run ``python stapled -vvvv -d testdata/*`` to get output printed to
your terminal. The ``testdata/[domain].[tld]`` directories will be populated
with ``[domain].[tld].ocsp`` files.
Next you can run ``python -m stapled -vvvv -d testdata/*`` to get output
printed to your terminal. The ``testdata/[domain].[tld]`` directories will be
populated with ``[domain].[tld].ocsp`` files.
Caveats
=======
......
......@@ -20,4 +20,4 @@ for DOMAIN in $DOMAINS; do
sed -n "/-----BEGIN/,/-----END/ w $TESTDATA_PATH$DOMAIN/chain.pem"
done
echo "[+] Start stapled with the desired verbosity and test directory e.g.:"
echo "python -m stapled -vvvv -d /tmp/testdata/ --recursive"
echo "stapled -p /tmp/testdata/ --recursive --interactive --no-haproxy-sockets -vvvv --one-off"
# Project deps
ConfigArgParse==0.14.0
future==0.17.1
cffi==1.12.2
cffi==1.12.3
## python-daemon deps
lockfile==0.12.2
docutils==0.14
docutils==0.15.2
python-daemon==2.2.3
# Packaging and distribution
twine==1.13.0
tqdm==4.31.1
twine==1.14.0
tqdm==4.35.0
idna==2.8
certifi==2018.11.29
certifi==2019.6.16
chardet==3.0.4
pkginfo==1.5.0.1
requests==2.21.0
requests==2.22.0
requests-toolbelt==0.9.1
urllib3==1.24.1
urllib3==1.25.3
stdeb==0.8.5
wheel==0.33.1
wheel==0.33.6
# Docs
Sphinx==1.8.4
Sphinx==2.2.0
sphinx-argparse==0.2.5
sphinx-rtd-theme==0.4.3
# Linting
pydocstyle==3.0.0
pydocstyle==4.0.1
pycodestyle==2.5.0
pylint==2.3.0
pylint==2.3.1
# Testing
pytest==4.3.0
pytest==5.1.2
......@@ -39,21 +39,21 @@ setup(
url='https://github.com/greenhost/stapled',
packages=all_packages(),
package_dir=find_lib_path_dict(),
python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, <4',
python_requires='!=3.0.*, !=3.1.*, !=3.2.*, <4',
install_requires=[
'python-daemon>=2.2.0',
'configargparse>=0.13.0',
'python-daemon>=2.2.3',
'configargparse>=0.14.0',
# Required by deps in `stapled/libs`
'future>=0.17.1',
'cffi>=1.11.5',
'cffi>=1.12.3',
# Required for python-daemon (package dep not specified)
'docutils>=0.14',
'docutils>=0.15.2',
],
extras_require={
'docs': [
'Sphinx>=1.8.2',
'Sphinx>=2.2.0',
'sphinx-argparse>=0.2.5',
'sphinx_rtd_theme>=0.4.2',
'sphinx_rtd_theme>=0.4.3',
]
},
license='Apache Version 2.0',
......@@ -64,8 +64,6 @@ setup(
'License :: OSI Approved :: Apache Version 2.0',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
......
......@@ -217,8 +217,7 @@ def get_cli_arg_parser():
)
parser.add(
'--no-haproxy-sockets',
action='store_false',
dest='haproxy_sockets',
action='store_true',
help=(
"Disable HAProxy sockets, overrides ``--haproxy-sockets`` if "
"specified in the config file."
......@@ -348,8 +347,8 @@ def init():
# config files, de-duplicated.
cert_paths = haproxy_socket_mapping.keys()
# Determine if we need to start a stapleadder thread.
if not any(haproxy_socket_mapping.values()):
# Determine if we need to start a staple adder thread.
if args.no_haproxy_sockets or not any(haproxy_socket_mapping.values()):
haproxy_socket_mapping = None
daemon_kwargs = dict(
......@@ -499,15 +498,19 @@ def __get_haproxy_socket_mapping(args):
# files in the sockets dictionary.
for i, paths in enumerate(conf_cert_paths):
for path in paths:
if path in mapping:
# When certificate path is already in the mapping add the socket
# file to the mapping if haproxy_sockets is not disabled.
if args.no_haproxy_sockets:
# haproxy_sockets are disabled, just ensure the path is in the
# mapping without sockets.
mapping[path] = []
else:
# Make sure paths are mapped to all unique sockets.
mapping[path] = unique(
mapping[path] + conf_haproxy_sockets[i],
mapping.get(path, []) + conf_haproxy_sockets[i],
preserve_order=False
)
else:
mapping[path] = conf_haproxy_sockets[i]
logger.debug("Paths to socket mapping: %s", str(mapping))
logger.debug("Paths to socket mappings: %s", str(mapping))
return mapping
......@@ -537,47 +540,6 @@ def __get_validated_args():
return args
def __get_haproxy_socket_mapping(args):
"""
Get mapping of configured sockets and certificate directories.
From: haproxy config, stapled config and command line arguments.
:param Namespace args: Argparser argument list.
:return dict Of cert-paths and sockets for inform of changes.
"""
# Parse the cert_paths argument
arg_cert_paths = __get_arg_cert_paths(args)
# Parse haproxy_sockets argument.
arg_haproxy_sockets = __get_arg_haproxy_sockets(args)
# Make a mapping from certificate paths to sockets in a dict.
mapping = dict(zip(arg_cert_paths, arg_haproxy_sockets))
# Parse HAProxy config files.
try:
conf_cert_paths, conf_haproxy_sockets = parse_haproxy_config(
args.haproxy_config
)
except (OSError) as exc:
logger.critical(exc)
exit(1)
# Combine the socket and certificate paths of the arguments and config
# files in the sockets dictionary.
for i, paths in enumerate(conf_cert_paths):
for path in paths:
if path in mapping:
mapping[path] = unique(
mapping[path] + conf_haproxy_sockets[i],
preserve_order=False
)
else:
mapping[path] = conf_haproxy_sockets[i]
logger.debug("Paths to socket mapping: %s", str(mapping))
return mapping
if __name__ == '__main__':
try:
init()
......
......@@ -285,7 +285,7 @@ class Stapledaemon(object):
]
# Check if enabled before adding stapleadder queue.
if self.staple_adder:
stop_threads.append(('adder', [self.staple_adder]))
stop_threads.append(('proxy-add', [self.staple_adder]))
def one_off_generator():
"""Make generator to iteratively end the stapled process."""
......@@ -297,7 +297,8 @@ class Stapledaemon(object):
for queue, threads in stop_threads:
# Queue must exist, if it doesn't it wasn't yet created.
if not self.scheduler.queue_exists(queue):
# Wait for it to be created.
while not self.scheduler.queue_exists(queue):
yield False
# Wait until queues are empty..
......
Subproject commit 7f587d56f2b5c4815bfed835cfc10abd9c49cdb2
Subproject commit 6060d29ac5ef67abc9f4a8347813c0ca87f328bd
Subproject commit 4383a4bfd5e769679bc4eedd1e4d334eb0c7d85a
Subproject commit b69d3b745b5af9e5ccd4b9781407ab7e82076d6b
Subproject commit 2db8721710d193107c7787f8c094440443b7fbfa
Subproject commit 64e7a1f02615e8ec1e7826a097883c480ff73a1c
Subproject commit ef86af48af8e0362b163c9807adf389460debac1
Subproject commit 585759457f60a0979274862db2e8dc6211b28a24
__version__ = '1.3'
__version__ = '1.4'
__app_name__ = 'stapled'
__debian_version__ = 'stretch'
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment