diff --git a/README.md b/README.md index 239158df..7a473ddc 100644 --- a/README.md +++ b/README.md @@ -97,9 +97,15 @@ docker run --rm circa10a/ouroboros --help - `--cleanup`, `-c` Remove the older docker image if a new one is found and updated. - Default is `False`. - Environment variable: `CLEANUP=true` -- `--keep-tag`, `-k` Only monitor if updates are made to the tag of the image that the container was created with instead of using `latest`. +- `--keep-tag`, `-k` Only monitor if updates are made to the tag of the image that the container was created with instead of using `latest`. This will enable [watchtower](https://github.com/v2tec/watchtower)-like functionality. - Default is `False`. - Environment variable: `KEEPTAG=true` +- `--metrics-addr` What address for the prometheus endpoint to bind to. Runs on `127.0.0.1` by default if `--metrics-addr` is not supplied. + - Default is `127.0.0.1`. + - Environment variable: `METRICS_ADDR=127.0.0.1` +- `--metrics-port` What port to run prometheus endpoint on. Running on port `8000` by default if `--metrics-port` is not supplied. + - Default is `8000`. + - Environment variable: `METRICS_PORT=8000` ### Private Registries @@ -107,7 +113,7 @@ If your running containers' docker images are stored in a secure registry that r ```bash docker run -d --name ouroboros \ - -v REPO_USER=myUser -e REPO_PASS=myPassword \ + -e REPO_USER=myUser -e REPO_PASS=myPassword \ -v /var/run/docker.sock:/var/run/docker.sock \ circa10a/ouroboros ``` @@ -205,6 +211,55 @@ docker run -d --name ouroboros \ circa10a/ouroboros --cleanup ``` +### Prometheus metrics + +Ouroboros keeps track of containers being updated and how many are being monitored. Said metrics are exported using [prometheus](https://prometheus.io/). Metrics are collected by ouroboros with or without this flag, it is up to you if you would like to expose the port or not. You can also bind the http server to a different interface for systems using multiple networks. `--metrics-port` and `--metrics-addr` can run independently of each other without issue. + +#### Port + +> Default is `8000` + +```bash +docker run -d --name ouroboros \ + -p 5000:5000 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + circa10a/ouroboros --metrics-port 5000 +``` + +You should then be able to see the metrics at http://localhost:5000/ + +#### Bind Address + +Ouroboros allows you to bind the exporter to a different interface using the `--metrics-addr` argument. + +> Default is `127.0.0.1` + +```bash +docker run -d --name ouroboros \ + -p 8000:8000 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + circa10a/ouroboros --metrics-addr 10.0.0.1 +``` + +Then access via http://10.0.0.1:8000/ + +**Example text from endpoint:** + +``` +# HELP containers_updated_total Count of containers updated +# TYPE containers_updated_total counter +containers_updated_total{container="all"} 2.0 +containers_updated_total{container="alpine"} 1.0 +containers_updated_total{container="busybox"} 1.0 +# TYPE containers_updated_created gauge +containers_updated_created{container="all"} 1542152615.625264 +containers_updated_created{container="alpine"} 1542152615.6252713 +containers_updated_created{container="busybox"} 1542152627.7476819 +# HELP containers_being_monitored Count of containers being monitored +# TYPE containers_being_monitored gauge +containers_being_monitored 2.0 +``` + ## Execute Tests > Script will install dependencies from `requirements-dev.txt` diff --git a/dev-environment.yml b/dev-environment.yml index 4e941fba..e5b963e6 100644 --- a/dev-environment.yml +++ b/dev-environment.yml @@ -3,6 +3,7 @@ dependencies: - pip: - docker - schedule + - prometheus_client - pytest - pytest-cov - pytest-mock diff --git a/environment.yml b/environment.yml index 6b5bb754..8a35e3c2 100644 --- a/environment.yml +++ b/environment.yml @@ -3,3 +3,4 @@ dependencies: - pip: - docker - schedule + - prometheus_client diff --git a/ouroboros/cli.py b/ouroboros/cli.py index 94a35efc..59ee864a 100644 --- a/ouroboros/cli.py +++ b/ouroboros/cli.py @@ -35,11 +35,10 @@ def checkURI(uri): return re.match(regex, uri) -def get_interval_env(): - """Attempt to convert INTERVAL environment variable to int""" - int_env = environ.get('INTERVAL') +def get_int_env_var(env_var): + """Attempt to convert environment variable to int""" try: - return int(int_env) + return int(env_var) except (ValueError, TypeError): return False @@ -53,7 +52,7 @@ def parse(sysargs): parser.add_argument('-u', '--url', default=defaults.LOCAL_UNIX_SOCKET, help='Url for tcp host (defaults to "unix://var/run/docker.sock")') - parser.add_argument('-i', '--interval', type=int, default=get_interval_env() or defaults.INTERVAL, dest='interval', + parser.add_argument('-i', '--interval', type=int, default=get_int_env_var(env_var=environ.get('INTERVAL')) or defaults.INTERVAL, dest='interval', help='Interval in seconds between checking for updates (defaults to 300s)') parser.add_argument('-m', '--monitor', nargs='+', default=environ.get('MONITOR') or [], dest='monitor', @@ -74,6 +73,12 @@ def parse(sysargs): parser.add_argument('-k', '--keep-tag', default=environ.get('KEEPTAG') or False, dest='keep_tag', help='Check for image updates of the same tag instead of pulling latest', action='store_true') + + parser.add_argument('--metrics-addr', default=environ.get('METRICS_ADDR') or defaults.METRICS_ADDR, dest='metrics_addr', + help='Bind address to run Prometheus exporter on') + + parser.add_argument('--metrics-port', type=int, default=get_int_env_var(env_var=environ.get('METRICS_PORT')) or defaults.METRICS_PORT, dest='metrics_port', + help='Port to run Prometheus exporter on') args = parser.parse_args(sysargs) if not args.url: diff --git a/ouroboros/defaults.py b/ouroboros/defaults.py index 4fa73f4c..4355e913 100644 --- a/ouroboros/defaults.py +++ b/ouroboros/defaults.py @@ -5,3 +5,5 @@ RUNONCE = False CLEANUP = False KEEPTAG = False +METRICS_ADDR = '127.0.0.1' +METRICS_PORT = 8000 diff --git a/ouroboros/main.py b/ouroboros/main.py index 7ea66e1f..c557b860 100644 --- a/ouroboros/main.py +++ b/ouroboros/main.py @@ -3,6 +3,7 @@ import docker from ouroboros import container from ouroboros import image +from ouroboros import metrics def main(args, api_client): @@ -12,7 +13,10 @@ def main(args, api_client): log.info('No containers are running') else: updated_count = 0 - for running_container in container.to_monitor(monitor=args.monitor, ignore=args.ignore, api_client=api_client): + monitored_containers = container.to_monitor(monitor=args.monitor, ignore=args.ignore, api_client=api_client) + metrics.monitored_containers(num=len(monitored_containers)) + for running_container in monitored_containers: + container_name = f'{container.get_name(container_object=running_container)}' current_image = api_client.inspect_image(running_container['Config']['Image']) try: @@ -40,6 +44,10 @@ def main(args, api_client): if args.cleanup: image.remove(old_image=current_image, api_client=api_client) updated_count += 1 + + metrics.container_updates(label='all') + metrics.container_updates(label=container_name) + log.info(f'{updated_count} container(s) updated') if args.run_once: exit(0) diff --git a/ouroboros/metrics.py b/ouroboros/metrics.py new file mode 100644 index 00000000..29a3e405 --- /dev/null +++ b/ouroboros/metrics.py @@ -0,0 +1,17 @@ +from prometheus_client import Counter, Gauge + +updated_containers_counter = Counter( + 'containers_updated', 'Count of containers updated', ['container']) + +monitored_containers_gauge = Gauge( + 'containers_being_monitored', 'Count of containers being monitored', []) + + +def container_updates(label): + """Increment container update count based on label""" + updated_containers_counter.labels(container=label).inc() + + +def monitored_containers(num): + """Set number of containers being monitoring with a gauge""" + monitored_containers_gauge.set(num) diff --git a/ouroboros/ouroboros b/ouroboros/ouroboros index 74e8a148..26be3c4c 100644 --- a/ouroboros/ouroboros +++ b/ouroboros/ouroboros @@ -4,12 +4,14 @@ if __name__ == "__main__": import sys import ouroboros.cli as cli from ouroboros.logger import set_logger + from prometheus_client import start_http_server import docker import schedule import logging from ouroboros.main import main import time args = cli.parse(sys.argv[1:]) + start_http_server(args.metrics_port, addr=args.metrics_addr) api_client = docker.APIClient(base_url=args.url) logging.basicConfig(**set_logger(args.loglevel)) schedule.every(args.interval).seconds.do( diff --git a/requirements-dev.txt b/requirements-dev.txt index 9f79b0a3..440fd4cf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ docker schedule +prometheus_client pytest pytest-cov pytest-mock diff --git a/requirements.txt b/requirements.txt index 5861bdf4..ce2cb500 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ docker -schedule \ No newline at end of file +schedule +prometheus_client \ No newline at end of file diff --git a/setup.py b/setup.py index d0da75dc..e6077734 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ def read_reqs(requirements): setup( name='ouroboros-cli', - version='0.2.3', + version='0.3.0', description='Automatically update running docker containers', long_description=readme(), long_description_content_type='text/markdown', diff --git a/tests/unit/_metrics_test.py b/tests/unit/_metrics_test.py new file mode 100644 index 00000000..08d364ef --- /dev/null +++ b/tests/unit/_metrics_test.py @@ -0,0 +1,17 @@ +import pytest +from prometheus_client import REGISTRY +import ouroboros.metrics as metrics + + +def test_container_updates(): + test_label = 'test' + metrics.container_updates(label=test_label) + increment = REGISTRY.get_sample_value('containers_updated_total', labels={'container': test_label}) + assert increment == 1.0 + + +def test_monitored_containers(): + test_count = 5.0 + metrics.monitored_containers(num=test_count) + num_monitored = REGISTRY.get_sample_value('containers_being_monitored') + assert num_monitored == test_count \ No newline at end of file diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 4e3010f0..c31d382a 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -1,3 +1,4 @@ +from os import environ import pytest import ouroboros.cli as cli import ouroboros.defaults as defaults @@ -22,16 +23,15 @@ def test_url_args(mocker, url_args, url_result): args = cli.parse(url_args) assert args.url == url_result -# Interval - +# Interval @pytest.mark.parametrize('interval_env, interval_env_result', [ ({'INTERVAL': 't'}, False), ({'INTERVAL': '10'}, 10), ]) -def test_get_interval_env(mocker, interval_env, interval_env_result): +def get_int_env_var(mocker, interval_env, interval_env_result): mocker.patch.dict('os.environ', interval_env) - assert cli.get_interval_env() == interval_env_result + assert cli.get_int_env_var(env_var=environ.get('INTERVAL')) == interval_env_result def test_interval_arg_invalid_value(mocker): @@ -45,9 +45,8 @@ def test_interval_arg_valid_value(mocker): mocker.patch('ouroboros.cli') assert cli.parse(['--interval', 0]) -# Monitor - +# Monitor @pytest.mark.parametrize('monitor_args, monitor_result', [ (['-m', 'test1', 'test2', 'test3'], ['test1', 'test2', 'test3']), (['--monitor', 'test1', 'test2', 'test3'], ['test1', 'test2', 'test3']), @@ -105,6 +104,7 @@ def test_loglevel_env_var(mocker, loglevel_env_var, loglevel_env_var_result): assert args.loglevel == loglevel_env_var_result +# Run once @pytest.mark.parametrize('runonce_args, runonce_result', [ (['-r', ], True), (['--runonce', ], True) @@ -126,6 +126,7 @@ def test_runonce_env_var(mocker, runonce_env_var, runonce_env_var_result): assert args.run_once == runonce_env_var_result +# Cleanup @pytest.mark.parametrize('cleanup_args, cleanup_result', [ (['-c', ], True), (['--cleanup', ], True) @@ -147,6 +148,7 @@ def test_cleanup_env_var(mocker, cleanup_env_var, cleanup_env_var_result): assert args.cleanup == cleanup_env_var_result +# Keeptag @pytest.mark.parametrize('keeptag_args, keeptag_result', [ (['-k', ], True), (['--keep-tag', ], True) @@ -165,4 +167,50 @@ def test_keeptag_env_var(mocker, keeptag_env_var, keeptag_env_var_result): mocker.patch.dict('os.environ', keeptag_env_var) mocker.patch('ouroboros.cli') args = cli.parse([]) - assert args.keep_tag == keeptag_env_var_result \ No newline at end of file + assert args.keep_tag == keeptag_env_var_result + + +# METRICS_ADDR +@pytest.mark.parametrize('metrics_addr_args, metrics_addr_result', [ + (['--metrics-addr', '127.0.0.0'], '127.0.0.0') +]) +def test_metrics_addr_args(mocker, metrics_addr_args, metrics_addr_result): + mocker.patch('ouroboros.cli') + args = cli.parse(metrics_addr_args) + assert args.metrics_addr == metrics_addr_result + + +@pytest.mark.parametrize('metrics_addr_env_var, metrics_addr_env_var_result', [ + ({'METRICS_ADDR': '127.0.0.0'}, '127.0.0.0'), +]) +def test_metrics_addr_env_var(mocker, metrics_addr_env_var, metrics_addr_env_var_result): + mocker.patch.dict('os.environ', metrics_addr_env_var) + mocker.patch('ouroboros.cli') + args = cli.parse([]) + assert args.metrics_addr == metrics_addr_env_var_result + + +# METRICS_PORT +@pytest.mark.parametrize('metrics_port_args, metrics_port_result', [ + (['--metrics-port', '8001'], 8001) +]) +def test_metrics_port_args(mocker, metrics_port_args, metrics_port_result): + mocker.patch('ouroboros.cli') + args = cli.parse(metrics_port_args) + assert args.metrics_port == metrics_port_result + + +@pytest.mark.parametrize('metrics_port_env_var, metrics_port_env_varresult', [ + ({'METRICS_PORT': 'test'}, False), + ({'METRICS_PORT': '8001'}, 8001), +]) +def get_metrics_port_int_env_var(mocker, metrics_port_env_var, metrics_port_env_var_result): + mocker.patch.dict('os.environ', metrics_port_env_var) + assert cli.get_int_env_var(env_var=environ.get('METRICS_PORT')) == metrics_port_env_var_result + + +def test_metrics_port_arg_invalid_value(mocker): + mocker.patch('ouroboros.cli') + with pytest.raises(SystemExit) as pytest_wrapped_e: + cli.parse(['--metrics-port', 'test']) + assert pytest_wrapped_e.type == SystemExit diff --git a/tests/unit/defaults_test.py b/tests/unit/defaults_test.py index e3852494..e758eac9 100644 --- a/tests/unit/defaults_test.py +++ b/tests/unit/defaults_test.py @@ -7,8 +7,9 @@ (defaults.MONITOR, []), (defaults.LOGLEVEL, 'info'), (defaults.RUNONCE, False), - (defaults.CLEANUP, False) + (defaults.CLEANUP, False), + (defaults.METRICS_ADDR, '127.0.0.1'), + (defaults.METRICS_PORT, 8000) ]) - def test_defaults(default, result): assert default == result diff --git a/tests/unit/logger_test.py b/tests/unit/logger_test.py index 7a6dfd15..3dfb219f 100644 --- a/tests/unit/logger_test.py +++ b/tests/unit/logger_test.py @@ -8,9 +8,9 @@ ('error', 40), ('critical', 50), ]) - def test_logger_levels(level_string, level_code): assert log.set_logger(level_string)['level'] == level_code + def test_logger_invalid_level(): - assert log.set_logger('wrong')['level'] == 20 \ No newline at end of file + assert log.set_logger('wrong')['level'] == 20