From 7ef59b59dc3d8884bd91a842cbd9e0efd46c6d35 Mon Sep 17 00:00:00 2001 From: John Walstra <70371225+jsupun@users.noreply.github.com> Date: Wed, 2 Feb 2022 10:01:28 -0600 Subject: [PATCH] Ansible update (#216) Ansible Galaxy The main goal of the changes were to be able to publish to Ansible Galaxy. Most of the magic happens inside of a GitHub action script that moves files into the proper directories and changes some of the module names spaces. The keeper_init action plugin was added to allow the one-time access token to be initialized via a task, since you will not have a the `keeper_ansible` script to init the token if installing from Ansible Galaxy. Included in the collection is an actual role called `keeper_init_token` to initialize a token. This branch also included the ability to turn on caching using the `keeper_use_cache` option. SDK changes were made to allow the cache directory to be changed inside of writing to working directory. Inside of Ansible the option `keeper_cache_dir` will set it. A stdout callback plugin called `keeper_redact` was created in an attempt to hide secrets and keeper config variables from being displayed in the log. It best to just use `no_log: True` which if 100%. This plugin doesn't work for lookup. :( --- .../keeper_secrets_manager_ansible/README.md | 3 + .../keeper_secrets_manager/README.md | 112 ++++++++ .../keeper_secrets_manager/galaxy.yml | 62 +++++ .../keeper_secrets_manager/meta/runtime.yml | 8 + .../roles/keeper_init_token/.travis.yml | 29 +++ .../roles/keeper_init_token/README.md | 41 +++ .../roles/keeper_init_token/defaults/main.yml | 2 + .../roles/keeper_init_token/handlers/main.yml | 2 + .../roles/keeper_init_token/meta/main.yml | 52 ++++ .../roles/keeper_init_token/tasks/main.yml | 13 + .../roles/keeper_init_token/tests/inventory | 2 + .../roles/keeper_init_token/tests/test.yml | 5 + .../roles/keeper_init_token/vars/main.yml | 2 + .../tower_execution_environment/README.md | 23 ++ .../execution-environment.yml | 5 + .../requirements.txt | 1 + .../requirements.yml | 3 + .../__init__.py | 239 ++++++++++++++---- .../plugins/action_plugins/keeper_cleanup.py | 51 ++++ .../plugins/action_plugins/keeper_copy.py | 223 +++++++++++++++- .../plugins/action_plugins/keeper_get.py | 116 ++++++++- .../plugins/action_plugins/keeper_init.py | 175 +++++++++++++ .../plugins/action_plugins/keeper_set.py | 67 ++++- .../plugins/callback_plugins/__init__.py | 0 .../plugins/callback_plugins/keeper_redact.py | 83 ++++++ .../plugins/lookup_plugins/keeper.py | 91 ++++++- .../requirements.txt | 2 +- .../keeper_secrets_manager_ansible/setup.py | 4 +- .../playbooks/keeper_cleanup.yml | 15 ++ .../ansible_example/playbooks/keeper_copy.yml | 8 +- .../ansible_example/playbooks/keeper_init.yml | 12 + .../playbooks/keeper_lookup.yml | 15 ++ .../tests/ansible_test_framework.py | 43 +++- .../tests/keeper_ansible_test.py | 21 ++ .../tests/keeper_cleanup.py | 73 ++++++ .../tests/keeper_copy_test.py | 16 +- .../tests/keeper_get_test.py | 11 +- .../tests/keeper_init.py | 107 ++++++++ .../tests/keeper_lookup_test.py | 27 +- .../tests/keeper_redact.py | 32 +++ .../tests/keeper_set_test.py | 14 +- .../utils/version.sh | 8 + .../core/keeper_secrets_manager_core/core.py | 9 +- sdk/python/core/setup.py | 2 +- 44 files changed, 1692 insertions(+), 137 deletions(-) create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/README.md create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/galaxy.yml create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/meta/runtime.yml create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/.travis.yml create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/README.md create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/defaults/main.yml create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/handlers/main.yml create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/meta/main.yml create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tasks/main.yml create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tests/inventory create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tests/test.yml create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/vars/main.yml create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/README.md create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/execution-environment.yml create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/requirements.txt create mode 100644 integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/requirements.yml create mode 100644 integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/action_plugins/keeper_cleanup.py create mode 100644 integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/action_plugins/keeper_init.py create mode 100644 integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/callback_plugins/__init__.py create mode 100644 integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/callback_plugins/keeper_redact.py create mode 100644 integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_cleanup.yml create mode 100644 integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_init.yml create mode 100644 integration/keeper_secrets_manager_ansible/tests/keeper_cleanup.py create mode 100644 integration/keeper_secrets_manager_ansible/tests/keeper_init.py create mode 100644 integration/keeper_secrets_manager_ansible/tests/keeper_redact.py create mode 100755 integration/keeper_secrets_manager_ansible/utils/version.sh diff --git a/integration/keeper_secrets_manager_ansible/README.md b/integration/keeper_secrets_manager_ansible/README.md index c3d08f44..8feb00f5 100644 --- a/integration/keeper_secrets_manager_ansible/README.md +++ b/integration/keeper_secrets_manager_ansible/README.md @@ -5,6 +5,9 @@ This module contains plugins that allow your Ansible automations to use Keeper S * `keeper_copy` - Similar to `ansible.builtin.copy`. Uses the KSM vault for the source/content. * `keeper_get` - Retrieve secrets from a record. * `keeper_set` - Update an existing record from Ansible information. +* `keeper_init` - Initialize a KSM configuration from a one-time access token. +* `keeper_cleanup` - Remove the cache file, if being used. * `keeper_lookup` - Retrieve secrets from a record using Ansible's lookup. +* `keeper_redact` - Stdout Callback plugin to redact secrets from logs. For more information see our official documentation page https://docs.keeper.io/secrets-manager/secrets-manager/integrations/ansible-plugin diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/README.md b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/README.md new file mode 100644 index 00000000..b152097d --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/README.md @@ -0,0 +1,112 @@ +![Ansible](https://github.com/Keeper-Security/secrets-manager/actions/workflows/test.ansible.yml/badge.svg) + +# Keeper Secrets Manager Collection + +This collection allows you retrieve and update records in your Keeper Vault. + +Additional documentation can be found on the [Keeper Secrets Manager Ansible](https://docs.keeper.io/secrets-manager/secrets-manager/integrations/ansible-plugin) +document portal. + +# Installation + +## Ansible Tower + +In your playbook's source repository, add `keepersecrity.keeper_secrets_manager` to the +`requirement.yml` collections list. + +There is an **Execution Environment** docker image location at +[https://hub.docker.com/repository/docker/keeper/keeper-secrets-manager-tower-ee](https://hub.docker.com/repository/docker/keeper/keeper-secrets-manager-tower-ee). +This **Execution Environment** contains the Python SDK. + +## Command Line + +This collection requires the [keeper-secrets-manager-core](https://pypi.org/project/keeper-secrets-manager-core/) +Python SDK. Use `pip` to install this module into the modules used by your installation of Ansible. + +```shell +$ pip3 install -U keeper-secrets-manager-core +``` +Then install the collection. + +```shell +$ ansible-galaxy collection install keepersecrity.keeper_secrets_manager +``` + +# Plugins + +If you wish, you can set the collections in your task and +just used the short name (ie keeper_copy) + +```yaml +- name: Keeper Task + collections: + - keepersecurity.keeper_secrets_manager + + tasks: + - name: "Copy My SSH Keys" + keeper_copy: + notation: "OlLZ6JLjnyMOS3CiIPHBjw/field/keyPair[{{ item.notation_key }}]" + dest: "/home/user/.ssh/{{ item.filename }}" + mode: "0600" + loop: + - { notation_key: "privateKey", filename: "id_rsa" } + - { notation_key: "publicKey", filename: "id_rsa.pub" } +``` +If you omit the `collections` , you will need to use the full plugin name. +```yaml + tasks: + - name: "Copy My SSH Keys" + keepersecurity.keeper_secrets_manager.keeper_copy: + notation: "OlLZ6JLjnyMOS3CiIPHBjw/field/keyPair[{{ item.notation_key }}]" +``` + +## Action + +* `keepersecurity.keeper_secrets_manager.keeper_copy` - Copy file, or value, from your vault to a remote server. +* `keepersecurity.keeper_secrets_manager.keeper_get` - Get a value from your vault. +* `keepersecurity.keeper_secrets_manager.keeper_set` - Set a value of an existing record in your vault. +* `keepersecurity.keeper_secrets_manager.keeper_cleanup` - Clean up Keeper related files. +* `keepersecurity.keeper_secrets_manager.keeper_init` - Init a one-time access token. Returns a configuration. + +## Lookup + +* `keepersecurity.keeper_secrets_manager.keeper` - Get a value from your vault via a lookup. + +## Callback + +* `keepersecurity.keeper_secrets_manager.keeper_redact` - Stdout callback plugin to redact secret values. + +## keeper_init_token Role + +Initializing a configuration from a one-time access token. Getting the +token is explained in the +[One Time Access Token](https://docs.keeper.io/secrets-manager/secrets-manager/about/one-time-token) document. + +Then create a simple playbook to initialize the token. + +```yaml +- name: Initialize the Keeper one time access token. + hosts: localhost + connection: local + collections: keepersecurity.keeper_secrets_manager + + roles: + - keeper_init_token +``` +Then run the playbook. Pass the token in using the extra var param (-e). +```shell +$ ansible-playbook keeper_init.yml -e keeper_token=US:XXX -e keeper_config_file=keeper-config.yml +``` +When done there will be a file called `keeper-config.yml` which will contain the configuration +for your device. + +```yaml +keeper_app_key: +U5Jao ... l5FmXymVI= +keeper_client_id: Fokc6j ... PlBwzAKlMUgFZHqLg== +keeper_hostname: US +keeper_private_key: MIGHf ... IcvCihUHyA7Oy +keeper_server_public_key_id: '10' +``` +The content of this YAML file can then be cut-n-pasted into a **group_vars**, **host_vars**, **all** +configuration file or even a playbook. + diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/galaxy.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/galaxy.yml new file mode 100644 index 00000000..c59788fd --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/galaxy.yml @@ -0,0 +1,62 @@ +### REQUIRED +# The namespace of the collection. This can be a company/brand/organization or product namespace under which all +# content lives. May only contain alphanumeric lowercase characters and underscores. Namespaces cannot start with +# underscores or numbers and cannot contain consecutive underscores +namespace: keepersecurity + +# The name of the collection. Has the same character restrictions as 'namespace' +name: keeper_secrets_manager + +# The version of the collection. Must be compatible with semantic versioning +version: VERSION_PLACEHOLDER + +# The path to the Markdown (.md) readme file. This path is relative to the root of the collection +readme: README.md + +# A list of the collection's content authors. Can be just the name or in the format 'Full Name (url) +# @nicks:irc/im.site#channel' +authors: +- John Walstra + + +### OPTIONAL but strongly recommended +# A short summary description of the collection +description: Retrieve secrets from your Keeper Vault. + +# Either a single license or a list of licenses for content inside of a collection. Ansible Galaxy currently only +# accepts L(SPDX,https://spdx.org/licenses/) licenses. This key is mutually exclusive with 'license_file' +license: +- MIT + +# The path to the license file for the collection. This path is relative to the root of the collection. This key is +# mutually exclusive with 'license' +license_file: '' + +# A list of tags you want to associate with the collection for indexing/searching. A tag name has the same character +# requirements as 'namespace' and 'name' +tags: ["secret", "password", "vault"] + +# Collections that this collection requires to be installed for it to be usable. The key of the dict is the +# collection label 'namespace.name'. The value is a version range +# L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version +# range specifiers can be set and are separated by ',' +dependencies: {} + +# The URL of the originating SCM repository +repository: https://github.com/Keeper-Security/secrets-manager + +# The URL to any online docs +documentation: https://docs.keeper.io/secrets-manager/secrets-manager/integrations/ansible-plugin + +# The URL to the homepage of the collection/project +homepage: https://www.keepersecurity.com + +# The URL to the collection issue tracker +issues: https://github.com/Keeper-Security/secrets-manager/issues + +# A list of file glob-like patterns used to filter any files or directories that should not be included in the build +# artifact. A pattern is matched from the relative path of the file or directory of the collection directory. This +# uses 'fnmatch' to match the files or directories. Some directories and files like 'galaxy.yml', '*.pyc', '*.retry', +# and '.git' are always filtered +build_ignore: [] + diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/meta/runtime.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/meta/runtime.yml new file mode 100644 index 00000000..6df1b489 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/meta/runtime.yml @@ -0,0 +1,8 @@ +--- +requires_ansible: '>=2.10' +action_groups: + keeper_secrets_manager: + - keeper_copy + - keeper_get + - keeper_set + - keeper_cleanup diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/.travis.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/.travis.yml new file mode 100644 index 00000000..36bbf620 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/.travis.yml @@ -0,0 +1,29 @@ +--- +language: python +python: "2.7" + +# Use the new container infrastructure +sudo: false + +# Install ansible +addons: + apt: + packages: + - python-pip + +install: + # Install ansible + - pip install ansible + + # Check ansible version + - ansible --version + + # Create ansible.cfg with correct roles_path + - printf '[defaults]\nroles_path=../' >ansible.cfg + +script: + # Basic role syntax check + - ansible-playbook tests/test.yml -i tests/inventory --syntax-check + +notifications: + webhooks: https://galaxy.ansible.com/api/v1/notifications/ \ No newline at end of file diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/README.md b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/README.md new file mode 100644 index 00000000..3ddd205f --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/README.md @@ -0,0 +1,41 @@ +Role Name +========= + +Initialize a one-time access token into a configuration. + +Requirements +------------ + +You will need a Keeper Vault account with Secrets Manager enabled. Follow the +[Quick Start Guide](https://docs.keeper.io/secrets-manager/secrets-manager/quick-start-guide) +to get a One-time Access Token. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +```yaml +- hosts: servers + roles: + - name: Init Token +``` + + +License +------- + +MIT + +Author Information +------------------ + +John Walstra \ No newline at end of file diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/defaults/main.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/defaults/main.yml new file mode 100644 index 00000000..fa995c7a --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for keeper_init_token diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/handlers/main.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/handlers/main.yml new file mode 100644 index 00000000..0853ddfd --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for keeper_init_token diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/meta/main.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/meta/main.yml new file mode 100644 index 00000000..4a926c9d --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/meta/main.yml @@ -0,0 +1,52 @@ +galaxy_info: + author: John Walstra + description: Initialize a configuration from a one-time access token. + company: Keeper Security + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: MIT + + min_ansible_version: 2.10 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: ['secret', 'password', 'vault'] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tasks/main.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tasks/main.yml new file mode 100644 index 00000000..f26e4622 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tasks/main.yml @@ -0,0 +1,13 @@ +--- +# tasks file for keeper_init_token + +- name: Check if keeper_token has been set. + fail: + msg: "The keeper_token has not been set. Use '-e keeper_token=XX:XXXX' to pass in the one time access token." + when: keeper_token is undefined + +- name: Init the one-time access token. + keepersecurity.keeper_secrets_manager.keeper_init: + token: "{{ keeper_token }}" + filename: "{{ keeper_config_file | default('') }}" + show_config: "{{ keeper_show_config | default(False) }}" \ No newline at end of file diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tests/inventory b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tests/inventory new file mode 100644 index 00000000..878877b0 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tests/test.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tests/test.yml new file mode 100644 index 00000000..8306670e --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - keeper_init_token diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/vars/main.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/vars/main.yml new file mode 100644 index 00000000..ceccaddf --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/keepersecurity/keeper_secrets_manager/roles/keeper_init_token/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for keeper_init_token diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/README.md b/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/README.md new file mode 100644 index 00000000..15187f29 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/README.md @@ -0,0 +1,23 @@ +# What is this? + +Tower/AWX uses an Execution Environment. This is a Docker container used to run +the playbooks. This is where a playbook gets its Python interpreter and modules. +This is where the KSM Python SDK is installed. + +First install the Ansible EE builder. +```shell +pip install ansible-builder +``` +Build the Docker image +```shell +$ ansible-builder build \ + --tag docker.io/keeper/keeper-secrets-manager-tower-ee:latest \ + --context ./context \ + --container-runtime docker +``` +Then push to Docker Hub + +```shell +$ docker push docker.io/keeper/keeper-secrets-manager-tower-ee:latest +``` + diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/execution-environment.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/execution-environment.yml new file mode 100644 index 00000000..5aa14625 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/execution-environment.yml @@ -0,0 +1,5 @@ +--- +version: 1 +dependencies: + galaxy: requirements.yml + python: requirements.txt diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/requirements.txt b/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/requirements.txt new file mode 100644 index 00000000..d71438e2 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/requirements.txt @@ -0,0 +1 @@ +keeper-secrets-manager-core diff --git a/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/requirements.yml b/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/requirements.yml new file mode 100644 index 00000000..53e2c64f --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/ansible_galaxy/tower_execution_environment/requirements.yml @@ -0,0 +1,3 @@ +--- +collections: + - name: keepersecurity.keeper_secrets_manager diff --git a/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/__init__.py b/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/__init__.py index 9fa20a42..97b38c5c 100644 --- a/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/__init__.py +++ b/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/__init__.py @@ -9,16 +9,29 @@ # Contact: ops@keepersecurity.com # -from keeper_secrets_manager_core import SecretsManager -from keeper_secrets_manager_core.storage import FileKeyValueStorage, InMemoryKeyValueStorage -from keeper_secrets_manager_core.configkeys import ConfigKeys from ansible.utils.display import Display from ansible.errors import AnsibleError +from ansible.module_utils.basic import missing_required_lib +from ansible.module_utils.common.text.converters import jsonify +from distutils.util import strtobool import os +import sys +import re import json from re import sub from enum import Enum - +import traceback + +# Check if the KSM SDK core has been installed +KSM_SDK_ERR = None +try: + import keeper_secrets_manager_core +except ImportError: + KSM_SDK_ERR = traceback.format_exc() +else: + from keeper_secrets_manager_core import SecretsManager + from keeper_secrets_manager_core.core import KSMCache + from keeper_secrets_manager_core.storage import FileKeyValueStorage, InMemoryKeyValueStorage display = Display() @@ -37,34 +50,63 @@ def get_enum(value): class KeeperAnsible: - """ A class containing common method used by the Ansible plugin and also talked to Keeper Python SDK """ KEY_PREFIX = "keeper" KEY_CONFIG_FILE_SUFFIX = "config_file" + KEY_CONFIG_BASE64 = "config" ALLOWED_FIELDS = ["field", "custom_field", "file"] TOKEN_ENV = "KSM_TOKEN" TOKEN_KEY = "token" + HOSTNAME_KEY = "hostname" CONFIG_CLIENT_KEY = "clientKey" FORCE_CONFIG_FILE = "force_config_write" KEY_SSL_VERIFY_SKIP = "verify_ssl_certs_skip" KEY_LOG_LEVEL = "log_level" + KEY_USE_CACHE = "use_cache" + KEY_CACHE_DIR = "cache_dir" + ENV_CACHE_DIR = "KSM_CACHE_DIR" DEFAULT_LOG_LEVEL = "ERROR" + REDACT_MODULE_MATCH = r"\.keeper_redact$" @staticmethod def get_client(**kwargs): return SecretsManager(**kwargs) - def __init__(self, task_vars): + @staticmethod + def keeper_key(key): + return "{}_{}".format(KeeperAnsible.KEY_PREFIX, key) + + @staticmethod + def fail_json(msg, **kwargs): + kwargs['failed'] = True + kwargs['msg'] = msg + print('\n%s' % jsonify(kwargs)) + sys.exit(0) + + def __init__(self, task_vars, force_in_memory=False): """ Build the config used by the Keeper Python SDK The configuration is mainly read from a JSON file. """ + if KSM_SDK_ERR is not None: + self.fail_json(msg=missing_required_lib('keeper-secrets-manager-core'), exception=KSM_SDK_ERR) + self.config_file = None self.config_created = False + self.using_cache = False + + # Check if we have the keeper redact callback stdout plugin is enabled. + self.has_redact = False + for module in sys.modules: + if re.search(KeeperAnsible.REDACT_MODULE_MATCH, module) is not None: + self.has_redact = True + break + + self.secret_values = [] def camel_case(text): text = sub(r"([_\-])+", " ", text).title().replace(" ", "") @@ -73,11 +115,7 @@ def camel_case(text): try: # Match the SDK log level to Ansible log level log_level_key = KeeperAnsible.keeper_key(KeeperAnsible.KEY_LOG_LEVEL) - log_level = KeeperAnsible.DEFAULT_LOG_LEVEL - - # If the log level is in the vars then use that value - if log_level_key in task_vars: - log_level = task_vars[log_level] + log_level = task_vars.get(log_level_key, KeeperAnsible.DEFAULT_LOG_LEVEL) # Else try is give logging level based on the Ansible display level if display.verbosity == 1: @@ -98,55 +136,90 @@ def camel_case(text): if self.config_file is None: self.config_file = FileKeyValueStorage.default_config_file_location - if os.path.isfile(self.config_file) is True: - display.debug("Loading keeper config file file {}.".format(self.config_file)) + # Should we be using the cache? + use_cache_key = KeeperAnsible.keeper_key(KeeperAnsible.KEY_USE_CACHE) + custom_post_function = None + if bool(strtobool(str(task_vars.get(use_cache_key, "False")))) is True: + custom_post_function = KSMCache.caching_post_function + + # We are using the cache, what directory should the cache file be stored in. + cache_dir_key = KeeperAnsible.keeper_key(KeeperAnsible.KEY_CACHE_DIR) + if task_vars.get(cache_dir_key) is not None and os.environ.get(KeeperAnsible.ENV_CACHE_DIR) is None: + os.environ[KeeperAnsible.ENV_CACHE_DIR] = task_vars.get(cache_dir_key) + + display.vvv("Keeper Secrets Manager is using cache. Cache directory is {}.".format( + os.environ.get(KeeperAnsible.ENV_CACHE_DIR) + if os.environ.get(KeeperAnsible.ENV_CACHE_DIR) is not None else "current working directory")) + + self.using_cache = True + else: + display.vvv("Keeper Secrets Manager is not using a cache.") + + if os.path.isfile(self.config_file) is True and force_in_memory is False: + display.vvv("Loading keeper config file file {}.".format(self.config_file)) self.client = KeeperAnsible.get_client( config=FileKeyValueStorage(config_file_location=self.config_file), - log_level=log_level + log_level=log_level, + custom_post_function=custom_post_function ) # Else config values in the Ansible variable. else: - display.debug("Loading keeper config from Ansible vars.") - - config_dict = {} - # Convert Ansible variables into the keys used by Secrets Manager's config. - for key in ["url", "client_id", "client_key", "app_key", "private_key", "bat", "binding_key", - "hostname"]: - keeper_key = KeeperAnsible.keeper_key(key) - camel_key = camel_case(key) - if keeper_key in task_vars: - config_dict[camel_key] = task_vars[keeper_key] - - # token is the odd ball. we need it to be client key in the SDK config. - token_key = KeeperAnsible.keeper_key(KeeperAnsible.TOKEN_KEY) - if token_key in task_vars: - config_dict[KeeperAnsible.CONFIG_CLIENT_KEY] = task_vars[token_key] - - # If the secret client key is in the environment, override the Ansible var. - if os.environ.get(KeeperAnsible.TOKEN_ENV) is not None: - config_dict[KeeperAnsible.CONFIG_CLIENT_KEY] = os.environ.get(KeeperAnsible.TOKEN_ENV) - elif token_key in task_vars: - config_dict[KeeperAnsible.CONFIG_CLIENT_KEY] = task_vars[token_key] - - # If no variables were passed in throw an error. - if len(config_dict) == 0: - raise AnsibleError("There is no config file and the Ansible variable contain no config keys. Will" - " not be able to connect to the Keeper server.") + display.vvv("Loading keeper config from Ansible vars.") # Since we are getting our variables from Ansible, we want to default using the in memory storage so # not to leave config files laying around. in_memory_storage = True - # Does the user want to write the config to a file? Then don't use the in memory storage. - if bool(task_vars.get(KeeperAnsible.keeper_key(KeeperAnsible.FORCE_CONFIG_FILE), False)) is True: - in_memory_storage = False - # If the is only 1 key, we want to force the config to write to the file. - elif len(config_dict) == 1 and KeeperAnsible.CONFIG_CLIENT_KEY in config_dict: - in_memory_storage = False + # If be have parameter with a Base64 config, use it for the config_option and force + # the config to be in memory. + base64_key = KeeperAnsible.keeper_key(KeeperAnsible.KEY_CONFIG_BASE64) + if base64_key in task_vars: + config_option = task_vars.get(base64_key) + force_in_memory = True + # Else try to discover the config values. + else: + + # Config is not a Base64 string, make a dictionary to hold config values. + config_option = {} + # Convert Ansible variables into the keys used by Secrets Manager's config. + for key in ["url", "client_id", "client_key", "app_key", "private_key", "bat", "binding_key", + "hostname"]: + keeper_key = KeeperAnsible.keeper_key(key) + camel_key = camel_case(key) + if keeper_key in task_vars: + config_option[camel_key] = task_vars[keeper_key] + + # Token is the odd ball. we need it to be client key in the SDK config. SDK will remove it + # when it is done. + token_key = KeeperAnsible.keeper_key(KeeperAnsible.TOKEN_KEY) + if token_key in task_vars: + config_option[KeeperAnsible.CONFIG_CLIENT_KEY] = task_vars[token_key] + + # If the secret client key is in the environment, override the Ansible var. + if os.environ.get(KeeperAnsible.TOKEN_ENV) is not None: + config_option[KeeperAnsible.CONFIG_CLIENT_KEY] = os.environ.get(KeeperAnsible.TOKEN_ENV) + elif token_key in task_vars: + config_option[KeeperAnsible.CONFIG_CLIENT_KEY] = task_vars[token_key] + + # If no variables were passed in throw an error. + if len(config_option) == 0: + raise AnsibleError("There is no config file and the Ansible variable contain no config keys." + " Will not be able to connect to the Keeper server.") + + # Does the user want to write the config to a file? Then don't use the in memory storage. + if bool(task_vars.get(KeeperAnsible.keeper_key(KeeperAnsible.FORCE_CONFIG_FILE), False)) is True: + in_memory_storage = False + # If the is only 1 key, we want to force the config to write to the file. + elif len(config_option) == 1 and KeeperAnsible.CONFIG_CLIENT_KEY in config_option: + in_memory_storage = False + + # Sometime we don't want a JSON file, ever. Force the config to be in memory. + if force_in_memory is True: + in_memory_storage = True if in_memory_storage is True: - config_instance = InMemoryKeyValueStorage(config=config_dict) + config_instance = InMemoryKeyValueStorage(config=config_option) else: if self.config_file is None: self.config_file = FileKeyValueStorage.default_config_file_location @@ -154,9 +227,10 @@ def camel_case(text): elif os.path.isfile(self.config_file) is False: self.config_created = True - # Write the variables we have to a JSON file. + # Write the variables we have to a JSON file. If we are in here config_option is a dictionary, + # not a Base64 string. with open(self.config_file, "w") as fh: - json.dump(config_dict, fh, indent=4) + json.dump(config_option, fh, indent=4) fh.close() config_instance = FileKeyValueStorage(config_file_location=self.config_file) @@ -165,16 +239,13 @@ def camel_case(text): self.client = KeeperAnsible.get_client( config=config_instance, verify_ssl_certs=not ssl_certs_skip, - log_level=log_level + log_level=log_level, + custom_post_function=custom_post_function ) except Exception as err: raise AnsibleError("Keeper Ansible error: {}".format(err)) - @staticmethod - def keeper_key(key): - return "{}_{}".format(KeeperAnsible.KEY_PREFIX, key) - def get_record(self, uid): try: @@ -186,10 +257,44 @@ def get_record(self, uid): return records[0] + @staticmethod + def _gather_secrets(obj): + """ Walk the secret structure and get values. These should just be str, list, and dict. Warn if the SDK + return something different. + """ + result = [] + if type(obj) is str: + result.append(obj) + elif type(obj) is list: + for item in obj: + result += KeeperAnsible._gather_secrets(item) + elif type(obj) is dict: + for k, v in obj.items(): + result += KeeperAnsible._gather_secrets(v) + else: + display.warning("Result item is not string, list, or dictionary, can't get secret values: " + + str(type(obj))) + return result + + def stash_secret_value(self, value): + """ Parse the result of the secret retrieval and add values to list of secret values. + """ + for secret_value in self._gather_secrets(value): + if secret_value not in self.secret_values: + self.secret_values.append(secret_value) + + def get_value_via_notation(self, notation): + value = self.client.get_notation(notation) + self.stash_secret_value(value) + return value + def get_value(self, uid, field_type, key, allow_array=False): record = self.get_record(uid) + # Make sure the boolean is a boolean. + allow_array = bool(strtobool(str(allow_array))) + values = None if field_type == KeeperFieldType.FIELD: values = record.field(key) @@ -211,6 +316,8 @@ def get_value(self, uid, field_type, key, allow_array=False): uid, field_type.name, key)) return None + self.stash_secret_value(values) + # If we want the entire array, then just return what we got from the field. if allow_array is True: return values @@ -236,7 +343,7 @@ def set_value(self, uid, field_type, key, value): @staticmethod def get_field_type_enum_and_key(args): - """Get the field type enum and field key in the Ansible args for a task. + """ Get the field type enum and field key in the Ansible args for a task. For a task that, only allowed one of the allowed field, this method will find the type of field and the key/label for that field. @@ -262,3 +369,25 @@ def get_field_type_enum_and_key(args): "custom_field or file.") return KeeperFieldType.get_enum(field_type[0]), field_key + + def add_secret_values_to_results(self, results): + """ If the redact stdout callback is being used, add the secrets to the results dictionary. The redact + stdout callback will remove it from the results. It will use value to remove values from stdout. + """ + + # If we are using the redact stdout callback, add the secrets we retrieve to the special key. The redact + # stdout callback will make sure the value is not in the stdout. + if self.has_redact is True: + results["_secrets"] = self.secret_values + return results + + def cleanup(self): + + status = {} + + # If we are using the cache, remove the cache file. + if self.using_cache is True: + KSMCache.remove_cache_file() + status["removed_ksm_cache"] = True + + return status diff --git a/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/action_plugins/keeper_cleanup.py b/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/action_plugins/keeper_cleanup.py new file mode 100644 index 00000000..f3e7196e --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/action_plugins/keeper_cleanup.py @@ -0,0 +1,51 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | ' 0: + reg_exp_items = [] + # Sort secret from longest to shortest + sorted_secrets = sorted(secrets, key=len, reverse=True) + for item in sorted_secrets: + # Escape an regular expression characters. Item will be ansible.utils.unsafe_proxy.AnsibleUnsafeText, + # which we will just convert to a str. + reg_exp_items.append(re.escape(str(item))) + redact_regexp = "|".join(reg_exp_items) + + json_result = json.dumps(clean_result, indent=4) + if redact_regexp is not None: + json_result = re.sub(redact_regexp, '****', json_result, re.MULTILINE) + + return json_result diff --git a/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/lookup_plugins/keeper.py b/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/lookup_plugins/keeper.py index bd22c95d..cb7da151 100644 --- a/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/lookup_plugins/keeper.py +++ b/integration/keeper_secrets_manager_ansible/keeper_secrets_manager_ansible/plugins/lookup_plugins/keeper.py @@ -13,6 +13,76 @@ from ansible.errors import AnsibleError from ansible.plugins.lookup import LookupBase +DOCUMENTATION = r''' +--- +module: keeper_get + +short_description: Get value(s) from the Keeper Vault + +version_added: "1.0.0" + +description: + - Copy a value from the Keeper Vault into a variable. + - If value is not a literal value, the structure will be retrieved. +author: + - John Walstra +options: + uid: + description: + - The UID of the Keeper Vault record. + type: str + required: no + field: + description: + - The label, or type, of the standard field in record that contains the value. + - If the value has a complex value, use notation to get the specific value from the complex value. + type: str + required: no + custom_field: + description: + - The label, or type, of the user added customer field in record that contains the value. + - If the value has a complex value, use notation to get the specific value from the complex value. + type: str + required: no + allow_array: + description: + - Allow array of values instead of taking the first value. + - If enabled, the value will be returned all the values for a field. + - This does not work with notation since notation defines if an array is returned. + type: bool + default: no + required: no + version_added: '1.0.1' + notation: + description: + - The Keeper notation to access record that contains the value. + - Use notation when you want a specific value. + - + - See https://docs.keeper.io/secrets-manager/secrets-manager/about/keeper-notation for more information/ + type: str + required: no + version_added: '1.0.1' +''' + +EXAMPLES = r''' +- name: Get login name + debug: + msg: "{{ lookup('keeper', uid='XXX', field='login') }} +- name: Get all phone numbers + debug: + msg: "{{ lookup('keeper', uid='XXX', custom_field='phone', allow_array='True') }} +- name: Get all phone numbers via notation + debug: + msg: "{{ lookup('keeper', notation='XXX/custom_field/phone') }} +''' + +RETURN = ''' + _list: + description: list of list of lines or content of record field(s) + type: list + elements: str +''' + class LookupModule(LookupBase): @@ -20,13 +90,20 @@ def run(self, terms, variables=None, **kwargs): keeper = KeeperAnsible(task_vars=variables) - uid = kwargs.get("uid", None) - if uid is None: - raise AnsibleError("The uid is blank. keeper lookup requires this value to be set.") + if kwargs.get("notation") is not None: + value = keeper.get_value_via_notation(kwargs.get("notation")) + else: + uid = kwargs.get("uid") + if uid is None: + raise AnsibleError("The uid is blank. keeper_get requires this value to be set.") + + # Try to get either the field, custom_field, or file name. + field_type_enum, field_key = keeper.get_field_type_enum_and_key(args=kwargs) - # Try to get either the field, custom_field, or file name. - field_type_enum, field_key = keeper.get_field_type_enum_and_key(args=kwargs) + allow_array = kwargs.get("allow_array", False) + value = keeper.get_value(uid, field_type=field_type_enum, key=field_key, allow_array=allow_array) - value = keeper.get_value(uid, field_type=field_type_enum, key=field_key) + if type(value) is not list: + value = [value] - return [value] + return value diff --git a/integration/keeper_secrets_manager_ansible/requirements.txt b/integration/keeper_secrets_manager_ansible/requirements.txt index f4b5f6fd..21356fc6 100644 --- a/integration/keeper_secrets_manager_ansible/requirements.txt +++ b/integration/keeper_secrets_manager_ansible/requirements.txt @@ -1,3 +1,3 @@ ansible importlib_metadata -keeper-secrets-manager-core>=16.1.7 \ No newline at end of file +keeper-secrets-manager-core>=16.2.0 \ No newline at end of file diff --git a/integration/keeper_secrets_manager_ansible/setup.py b/integration/keeper_secrets_manager_ansible/setup.py index 8fd0aba9..d06f4ca0 100644 --- a/integration/keeper_secrets_manager_ansible/setup.py +++ b/integration/keeper_secrets_manager_ansible/setup.py @@ -9,14 +9,14 @@ long_description = fp.read() install_requires = [ - 'keeper-secrets-manager-core', + 'keeper-secrets-manager-core>=16.2.0', 'importlib_metadata', 'ansible' ] setup( name="keeper-secrets-manager-ansible", - version='1.0.0', + version='1.1.0', description="Keeper Secrets Manager plugins for Ansible.", long_description=long_description, long_description_content_type="text/markdown", diff --git a/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_cleanup.yml b/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_cleanup.yml new file mode 100644 index 00000000..81580129 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_cleanup.yml @@ -0,0 +1,15 @@ +# vim: set shiftwidth=2 tabstop=2 softtabstop=-1 expandtab: +--- +- name: Keeper Get + hosts: "my_systems" + vars: + keeper_use_cache: True + + tasks: + - name: "Get Value" + keeper_get: + uid: "{{ uid }}" + field: "password" + + - name: Clean up all. + keeper_cleanup: diff --git a/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_copy.yml b/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_copy.yml index 774d6acd..a016a901 100644 --- a/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_copy.yml +++ b/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_copy.yml @@ -16,4 +16,10 @@ uid: "{{ file_uid }}" file: "{{ file_name }}" dest: "{{ tmp_dir }}/video.mp4" - mode: '0777' \ No newline at end of file + mode: '0777' + + - name: "Copy the login" + keeper_copy: + notation: "{{ password_uid }}/field/login" + dest: "{{ tmp_dir }}/login" + mode: '0600' \ No newline at end of file diff --git a/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_init.yml b/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_init.yml new file mode 100644 index 00000000..7829f565 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_init.yml @@ -0,0 +1,12 @@ +# vim: set shiftwidth=2 tabstop=2 softtabstop=-1 expandtab: +--- +- name: Keeper Get + hosts: "my_systems" + + tasks: + + - name: Init the one-time access token. + keeper_init: + token: "{{ keeper_token }}" + filename: "{{ keeper_config_file }}" + show_config: "{{ show_config }}" \ No newline at end of file diff --git a/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_lookup.yml b/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_lookup.yml index 5fab3119..35018e67 100644 --- a/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_lookup.yml +++ b/integration/keeper_secrets_manager_ansible/tests/ansible_example/playbooks/keeper_lookup.yml @@ -7,4 +7,19 @@ - name: "Print Password" debug: msg: "My password is {{ lookup('keeper', uid='TRd_567FkHy-CeGsAzs8aA', field='Password') }}" + verbosity: 0 + + - name: "Print Login via Notation" + debug: + msg: "My login is {{ lookup('keeper', notation='TRd_567FkHy-CeGsAzs8aA/field/login') }}" + verbosity: 0 + + - name: "Print Phone" + debug: + msg: "My phone_1 is {{ lookup('keeper', uid='TRd_567FkHy-CeGsAzs8aA', field='phone') }}" + verbosity: 0 + + - name: "Print Phone Full Array" + debug: + msg: "My phone_2 is {{ lookup('keeper', uid='TRd_567FkHy-CeGsAzs8aA', field='phone', allow_array='True') }}" verbosity: 0 \ No newline at end of file diff --git a/integration/keeper_secrets_manager_ansible/tests/ansible_test_framework.py b/integration/keeper_secrets_manager_ansible/tests/ansible_test_framework.py index 694692b0..d4f0c08d 100644 --- a/integration/keeper_secrets_manager_ansible/tests/ansible_test_framework.py +++ b/integration/keeper_secrets_manager_ansible/tests/ansible_test_framework.py @@ -125,23 +125,38 @@ class RecordMaker: secret = b"11111111111111111111111111111111" @staticmethod - def make_record(uid, title, value=None): - - if value is None: - value = uid + def make_record(uid, title, record_type=None, fields=None, custom_fields=None): + + if record_type is None: + record_type = "login" + + data = { + "title": title, + "type": record_type + } + if fields is not None: + data["fields"] = [] + for field_type, value in fields.items(): + if type(value) is not list: + value = [value] + data["fields"].append({ + "type": field_type, + "value": value + }) + if custom_fields is not None: + data["custom"] = [] + for field_type, value in custom_fields.items(): + if type(value) is not list: + value = [value] + data["fields"].append({ + "label": field_type, + "type": field_type, + "value": value + }) return Record({ "recordUid": uid, - "data": CryptoUtils.encrypt_aes(json.dumps({ - "title": title, - "type": "login", - "fields": [ - {"type": "login", "value": ["login_{}".format(value)]}, - {"type": "password", "value": ["password_{}".format(value)]}, - {"type": "url", "value": ["https://{}".format(value)]}, - {"type": "fileRef", "value": []} - ] - }).encode(), RecordMaker.secret) + "data": CryptoUtils.encrypt_aes(json.dumps(data).encode(), RecordMaker.secret) }, RecordMaker.secret) @staticmethod diff --git a/integration/keeper_secrets_manager_ansible/tests/keeper_ansible_test.py b/integration/keeper_secrets_manager_ansible/tests/keeper_ansible_test.py index 852d96fc..cec4f894 100644 --- a/integration/keeper_secrets_manager_ansible/tests/keeper_ansible_test.py +++ b/integration/keeper_secrets_manager_ansible/tests/keeper_ansible_test.py @@ -7,6 +7,7 @@ from keeper_secrets_manager_ansible import KeeperAnsible from keeper_secrets_manager_ansible.__main__ import main from keeper_secrets_manager_core.mock import MockConfig +from keeper_secrets_manager_core.configkeys import ConfigKeys import io from contextlib import redirect_stdout @@ -74,6 +75,26 @@ def test_config_in_ansible_task_vars(self, mock_get_secrets): ka.client.get_secrets() mock_get_secrets.assert_called_once() + @patch("keeper_secrets_manager_core.core.SecretsManager.get_secrets", side_effect=get_secrets) + def test_config_base_64(self, mock_get_secrets): + + values = MockConfig.make_config() + base64_config = MockConfig.make_base64(config=values) + + task_vars = { + "keeper_config": base64_config + } + + ka = KeeperAnsible(task_vars=task_vars) + ka.client.get_secrets() + mock_get_secrets.assert_called_once() + self.assertEqual(values.get("clientId"), ka.client.config.get(ConfigKeys.KEY_CLIENT_ID), + "base64 client ids are not the same") + self.assertEqual(values.get("appKey"), ka.client.config.get(ConfigKeys.KEY_APP_KEY), + "base64 app key are not the same") + self.assertEqual(values.get("privateKey"), ka.client.config.get(ConfigKeys.KEY_PRIVATE_KEY), + "base64 private key are not the same") + @patch("keeper_secrets_manager_core.core.SecretsManager.get_secrets", side_effect=get_secrets) def test_ansible_cli_init(self, _): diff --git a/integration/keeper_secrets_manager_ansible/tests/keeper_cleanup.py b/integration/keeper_secrets_manager_ansible/tests/keeper_cleanup.py new file mode 100644 index 00000000..65d5ad60 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/tests/keeper_cleanup.py @@ -0,0 +1,73 @@ +import unittest +from unittest.mock import patch +import os +from .ansible_test_framework import AnsibleTestFramework, RecordMaker +import keeper_secrets_manager_ansible.plugins +import tempfile + + +records = { + "TRd_567FkHy-CeGsAzs8aA": RecordMaker.make_record( + uid="TRd_567FkHy-CeGsAzs8aA", + title="JW-F1-R1", + fields={ + "password": "ddd" + } + ), + "A_7YpGBUgRTeDEQLhVRo0Q": RecordMaker.make_file( + uid="A_7YpGBUgRTeDEQLhVRo0Q", + title="JW-F1-R2-File", + files=[ + {"name": "nailing it.mp4", "type": "video/mp4", "url": "http://localhost/abc", "data": "ABC123"}, + {"name": "video_file.mp4", "type": "video/mp4", "url": "http://localhost/xzy", "data": "XYZ123"}, + ] + ) +} + + +def mocked_get_secrets(*args): + + if len(args) > 0: + uid = args[0][0] + ret = [records[uid]] + else: + ret = [records[x] for x in records] + return ret + + +class KeeperCleanupTest(unittest.TestCase): + + def setUp(self): + + # Add in addition Python libs. This includes the base + # module for Keeper Ansible and the Keeper SDK. + self.base_dir = os.path.dirname(os.path.realpath(__file__)) + self.ansible_base_dir = os.path.join(self.base_dir, "ansible_example") + + def _common(self): + with tempfile.TemporaryDirectory() as temp_dir: + a = AnsibleTestFramework( + base_dir=self.ansible_base_dir, + playbook=os.path.join("playbooks", "keeper_cleanup.yml"), + inventory=os.path.join("inventory", "all"), + plugin_base_dir=os.path.join(os.path.dirname(keeper_secrets_manager_ansible.plugins.__file__)), + vars={ + "tmp_dir": temp_dir, + "uid": "TRd_567FkHy-CeGsAzs8aA" + } + ) + r, out, err = a.run() + print(out) + result = r[0]["localhost"] + self.assertEqual(result["ok"], 3, "3 things didn't happen") + self.assertEqual(result["failures"], 0, "failures was not 0") + self.assertEqual(result["changed"], 0, "0 things didn't change") + + # @unittest.skip + @patch("keeper_secrets_manager_core.core.SecretsManager.get_secrets", side_effect=mocked_get_secrets) + def test_keeper_get_mock(self, _): + self._common() + + @unittest.skip + def test_keeper_get_live(self): + self._common() diff --git a/integration/keeper_secrets_manager_ansible/tests/keeper_copy_test.py b/integration/keeper_secrets_manager_ansible/tests/keeper_copy_test.py index 2f82faa0..31b3b8ce 100644 --- a/integration/keeper_secrets_manager_ansible/tests/keeper_copy_test.py +++ b/integration/keeper_secrets_manager_ansible/tests/keeper_copy_test.py @@ -8,15 +8,13 @@ # Our fake data. Two login records and a file record with two attached files. records = { - "EG6KdJaaLG7esRZbMnfbFA": RecordMaker.make_record( - uid="EG6KdJaaLG7esRZbMnfbFA", - title="JW-F1-R1", - value="aaa" - ), "TRd_567FkHy-CeGsAzs8aA": RecordMaker.make_record( - uid="EG6KdJaaLG7esRZbMnfbFA", + uid="TRd_567FkHy-CeGsAzs8aA", title="JW-F1-R1", - value="ddd" + fields={ + "login": "aaa", + "password": "ddd" + } ), "A_7YpGBUgRTeDEQLhVRo0Q": RecordMaker.make_file( uid="A_7YpGBUgRTeDEQLhVRo0Q", @@ -73,9 +71,9 @@ def _common(self): ) r, out, err = a.run() result = r[0]["localhost"] - self.assertEqual(result["ok"], 3, "3 things didn't happen") + self.assertEqual(result["ok"], 4, "4 things didn't happen") self.assertEqual(result["failures"], 0, "failures was n ot 0") - self.assertEqual(result["changed"], 2, "2 things didn't change") + self.assertEqual(result["changed"], 3, "3 things didn't change") ls = os.listdir(temp_dir) self.assertTrue("password" in ls, "did not find file password") self.assertTrue("video.mp4" in ls, "did not find file video.mp4") diff --git a/integration/keeper_secrets_manager_ansible/tests/keeper_get_test.py b/integration/keeper_secrets_manager_ansible/tests/keeper_get_test.py index 88b0a1f7..2c7b5dd2 100644 --- a/integration/keeper_secrets_manager_ansible/tests/keeper_get_test.py +++ b/integration/keeper_secrets_manager_ansible/tests/keeper_get_test.py @@ -7,15 +7,12 @@ records = { - "EG6KdJaaLG7esRZbMnfbFA": RecordMaker.make_record( - uid="EG6KdJaaLG7esRZbMnfbFA", - title="JW-F1-R1", - value="aaa" - ), "TRd_567FkHy-CeGsAzs8aA": RecordMaker.make_record( uid="TRd_567FkHy-CeGsAzs8aA", title="JW-F1-R1", - value="ddd" + fields={ + "password": "ddd" + } ), "A_7YpGBUgRTeDEQLhVRo0Q": RecordMaker.make_file( uid="A_7YpGBUgRTeDEQLhVRo0Q", @@ -64,7 +61,7 @@ def _common(self): self.assertEqual(result["ok"], 3, "3 things didn't happen") self.assertEqual(result["failures"], 0, "failures was not 0") self.assertEqual(result["changed"], 0, "0 things didn't change") - self.assertRegex(out, r'password_ddd', "Did not find the password in the stdout") + self.assertRegex(out, r'ddd', "Did not find the password in the stdout") # @unittest.skip @patch("keeper_secrets_manager_core.core.SecretsManager.get_secrets", side_effect=mocked_get_secrets) diff --git a/integration/keeper_secrets_manager_ansible/tests/keeper_init.py b/integration/keeper_secrets_manager_ansible/tests/keeper_init.py new file mode 100644 index 00000000..c32c8263 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/tests/keeper_init.py @@ -0,0 +1,107 @@ +import unittest +from unittest.mock import patch +import os +from .ansible_test_framework import AnsibleTestFramework, RecordMaker +import keeper_secrets_manager_ansible.plugins +import tempfile + + +records = { + "TRd_567FkHy-CeGsAzs8aA": RecordMaker.make_record( + uid="TRd_567FkHy-CeGsAzs8aA", + title="JW-F1-R1", + fields={ + "password": "ddd" + } + ), + "A_7YpGBUgRTeDEQLhVRo0Q": RecordMaker.make_file( + uid="A_7YpGBUgRTeDEQLhVRo0Q", + title="JW-F1-R2-File", + files=[ + {"name": "nailing it.mp4", "type": "video/mp4", "url": "http://localhost/abc", "data": "ABC123"}, + {"name": "video_file.mp4", "type": "video/mp4", "url": "http://localhost/xzy", "data": "XYZ123"}, + ] + ) +} + + +def mocked_get_secrets(*args): + + if len(args) > 0: + uid = args[0][0] + ret = [records[uid]] + else: + ret = [records[x] for x in records] + return ret + + +class KeeperInitTest(unittest.TestCase): + + def setUp(self): + + self.yml_file_name = "test_keeper.yml" + self.json_file_name = "test_keeper.json" + + # Add in addition Python libs. This includes the base + # module for Keeper Ansible and the Keeper SDK. + self.base_dir = os.path.dirname(os.path.realpath(__file__)) + self.ansible_base_dir = os.path.join(self.base_dir, "ansible_example") + self.yml_file = os.path.join(os.path.join(self.ansible_base_dir, self.yml_file_name)) + self.json_file = os.path.join(os.path.join(self.ansible_base_dir, self.json_file_name)) + for file in [self.yml_file, self.json_file]: + if os.path.exists(file) is True: + os.unlink(file) + + def tearDown(self): + for file in [self.yml_file, self.json_file]: + if os.path.exists(file) is True: + os.unlink(file) + + def _common(self): + with tempfile.TemporaryDirectory() as temp_dir: + a = AnsibleTestFramework( + base_dir=self.ansible_base_dir, + playbook=os.path.join("playbooks", "keeper_init.yml"), + inventory=os.path.join("inventory", "all"), + plugin_base_dir=os.path.join(os.path.dirname(keeper_secrets_manager_ansible.plugins.__file__)), + vars={ + "keeper_token": "US:XXXXXX", + "keeper_config_file": self.yml_file_name, + "show_config": True + } + ) + r, out, err = a.run() + result = r[0]["localhost"] + self.assertEqual(result["ok"], 2, "1 things didn't happen") + self.assertEqual(result["failures"], 0, "failures was not 0") + self.assertEqual(result["changed"], 0, "0 things didn't change") + + self.assertTrue(os.path.exists(self.yml_file), "test_keeper.yml does not exist") + + a = AnsibleTestFramework( + base_dir=self.ansible_base_dir, + playbook=os.path.join("playbooks", "keeper_init.yml"), + inventory=os.path.join("inventory", "all"), + plugin_base_dir=os.path.join(os.path.dirname(keeper_secrets_manager_ansible.plugins.__file__)), + vars={ + "keeper_token": "US:XXXXXX", + "keeper_config_file": self.json_file_name, + "show_config": False + } + ) + r, out, err = a.run() + result = r[0]["localhost"] + self.assertEqual(result["ok"], 2, "1 things didn't happen") + self.assertEqual(result["failures"], 0, "failures was not 0") + self.assertEqual(result["changed"], 0, "0 things didn't change") + + self.assertTrue(os.path.exists(self.json_file), "test_keeper.json does not exist") + + # @unittest.skip + @patch("keeper_secrets_manager_core.core.SecretsManager.get_secrets", side_effect=mocked_get_secrets) + def test_keeper_get_mock(self, _): + self._common() + + @unittest.skip + def test_keeper_get_live(self): + self._common() diff --git a/integration/keeper_secrets_manager_ansible/tests/keeper_lookup_test.py b/integration/keeper_secrets_manager_ansible/tests/keeper_lookup_test.py index 4b4f1763..de82a2c2 100644 --- a/integration/keeper_secrets_manager_ansible/tests/keeper_lookup_test.py +++ b/integration/keeper_secrets_manager_ansible/tests/keeper_lookup_test.py @@ -5,16 +5,19 @@ from .ansible_test_framework import AnsibleTestFramework, RecordMaker import tempfile + records = { - "EG6KdJaaLG7esRZbMnfbFA": RecordMaker.make_record( - uid="EG6KdJaaLG7esRZbMnfbFA", - title="JW-F1-R1", - value="aaa" - ), "TRd_567FkHy-CeGsAzs8aA": RecordMaker.make_record( - uid="EG6KdJaaLG7esRZbMnfbFA", + uid="TRd_567FkHy-CeGsAzs8aA", title="JW-F1-R1", - value="ddd" + fields={ + "password": "ddd", + "login": "aaa", + "phone": [ + {'number': '(555) 123-2222', 'type': 'Work', 'ext': '6666'}, + {'number': '(555) 789-3333', 'type': 'Mobile'} + ] + } ), "A_7YpGBUgRTeDEQLhVRo0Q": RecordMaker.make_file( uid="A_7YpGBUgRTeDEQLhVRo0Q", @@ -61,11 +64,17 @@ def _common(self): ) r, out, err = a.run() result = r[0]["localhost"] - self.assertEqual(result["ok"], 2, "2 things didn't happen") + self.assertEqual(result["ok"], 5, "5 things didn't happen") self.assertEqual(result["failures"], 0, "failures was not 0") self.assertEqual(result["changed"], 0, "0 things didn't change") - self.assertRegex(out, r'My password is password_ddd', "did not find the debug message") + self.assertRegex(out, r'My password is ddd', "did not find the password debug message") + self.assertRegex(out, r'My login is aaa', "did not find the login debug message") + + self.assertRegex(out, r"My phone_1 is \{'number': '\(555\) 123-2222", + "did not find the phone_1 debug message") + self.assertRegex(out, r"My phone_2 is \[\{'number': '\(555\) 123-2222.*'number': '\(555\) 789-3333'", + "did not find the phone_2 debug message") # @unittest.skip @patch("keeper_secrets_manager_core.core.SecretsManager.get_secrets", side_effect=get_secrets) diff --git a/integration/keeper_secrets_manager_ansible/tests/keeper_redact.py b/integration/keeper_secrets_manager_ansible/tests/keeper_redact.py new file mode 100644 index 00000000..e6260148 --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/tests/keeper_redact.py @@ -0,0 +1,32 @@ +import unittest +from keeper_secrets_manager_ansible.plugins.callback_plugins.keeper_redact import CallbackModule + + +class KeeperRedactTest(unittest.TestCase): + + def test_redact_keeper_test(self): + my_dict = { + "keeper_config": "ABCDEF", + "keeper_client_id": "1234", + "keeper_private_key": "ZXY", + "Keeper_nothing": "ABC" + + } + CallbackModule._remove_special_keeper_values(my_dict) + self.assertEqual(my_dict["keeper_config"], "****") + self.assertEqual(my_dict["keeper_client_id"], "****") + self.assertEqual(my_dict["keeper_private_key"], "****") + self.assertEqual(my_dict["Keeper_nothing"], "ABC") + + real_example = { + "ansible_included_var_files": [ + "/runner/project/defaults/secrets.yml" + ], + "ansible_facts": { + "keeper_config": "ewo ... p9" + }, + "_ansible_no_log": False, + "changed": False + } + CallbackModule._remove_special_keeper_values(real_example) + self.assertEqual(real_example["ansible_facts"]["keeper_config"], "****") diff --git a/integration/keeper_secrets_manager_ansible/tests/keeper_set_test.py b/integration/keeper_secrets_manager_ansible/tests/keeper_set_test.py index b267a0fb..7e614062 100644 --- a/integration/keeper_secrets_manager_ansible/tests/keeper_set_test.py +++ b/integration/keeper_secrets_manager_ansible/tests/keeper_set_test.py @@ -8,20 +8,14 @@ import pickle -os.environ["OBJC_DISABLE_INITIALIZE_FORK_SAFETY"] = "Yes" - - # Our fake data. Two login records and a file record with two attached files. records = { - "EG6KdJaaLG7esRZbMnfbFA": RecordMaker.make_record( - uid="EG6KdJaaLG7esRZbMnfbFA", - title="JW-F1-R1", - value="aaa" - ), "TRd_567FkHy-CeGsAzs8aA": RecordMaker.make_record( uid="TRd_567FkHy-CeGsAzs8aA", title="JW-F1-R1", - value="ddd" + fields={ + "password": "ddd" + } ), "A_7YpGBUgRTeDEQLhVRo0Q": RecordMaker.make_file( uid="A_7YpGBUgRTeDEQLhVRo0Q", @@ -122,7 +116,7 @@ def _common(self): self.assertEqual(result["failures"], 0, "failures was not 0") self.assertEqual(result["changed"], 0, "0 things didn't change") - self.assertRegex(out, r'Current Password password_ddd', "did not find current password") + self.assertRegex(out, r'Current Password ddd', "did not find current password") self.assertRegex(out, r'New Password NEW PASSWORD', "did not find new password") # @unittest.skip diff --git a/integration/keeper_secrets_manager_ansible/utils/version.sh b/integration/keeper_secrets_manager_ansible/utils/version.sh new file mode 100755 index 00000000..251529fe --- /dev/null +++ b/integration/keeper_secrets_manager_ansible/utils/version.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +VERSION=$(keeper_ansible --version | awk -v FS="(Plugin Version: |\n)" '{print $2}' | tr -cd '[:alnum:]._-') +if [ -z "$VERSION" ]; then + echo "Cannot find version from installed module." + exit 1 +fi +echo $VERSION \ No newline at end of file diff --git a/sdk/python/core/keeper_secrets_manager_core/core.py b/sdk/python/core/keeper_secrets_manager_core/core.py index e9602f20..5be258c0 100644 --- a/sdk/python/core/keeper_secrets_manager_core/core.py +++ b/sdk/python/core/keeper_secrets_manager_core/core.py @@ -819,7 +819,9 @@ def inflate_field_value(self, uids, replace_fields): class KSMCache: - kms_cache_file_name = 'ksm_cache.bin' + # Allow the directory that will contain the cache to be set with environment variables. If not set, the + # cache file will be create in the current working directory. + kms_cache_file_name = os.path.join(os.environ.get("KSM_CACHE_DIR", ""), 'ksm_cache.bin') @staticmethod def save_cache(data): @@ -834,6 +836,11 @@ def get_cached_data(): cache_file.close() return cache_data + @staticmethod + def remove_cache_file(): + if os.path.exists(KSMCache.kms_cache_file_name) is True: + os.unlink(KSMCache.kms_cache_file_name) + @staticmethod def caching_post_function(url, transmission_key, encrypted_payload_and_signature, verify_ssl_certs=True): diff --git a/sdk/python/core/setup.py b/sdk/python/core/setup.py index 1067f496..65749f26 100644 --- a/sdk/python/core/setup.py +++ b/sdk/python/core/setup.py @@ -19,7 +19,7 @@ setup( name="keeper-secrets-manager-core", - version="16.2.1", + version="16.2.2", description="Keeper Secrets Manager for Python 3", long_description=long_description, long_description_content_type="text/markdown",