From 5d97cf399d328de64f723344501a8598c39ce43a Mon Sep 17 00:00:00 2001 From: Takuya Mishina Date: Fri, 7 Aug 2020 16:22:18 +0900 Subject: [PATCH] feat: cluster list fetchers and cluster resource fetcher --- arboretum/common/errors.py | 58 +++++++ arboretum/common/utils.py | 26 +++ arboretum/ibm_cloud/README.md | 54 +++++- .../ibm_cloud/fetchers/fetch_cluster_list.py | 70 ++++++++ arboretum/kubernetes/README.md | 76 ++++++++- .../kubernetes/fetchers/fetch_cluster_list.py | 37 +++++ .../fetchers/fetch_cluster_resource.py | 155 ++++++++++++++++++ 7 files changed, 474 insertions(+), 2 deletions(-) create mode 100644 arboretum/common/errors.py create mode 100644 arboretum/ibm_cloud/fetchers/fetch_cluster_list.py create mode 100644 arboretum/kubernetes/fetchers/fetch_cluster_list.py create mode 100644 arboretum/kubernetes/fetchers/fetch_cluster_resource.py diff --git a/arboretum/common/errors.py b/arboretum/common/errors.py new file mode 100644 index 00000000..ce7e359e --- /dev/null +++ b/arboretum/common/errors.py @@ -0,0 +1,58 @@ +# -*- mode:python; coding:utf-8 -*- +# Copyright (c) 2020 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common error classes.""" + + +class CommandExecutionError(RuntimeError): + """Represents error at executing command.""" + + def __init__(self, cmd, stdout, stderr, returncode): + """Initialize an instance. + + Initialize an instance with the return values of the command. + """ + self.__cmd = cmd + self.__stdout = stdout + self.__stderr = stderr + self.__returncode = returncode + + def __str__(self): + """Get information about the command line and its result.""" + return ( + f'Error running command: {self.cmd}\n' + f'returncode: {self.returncode}\n' + f'stdout: {self.stdout}\n' + f'stderr: {self.stderr}' + ) + + @property + def cmd(self): + """Get command line text.""" + return self.__cmd + + @property + def stdout(self): + """Get standard out text of the command.""" + return self.__stdout + + @property + def stderr(self): + """Get standard error text of the command.""" + return self.__stderr + + @property + def returncode(self): + """Get return code of the command.""" + return self.__returncode diff --git a/arboretum/common/utils.py b/arboretum/common/utils.py index ac5d6f53..d29b456c 100644 --- a/arboretum/common/utils.py +++ b/arboretum/common/utils.py @@ -14,6 +14,10 @@ # limitations under the License. """Common utility functions.""" +import subprocess + +from arboretum.common.errors import CommandExecutionError + from compliance.evidence import DAY, HOUR @@ -34,3 +38,25 @@ def parse_seconds(seconds): formatted.append(f'{q} {unit}') seconds = r return ', '.join(formatted) + + +def run_command(cmd, secrets=None): + """Run commands in a system.""" + if type(cmd) == str: + cmd = cmd.split(' ') + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True + ) + stdout, stderr = p.communicate() + + if p.returncode != 0: + secrets = secrets or [] + for s in secrets: + cmd = cmd.replace(s, '***') + stdout = stdout.replace(s, '***') + stderr = stderr.replace(s, '***') + raise CommandExecutionError(cmd, stdout, stderr, p.returncode) + return stdout diff --git a/arboretum/ibm_cloud/README.md b/arboretum/ibm_cloud/README.md index bc320031..0cf32bbb 100644 --- a/arboretum/ibm_cloud/README.md +++ b/arboretum/ibm_cloud/README.md @@ -18,10 +18,62 @@ how to include the fetchers and checks from this library in your downstream proj ## Fetchers -Fetchers coming soon... +### Cluster List + +* Class: [ClusterListFetcher][fetch-cluster-list] +* Purpose: Write the list of IBM Cloud clusters to the evidence locker. +* Behavior: Log in to IBM Cloud using `ibmcloud login` command, and save the result of `ibmcloud cs cluster ls` command. +* Expected configuration elements: + * org.ibm_cloud.cluster_list.config + * List of objects representing the IKS accounts + * Each object must have the following values + * `account` - a list containing names identifying the IKS account, this will map to an IAM token provided in the credentials file +* Expected configuration example: + ```json + { + "org": { + "ibm_cloud": { + "cluster_list": { + "config": { + "account": ["myaccount1"] + } + } + } + } + } + ``` +* Expected credentials: + * `ibm_cloud` credentials with read/view permissions are needed for this fetcher to successfully + retrieve the evidence. + * One IKS API key is required per account specified in the configuration. See above. Each account provided in the configuration must preceed `_api_key`. For example, if we have specified accounts "acct_a", "acct_b", and "acct_c" your configuration should look like: + + ```ini + [ibm_cloud] + acct_a_api_key=your-iks-api-key-for-acct-a + acct_b_api_key=your-iks-api-key-for-acct-b + acct_c_api_key=your-iks-api-key-for-acct-c + ``` + + * Expected Travis environment variable settings to generate credentials: + * `IBM_CLOUD_ACCT_A_API_KEY` + * `IBM_CLOUD_ACCT_B_API_KEY` + * `IBM_CLOUD_ACCT_C_API_KEY` + + * NOTE: [API Keys can be generated using the ibmcloud CLI][ic-api-key-create]. E.g. + + ```sh + ibmcloud iam api-key-create your-iks-api-key-for-acct-x + ``` +* Import statement: + + ```python + from arboretum.ibm_cloud.fetchers.fetch_cluster_list import ClusterListFetcher + ``` ## Checks Checks coming soon... [usage]: https://github.com/ComplianceAsCode/auditree-arboretum#usage +[ic-api-key-create]: https://cloud.ibm.com/docs/cli/reference/ibmcloud?topic=cloud-cli-ibmcloud_commands_iam#ibmcloud_iam_api_key_create +[fetch-cluster-list]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/ibm_cloud/fetchers/fetch_cluster_list.py \ No newline at end of file diff --git a/arboretum/ibm_cloud/fetchers/fetch_cluster_list.py b/arboretum/ibm_cloud/fetchers/fetch_cluster_list.py new file mode 100644 index 00000000..60a9126f --- /dev/null +++ b/arboretum/ibm_cloud/fetchers/fetch_cluster_list.py @@ -0,0 +1,70 @@ +# -*- mode:python; coding:utf-8 -*- +# Copyright (c) 2020 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""IKS cluster list fetcher.""" + +import json + +from arboretum.common.errors import CommandExecutionError +from arboretum.common.utils import run_command + +from compliance.evidence import store_raw_evidence +from compliance.fetch import ComplianceFetcher + + +class ClusterListFetcher(ComplianceFetcher): + """Fetch the list of IBM Cloud clusters.""" + + @classmethod + def setUpClass(cls): + """Initialize the fetcher object with configuration settings.""" + cls.logger = cls.locker.logger.getChild( + 'ibm_cloud.cluster_list_fetcher' + ) + return cls + + @store_raw_evidence('ibm_cloud/cluster_list.json') + def fetch_cluster_list(self): + """Fetch IBM Cloud cluster list.""" + for account in self.config.get( + 'org.ibm_cloud.cluster_list.config.account'): + return json.dumps({account: self._get_cluster_list(account)}) + + def _get_cluster_list(self, account): + + # get credential for the account + api_key = getattr(self.config.creds['ibm_cloud'], account + '_api_key') + + # login + run_command( + f'ibmcloud login --no-region --apikey {api_key}', + secrets=[api_key] + ) + + # get cluster list + cluster_list = None + cmd = 'ibmcloud cs cluster ls --json' + try: + cluster_list = run_command(cmd) + except CommandExecutionError as e: + if e.returncode == 2: # "2" means no plugin error + self.logger.warning( + 'Failed to execute "ibmcloud cs" command' + ' - trying to install the cs plugin' + ) + run_command('ibmcloud plugin install kubernetes-service') + cluster_list = run_command(cmd) + else: + raise e + return json.loads(cluster_list) diff --git a/arboretum/kubernetes/README.md b/arboretum/kubernetes/README.md index 0adce9f1..822d3602 100644 --- a/arboretum/kubernetes/README.md +++ b/arboretum/kubernetes/README.md @@ -18,10 +18,84 @@ how to include the fetchers and checks from this library in your downstream proj ## Fetchers -Fetchers coming soon... +### Cluster List + +* Class: [ClusterListFetcher][fetch-cluster-list] +* Purpose: Write the list of kubernetes clusters to the evidence locker. +* Behavior: Read BOM (Bill of Materials) data in config file and write it into `raw/kubernetes/cluster_list.json`. +* Expected configuration elements: + * org.kubernetes.cluster_list.config.bom + * List of target kubernetes clusters (see example below) +* Expected configuration example: + ```json + { + "org": { + "kubernetes": { + "cluster_list": { + "config": { + "bom": [ + { + "account": "ibmcloud_myaccount", + "name": "mycluster-free", + "kubeconfig": "/home/myaccount/.kube/mycluster-free.kubeconfig", + "type": "kubernetes" + } + ] + } + } + } + } + } + ``` +* Expected credentials: + * A kubeconfig file specified in org.kube.cluster_list.config.bom[].kubeconfig of config file must be a valid kubeconfig file. +* Import statement: + ```python + from arboretum.kubernetes.fetchers.fetch_cluster_list import ClusterListFetcher + ``` + +### Cluster Resource + +* Class: [ClusterResourceFetcher][fetch-cluster-resource] +* Purpose: Write the resources of clusters to the evidence locker. +* Behavior: Read a cluster list from `raw/CATEGORY/cluster_list.json` where `CATEGORY` is the category name specified in configuration, and fetch resources from the clusters. Fetch target resource types can be specified in config file. +* Expected configuration elements: + * org.kubernetes.cluster_resource.config + * `cluster_list_types`: cluster list types (same as category names) - for example, specify `kubernetes` if you want to read the cluster list by [ClusterListFetcher of kubernetes][fetch-cluster-list], and specify `ibm_cloud` if you want to read the cluster list by [ClusterListFetcher of ibm_cloud][fetch-cluster-list-ibmcloud]. + * `target_resource_types`: list of target resource types (default: [`node`, `configmap`]) +* Expected configuration example: + ```json + { + "org": { + "kubernetes": { + "cluster_resource": { + "config": { + "cluster_list_types": [ + "kubernetes", "ibm_cloud" + ], + "target_resource_types": [ + "node" + ] + } + } + } + } + } + ``` +* Expected credentials: + * Credentials (including kubeconfig file) are required for the clusters specified in the configuration. See documents of cluster list fetchers ([ClusterListFetcher of kubernetes][fetch-cluster-list], [ClusterListFetcher of ibm_cloud][fetch-cluster-list-ibmcloud]) because the cluster resource fetcher also use the credentials for the list fetchers. +* Import statement: + + ```python + from arboretum.kubernetes.fetchers.fetch_cluster_resource import ClusterResourceFetcher + ``` + ## Checks Checks coming soon... [usage]: https://github.com/ComplianceAsCode/auditree-arboretum#usage +[fetch-cluster-list]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/kubernetes/fetchers/fetch_cluster_list.py +[fetch-cluster-list-ibmcloud]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/ibm_cloud/fetchers/fetch_cluster_list.py +[fetch-cluster-resource]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/kubernetes/fetchers/fetch_cluster_resource.py diff --git a/arboretum/kubernetes/fetchers/fetch_cluster_list.py b/arboretum/kubernetes/fetchers/fetch_cluster_list.py new file mode 100644 index 00000000..9cf42e2b --- /dev/null +++ b/arboretum/kubernetes/fetchers/fetch_cluster_list.py @@ -0,0 +1,37 @@ +# -*- mode:python; coding:utf-8 -*- +# Copyright (c) 2020 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fetch cluster list by using other providers' fetch_cluster_list function.""" +import json + +from compliance.evidence import store_raw_evidence +from compliance.fetch import ComplianceFetcher + + +class ClusterListFetcher(ComplianceFetcher): + """Fetch BOM (Bill of Materials) as cluster list.""" + + @classmethod + def setUpClass(cls): + """Initialize the fetcher object with configuration settings.""" + cls.logger = cls.locker.logger.getChild( + 'kubernetes.cluster_list_fetcher' + ) + return cls + + @store_raw_evidence('kubernetes/cluster_list.json') + def fetch_cluster_list(self): + """Fetch BOM (Bill of Materials) as cluster list.""" + bom = self.config.get('org.kubernetes.cluster_list.config.bom') + return json.dumps(bom) diff --git a/arboretum/kubernetes/fetchers/fetch_cluster_resource.py b/arboretum/kubernetes/fetchers/fetch_cluster_resource.py new file mode 100644 index 00000000..d67dc6ab --- /dev/null +++ b/arboretum/kubernetes/fetchers/fetch_cluster_resource.py @@ -0,0 +1,155 @@ +# -*- mode:python; coding:utf-8 -*- +# Copyright (c) 2020 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""IKS cluster list fetcher.""" + +import json + +from arboretum.common.errors import CommandExecutionError +from arboretum.common.utils import run_command + +from compliance.evidence import evidences, store_raw_evidence +from compliance.fetch import ComplianceFetcher + + +class ClusterResourceFetcher(ComplianceFetcher): + """Fetch the resources of clusters.""" + + RESOURCE_TYPE_DEFAULT = ['node', 'pod', 'configmap'] + + @classmethod + def setUpClass(cls): + """Initialize the fetcher object with configuration settings.""" + cls.logger = cls.locker.logger.getChild( + 'kubernetes.cluster_resource_fetcher' + ) + return cls + + @store_raw_evidence('kubernetes/cluster_resource.json') + def fetch_cluster_resource(self): + """Fetch cluster resources of listed clusters.""" + cluster_list_types = self.config.get( + 'org.kubernetes.cluster_resource.config.cluster_list_types' + ) + + resources = {} + for cltype in cluster_list_types: + try: + if cltype == 'kubernetes': + resources['kubernetes'] = self._fetch_bom_resource() + elif cltype == 'ibm_cloud': + resources['ibm_cloud'] = self._fetch_ibm_cloud_resource() + else: + self.logger.error( + 'Specified cluster list type "%s" is not supported', + cltype + ) + except Exception as e: + self.logger.error( + 'Failed to fetch resources for cluster list "%s": %s', + cltype, + str(e) + ) + continue + + return json.dumps(resources) + + def _fetch_bom_resource(self): + resource_types = self.config.get( + 'org.kubernetes.cluster_resource.config.target_resource_types', + ClusterResourceFetcher.RESOURCE_TYPE_DEFAULT + ) + + bom = {} + with evidences(self.locker, 'raw/kubernetes/cluster_list.json') as ev: + bom = json.loads(ev.content) + + resources = {} + for c in bom: + cluster_resources = [] + for r in resource_types: + cmd = ( + f'kubectl --kubeconfig {c["kubeconfig"]}' + f' get {r} -A -o json' + ) + out = run_command(cmd) + cluster_resources.extend(json.loads(out)['items']) + resources[c['account']] = [ + { + 'name': c['name'], 'resources': cluster_resources + } + ] + return resources + + def _fetch_ibm_cloud_resource(self): + resource_types = self.config.get( + 'org.ibm_cloud.cluster_resource.config.target_resource_types', + ClusterResourceFetcher.RESOURCE_TYPE_DEFAULT + ) + cluster_list = {} + with evidences(self.locker, 'raw/ibm_cloud/cluster_list.json') as ev: + cluster_list = json.loads(ev.content) + + resources = {} + for account in cluster_list: + api_key = getattr( + self.config.creds['ibm_cloud'], account + '_api_key' + ) + # login + run_command( + f'ibmcloud login --no-region --apikey {api_key}', + secrets=[api_key] + ) + resources[account] = [] + for cluster in cluster_list[account]: + # get configuration + cmd = f'ibmcloud cs cluster config -s -c {cluster["name"]}' + try: + run_command(cmd) + except CommandExecutionError as e: + if e.returncode == 2: # 2 means no plugin error + self.logger.warning( + 'Failed to execute ' + '"ibmcloud cs" command. ' + 'trying to install the cs plugin.' + ) + run_command( + 'ibmcloud plugin install kubernetes-service' + ) + run_command(cmd) + else: + raise e + + # login using "oc" command if the target is openshift + if cluster['type'] == 'openshift': + run_command(f'oc login -u apikey -p {api_key}') + + # get resources + resource_list = [] + for resource in resource_types: + try: + output = run_command( + f'kubectl get {resource} -A -o json' + ) + resource_list.extend(json.loads(output)['items']) + except RuntimeError: + self.logger.warning( + 'Failed to get %s resource in cluster %s', + resource, + cluster['name'] + ) + cluster['resources'] = resource_list + resources[account].append(cluster) + + return resources