diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index c930926c2..e529bcc8d 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -12,10 +12,10 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.12" - name: Install software run: pip3 install yq - name: Parse kubernetes versions @@ -48,10 +48,10 @@ jobs: ssh-keyscan -H 172.17.0.1 >> ~/.ssh/known_hosts - name: Test ssh connection run: ssh -i ~/.ssh/id_rsa 172.17.0.1 echo "Test" - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.12" - name: Install Kubemarine with dependencies run: | python -m pip install --upgrade pip @@ -110,10 +110,10 @@ jobs: run: sudo ifconfig docker0:0 172.17.1.1 up - name: Test ssh connection run: ssh -i ~/.ssh/id_rsa 172.17.0.1 echo "Test" - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.12" - name: Install Kubemarine with dependencies run: | python -m pip install --upgrade pip @@ -160,7 +160,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.12" - name: Install dependencies run: pip install . && pip uninstall -y kubemarine - name: Run scripts/thirdparties/sync.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 557875ad4..8deeacb06 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.12" # Install coverage and kubemarine with all dependencies except ansible. # Then uninstall only kubemarine to avoid ambiguity and to surely run coverage on sources. - run: pip install coverage . && pip uninstall -y kubemarine @@ -41,7 +41,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.12" # Install pylint and kubemarine with all dependencies except ansible. # Then uninstall only kubemarine to avoid ambiguity and to surely run pylint on sources. - run: pip install .[pylint] && pip install -r requirements-pyinstaller.txt && pip uninstall -y kubemarine @@ -56,7 +56,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.12" - run: pip install radon xenon # Use `radon cc {paths} -a` locally for full report per function # xenon checks, if radon absolute result is A diff --git a/documentation/Kubecheck.md b/documentation/Kubecheck.md index a64811e24..41637950b 100644 --- a/documentation/Kubecheck.md +++ b/documentation/Kubecheck.md @@ -82,6 +82,7 @@ This section provides information about the Kubecheck functionality. - [229 Audit Policy Configuration](#229-audit-policy-configuration) - [231 Audit Daemon Rules](#231-audit-daemon-rules) - [232 Kernel Parameters Configuration](#232-kernel-parameters-configuration) + - [235 Kubernetes version](#235-kubernetes-version) - [Report File Generation](#report-file-generation) - [HTML Report](#html-report) - [CSV Report](#csv-report) @@ -437,6 +438,7 @@ The task tree is as follows: * audit * policy * admission + * version * etcd * health_status * control_plane @@ -778,6 +780,14 @@ This test compares the kernel parameters on the nodes with the parameters specified in the inventory or with the default parameters. If the configured parameters are not presented, the test fails. +##### 235 Kubernetes version + +*Task*: `kubernetes.version` + +This test checks if used kubernetes version is deprecated in current kubemarine release. +It means, that this version is going to be excluded from support in future kubemarine soon. +So it's recommended to update kubernetes version to new one. + ### Report File Generation In addition to the resulting table in the log output, the same report is presented in the form of files. diff --git a/documentation/Troubleshooting.md b/documentation/Troubleshooting.md index 4167d0a81..ccd17346f 100644 --- a/documentation/Troubleshooting.md +++ b/documentation/Troubleshooting.md @@ -42,6 +42,7 @@ This section provides troubleshooting information for Kubemarine and Kubernetes - [Troubleshooting an Installation That Ended Incorrectly](#troubleshooting-an-installation-that-ended-incorrectly) - [Upgrade Procedure to v1.28.3 Fails on ETCD Step](#upgrade-procedure-to-v1283-fails-on-etcd-step) - [kubectl logs and kubectl exec fail](#kubectl-logs-and-kubectl-exec-fail) + - [OpenSSH server becomes unavailable during cluster installation on Centos9](#openssh-server-becomes-unavailable-during-cluster-installation-on-centos9) # Kubemarine Errors @@ -1404,3 +1405,27 @@ Error from server: error dialing backend: remote error: tls: internal error **Root cause**: The `kubelet` server certificate is not approved, whereas the cluster has been configured not to use self-signed certificates for the `kubelet` server. **Solution**: Perform CSR approval steps from the maintenance guide. Refer to the [Kubelet Server Certificate Approval](https://github.com/Netcracker/KubeMarine/blob/main/documentation/internal/Hardening.md#kubelet-server-certificate-approval) section for details. + +## OpenSSH server becomes unavailable during cluster installation on Centos9 + +**Sympthoms**: Installation fails on `kubemarine.system.reboot_nodes`, OpenSSH server becomes unavailable due to OpenSSL version missmatch error. + +The following lines can be found in the OpenSSH server logs: +``` +OpenSSL version mismatch. Built against 30000070, you have 30200010 +sshd.service: Main process exited, code=exited, status=255/EXEPTION +sshd.service: Failed with result 'exit-code'. +Failed to start OpenSSH server daemon. +``` + +**Root cause**: Since OpenSSL is updated by default when deploying a cluster with KubeMarine, the version incompatibility problem arises. OpenSSH was compiled with OpenSSL version 3.0.0 (30000070) and after the update, version 3.2.0 (30200010) is installed. +Probably, OpenSSL does not provide backward compatibility. + +**Solution**: Add the upgrade section for OpenSSH server in the **cluster.yaml** file. + +```yaml +services: + packages: + upgrade: + - openssh-server +``` diff --git a/kubemarine/kubernetes/__init__.py b/kubemarine/kubernetes/__init__.py index 3d5c7d4e9..e560d0b27 100644 --- a/kubemarine/kubernetes/__init__.py +++ b/kubemarine/kubernetes/__init__.py @@ -904,11 +904,13 @@ def verify_allowed_version(version: str) -> str: return version -def verify_supported_version(target_version: str, logger: log.EnhancedLogger) -> None: +def verify_supported_version(target_version: str, logger: log.EnhancedLogger) -> bool: minor_version = utils.minor_version(target_version) supported_versions = static.KUBERNETES_VERSIONS['kubernetes_versions'] if not supported_versions.get(minor_version, {}).get("supported", False): - logger.warning(f"Specified target Kubernetes version {target_version!r} - is not supported!") + logger.warning(f"Specified target Kubernetes version {target_version!r} - is deprecated!") + return False + return True def expect_kubernetes_version(cluster: KubernetesCluster, version: str, diff --git a/kubemarine/patches/__init__.py b/kubemarine/patches/__init__.py index f9edd2698..7958d24dd 100644 --- a/kubemarine/patches/__init__.py +++ b/kubemarine/patches/__init__.py @@ -22,10 +22,8 @@ from typing import List from kubemarine.core.patch import Patch -from kubemarine.patches.p1_pin_kubernetes_version import PinKubernetesVersion patches: List[Patch] = [ - PinKubernetesVersion(), ] """ List of patches that is sorted according to the Patch.priority() before execution. diff --git a/kubemarine/patches/p1_pin_kubernetes_version.py b/kubemarine/patches/p1_pin_kubernetes_version.py deleted file mode 100644 index 7fca43321..000000000 --- a/kubemarine/patches/p1_pin_kubernetes_version.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2021-2023 NetCracker Technology Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from textwrap import dedent - -from kubemarine.core.action import Action -from kubemarine.core.patch import InventoryOnlyPatch -from kubemarine.core.resources import DynamicResources - - -class TheAction(Action): - def __init__(self) -> None: - super().__init__("Set previous default Kubernetes version", recreate_inventory=True) - - def run(self, res: DynamicResources) -> None: - logger = res.logger() - inventory = res.inventory() - if 'kubernetesVersion' not in inventory.get('services', {}).get('kubeadm', {}): - logger.debug("Set services.kubeadm.kubernetesVersion = v1.26.11 in the inventory") - inventory.setdefault('services', {}).setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.26.11' - else: - logger.info("Skipping the patch as services.kubeadm.kubernetesVersion is explicitly provided.") - - -class PinKubernetesVersion(InventoryOnlyPatch): - def __init__(self) -> None: - super().__init__("pin_kubernetes_version") - - @property - def action(self) -> Action: - return TheAction() - - @property - def description(self) -> str: - return dedent( - f"""\ - The patch sets previous default Kubernetes version in the inventory if the version was not explicitly specified. - """.rstrip() - ) \ No newline at end of file diff --git a/kubemarine/patches/software_upgrade.yaml b/kubemarine/patches/software_upgrade.yaml index 0027a50b0..87ec8fbbf 100644 --- a/kubemarine/patches/software_upgrade.yaml +++ b/kubemarine/patches/software_upgrade.yaml @@ -8,11 +8,7 @@ # The order of upgrade of defined by the implementation. thirdparties: - calicoctl: - - v1.28.9 - - v1.29.1 - - v1.29.4 - - v1.30.1 + calicoctl: [] crictl: [] packages: containerd: @@ -32,19 +28,7 @@ packages: version_rhel9: false version_debian: false plugins: - calico: - - v1.28.9 - - v1.29.1 - - v1.29.4 - - v1.30.1 - nginx-ingress-controller: - - v1.28.9 - - v1.29.1 - - v1.29.4 - - v1.30.1 + calico: [] + nginx-ingress-controller: [] kubernetes-dashboard: [] - local-path-provisioner: - - v1.28.9 - - v1.29.1 - - v1.29.4 - - v1.30.1 + local-path-provisioner: [] diff --git a/kubemarine/procedures/check_paas.py b/kubemarine/procedures/check_paas.py index c50eaedb9..390209f77 100755 --- a/kubemarine/procedures/check_paas.py +++ b/kubemarine/procedures/check_paas.py @@ -109,7 +109,7 @@ def _check_same_os(cluster: KubernetesCluster) -> None: different_os = set(os_ids.values()) if len(different_os) > 1: cluster.log.warning( - f"Nodes have different OS families or versions, packages versions cannot be checked. " + f"Nodes have different OS families or versions, packages versions cannot be checked." f"List of (OS family, version): {list(different_os)}") raise TestFailure(f"Nodes have different OS families or versions") @@ -1021,7 +1021,7 @@ def verify_modprobe_rules(cluster: KubernetesCluster) -> None: else: raise TestFailure('invalid', hint=f"Modprobe rules do not match those loaded in modprobe on cluster nodes. Check " - f"manually what the differences are and make changes on the appropriate nodes.") + f"the differences manually and make changes on the appropriate nodes.") def verify_sysctl_config(cluster: KubernetesCluster) -> None: @@ -1042,7 +1042,7 @@ def verify_sysctl_config(cluster: KubernetesCluster) -> None: else: raise TestFailure('invalid', hint=f"Some configured kernel parameters are not loaded on the cluster nodes.\n" - f"Check manually what the differences are, and make changes on the appropriate nodes.") + f"Check the differences manually and make changes on the appropriate nodes.") def verify_system_audit_rules(cluster: KubernetesCluster) -> None: @@ -1063,7 +1063,7 @@ def verify_system_audit_rules(cluster: KubernetesCluster) -> None: else: raise TestFailure('invalid', hint=f"Some configured Audit rules are not loaded on the cluster nodes.\n" - f"Check manually what the differences are, and make changes on the appropriate nodes.") + f"Check the differences manually and make changes on the appropriate nodes.") def etcd_health_status(cluster: KubernetesCluster) -> None: @@ -1569,6 +1569,18 @@ def kubernetes_admission_status(cluster: KubernetesCluster) -> None: cluster.log.debug(kube_admission_status) tc.success(results='enabled') +def verify_kubernetes_version(cluster: KubernetesCluster) -> None: + """ + The method checks if used kubernetes version is deprecated in kubemarine + """ + with TestCase(cluster, '225', "Kubernetes", "Version") as tc: + target_version = cluster.inventory['services']['kubeadm']['kubernetesVersion'] + if not kubernetes.verify_supported_version(target_version, cluster.log): + raise TestWarn(f"Kubernetes version {target_version} is deprecated", + hint=f"Used Kubernetes version is deprecated and will be excluded from support " + f"in future Kubemarine releases. Please plan upgrade to the newer version.") + + tc.success(results='Kubernetes version is OK') def geo_check(cluster: KubernetesCluster) -> None: """ @@ -1606,7 +1618,7 @@ def geo_check(cluster: KubernetesCluster) -> None: peers = yaml.safe_load(io.StringIO(peers_result)) if len(peers) == 0: - raise TestFailure("configuration error", hint="geo-monitor instance has no peers") + raise TestFailure("configuration error", hint="geo-monitor instance has no peers.") for peer in peers: status = peer["clusterIpStatus"] @@ -1636,7 +1648,7 @@ def geo_check(cluster: KubernetesCluster) -> None: with TestCase(cluster, '226', "Geo Monitor", "Geo check - Pod-to-service") as tc_svc: if not status_collected: - raise TestFailure("configuration error", hint="DNS check failed with error, statuses not collected") + raise TestFailure("configuration error", hint="DNS check failed with error, statuses not collected.") if svc_status["failed"]: raise TestFailure("found unavailable peer services", @@ -1647,7 +1659,7 @@ def geo_check(cluster: KubernetesCluster) -> None: with TestCase(cluster, '226', "Geo Monitor", "Geo check - Pod-to-pod") as tc_pod: if not status_collected: - raise TestFailure("configuration error", hint="DNS check failed with error, statuses not collected") + raise TestFailure("configuration error", hint="DNS check failed with error, statuses not collected.") if pod_status["failed"]: raise TestFailure("found unavailable peer pod", @@ -1703,7 +1715,7 @@ def verify_apparmor_config(cluster: KubernetesCluster) -> None: tc.success(results='valid') else: raise TestFailure('invalid', - hint=f"Some nodes do not have properly configured Apparmor service") + hint=f"Some nodes do not have properly configured Apparmor service.") else: tc.success(results='skipped') @@ -1793,6 +1805,7 @@ def verify_apparmor_config(cluster: KubernetesCluster) -> None: 'policy': kubernetes_audit_policy_configuration, }, 'admission': kubernetes_admission_status, + 'version': verify_kubernetes_version, }, 'etcd': { "health_status": etcd_health_status diff --git a/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml b/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml index bf178d8c2..6b3749200 100644 --- a/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml +++ b/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml @@ -1,7 +1,7 @@ # This section can be changed manually, but it is also synchronized automatically with the 'compatibility_map' section below. kubernetes_versions: v1.26: - supported: true + supported: false v1.27: supported: true v1.28: diff --git a/kubemarine/version b/kubemarine/version index c6a2605c4..2c768c559 100644 --- a/kubemarine/version +++ b/kubemarine/version @@ -1 +1 @@ -v0.32.2 +v0.32.3 diff --git a/pyproject.toml b/pyproject.toml index 96dea4882..2217bf8d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ "deepdiff==7.0.*", "ordered-set==4.1.*", # Each time cryptography version is updated, need to regenerate scripts/ci/custom-hooks/hook-cryptography.py - "cryptography==42.0.7", + "cryptography==43.0.1", "paramiko==3.4.*", "jsonschema==4.22.*", "referencing==0.35.*", @@ -84,7 +84,7 @@ Issues = "https://github.com/Netcracker/KubeMarine/issues/" # 1. pip install bumpver # 2. bumpver update --set-version [tool.bumpver] -current_version = "v0.32.2" +current_version = "v0.32.3" version_pattern = "vMAJOR.MINOR.PATCH" commit_message = "bump version to {new_version}" commit = true diff --git a/scripts/thirdparties/src/run.py b/scripts/thirdparties/src/run.py index 4ea9f9ee8..7c55b0291 100644 --- a/scripts/thirdparties/src/run.py +++ b/scripts/thirdparties/src/run.py @@ -16,6 +16,7 @@ from kubemarine.core import static from .compatibility import KubernetesVersions from .software import InternalCompatibility, UpgradeConfig, CompatibilityMap, SoftwareType +from .software.defaults import KubemarineDefaults from .software.kubernetes_images import KubernetesImagesResolver, KubernetesImages from .software.packages import Packages from .software.plugins import ManifestResolver, Plugins, ManifestsEnrichment @@ -27,6 +28,7 @@ class Synchronization: def __init__(self, + defaults: KubemarineDefaults, compatibility: InternalCompatibility, kubernetes_versions: KubernetesVersions, images_resolver: KubernetesImagesResolver, @@ -35,6 +37,7 @@ def __init__(self, manifests_enrichment: ManifestsEnrichment, upgrade_config: UpgradeConfig, ): + self.defaults = defaults self.compatibility = compatibility self.kubernetes_versions = kubernetes_versions self.images_resolver = images_resolver @@ -44,6 +47,8 @@ def __init__(self, self.upgrade_config = upgrade_config def run(self) -> SummaryTracker: + self.validate() + tracker = SummaryTracker(self.kubernetes_versions.compatibility_map) software: List[SoftwareType] = [ @@ -75,3 +80,12 @@ def run(self) -> SummaryTracker: tracker.print() return tracker + + def validate(self) -> None: + # Validate version + default_version = self.defaults.default_version() + if default_version not in self.kubernetes_versions.compatibility_map: + raise Exception(f"Kubemarine default version {default_version} " + f"does not exist in compatibility map. " + f"Default version should be updated before excluding it from support." + ) diff --git a/scripts/thirdparties/src/software/defaults.py b/scripts/thirdparties/src/software/defaults.py new file mode 100644 index 000000000..6d21c86b3 --- /dev/null +++ b/scripts/thirdparties/src/software/defaults.py @@ -0,0 +1,29 @@ +# Copyright 2021-2023 NetCracker Technology Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from kubemarine.core import utils + +# pylint: disable=bad-builtin + +YAML = utils.yaml_structure_preserver() +RESOURCE_PATH = utils.get_internal_resource_path("resources/configurations/defaults.yaml") + + +class KubemarineDefaults: + def __init__(self) -> None: + with utils.open_internal(RESOURCE_PATH) as stream: + self._defaults = YAML.load(stream) + + def default_version(self) -> str: + return str(self._defaults['services']['kubeadm']['kubernetesVersion']) diff --git a/scripts/thirdparties/sync.py b/scripts/thirdparties/sync.py index 9e8a7c532..fea4e8e2a 100644 --- a/scripts/thirdparties/sync.py +++ b/scripts/thirdparties/sync.py @@ -32,7 +32,7 @@ from src.software.kubernetes_images import KubernetesImagesResolver from src.software.plugins import ManifestResolver, ManifestsEnrichment from src.software.thirdparties import ThirdpartyResolver - +from src.software.defaults import KubemarineDefaults if __name__ == '__main__': if platform.system() != 'Linux': @@ -46,6 +46,7 @@ args = parser.parse_args() Synchronization( + KubemarineDefaults(), InternalCompatibility(), KubernetesVersions(), KubernetesImagesResolver(), diff --git a/test/unit/tools/thirdparties/stub.py b/test/unit/tools/thirdparties/stub.py index 4e8ead1c6..c3d7a8eb7 100644 --- a/test/unit/tools/thirdparties/stub.py +++ b/test/unit/tools/thirdparties/stub.py @@ -28,6 +28,8 @@ from scripts.thirdparties.src.software.kubernetes_images import KubernetesImagesResolver from scripts.thirdparties.src.software.plugins import ManifestResolver, ManifestsEnrichment from scripts.thirdparties.src.software.thirdparties import ThirdpartyResolver +from scripts.thirdparties.src.software.defaults import KubemarineDefaults + from scripts.thirdparties.src.tracker import SummaryTracker @@ -163,6 +165,7 @@ def store(self): class FakeSynchronization(Synchronization): def __init__(self, + defaults: KubemarineDefaults, compatibility: FakeInternalCompatibility, kubernetes_versions: FakeKubernetesVersions, images_resolver: FakeKubernetesImagesResolver, @@ -171,6 +174,7 @@ def __init__(self, upgrade_config=FakeUpgradeConfig(), ): super().__init__( + defaults, compatibility, kubernetes_versions, images_resolver, diff --git a/test/unit/tools/thirdparties/test_sync.py b/test/unit/tools/thirdparties/test_sync.py index 768e4cb02..3786eae85 100644 --- a/test/unit/tools/thirdparties/test_sync.py +++ b/test/unit/tools/thirdparties/test_sync.py @@ -19,6 +19,7 @@ from copy import deepcopy from typing import List, Dict, ContextManager from unittest import mock + from test.unit import utils as test_utils from test.unit.tools.thirdparties.stub import ( FakeSynchronization, FakeInternalCompatibility, FakeKubernetesVersions, @@ -30,6 +31,7 @@ from kubemarine.plugins import builtin from kubemarine.plugins.manifest import Manifest, Processor, Identity from scripts.thirdparties.src.software import thirdparties, plugins, kubernetes_images +from scripts.thirdparties.src.software.defaults import KubemarineDefaults from scripts.thirdparties.src.software.plugins import ( ManifestResolver, ManifestsEnrichment, ERROR_UNEXPECTED_IMAGE, ERROR_SUSPICIOUS_ABA_VERSIONS @@ -47,6 +49,7 @@ class SynchronizationTest(unittest.TestCase): def setUp(self) -> None: + self.defaults = KubemarineDefaults() self.compatibility = FakeInternalCompatibility() self.kubernetes_versions = FakeKubernetesVersions() self.images_resolver = FakeKubernetesImagesResolver() @@ -580,6 +583,7 @@ def load_compatibility_map_mocked(filename: str) -> dict: def run_sync(self) -> SummaryTracker: return FakeSynchronization( + self.defaults, self.compatibility, self.kubernetes_versions, self.images_resolver,