feat: add cephadm support

.github/workflows/molecule.yml

@@ -28,7 +28,7 @@ jobs:
       matrix:
         version:
           - 16.2.9
-          - 17.2.6
+          - 17.2.7
         scenario:
           - aio
           - ha

galaxy.yml

@@ -10,6 +10,7 @@ license:
 dependencies:
   community.general: ">=4.5.0"
   ansible.utils: 2.9.0
+  vexxhost.containers: ">=1.1.0"
 tags:
   - application
   - cloud

molecule/ceph/converge.yml

@@ -13,3 +13,5 @@
 # under the License.
 
 - ansible.builtin.import_playbook: vexxhost.ceph.site
+  vars:
+    ceph_public_network: 172.17.0.0/16

molecule/ceph/molecule.yml

@@ -0,0 +1,65 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: instance
+    image: geerlingguy/docker-${MOLECULE_DISTRO:-ubuntu2004}-ansible:latest
+    command: ${MOLECULE_DOCKER_COMMAND:-""}
+    privileged: true
+    cgroupns_mode: host
+    pre_build_image: true
+    environment:
+      container: docker
+    security_opts:
+      - apparmor=unconfined
+    volumes:
+      - /dev:/dev
+      - /lib/modules:/lib/modules:ro
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+      - /usr/src:/usr/src:ro
+    groups:
+      - controllers
+      - cephs
+provisioner:
+  name: ansible
+  config_options:
+    connection:
+      pipelining: true
+  inventory:
+    group_vars:
+      all:
+        ceph_fsid: ${MOLECULE_CEPH_FSID:-"1dff0e0f-3c44-48da-81cd-4f3c6e8722b2"}
+        ceph_version: ${MOLECULE_CEPH_VERSION:-"16.2.9"}
+        ceph_conf_overrides:
+          - section: global
+            option: osd crush chooseleaf type
+            value: 0
+          - section: mon
+            option: auth allow insecure global id reclaim
+            value: false
+          - section: mon
+            option: mon data avail warn
+            value: 10
+      controllers:
+      cephs:
+        ceph_osd_devices:
+          - "/dev/ceph-{{ inventory_hostname_short }}-osd0/data"
+          - "/dev/ceph-{{ inventory_hostname_short }}-osd1/data"
+          - "/dev/ceph-{{ inventory_hostname_short }}-osd2/data"
+verifier:
+  name: ansible

playbooks/site.yml

@@ -16,6 +16,9 @@
   hosts: "{{ ceph_control_plane_group | default('controllers') }}"
   become: true
   roles:
+    - role: cephadm
+      tags:
+        - cephadm
     - role: mon
       tags:
         - ceph-mon

requirements.txt

@@ -1,4 +1,5 @@
 ansible-core
+docker-image-py
 jmespath
 molecule
 molecule-plugins[docker]

roles/cephadm/README.md

@@ -0,0 +1 @@
+# `cephadm`

roles/mon/defaults/main.yml → roles/cephadm/defaults/main.yml

@@ -24,3 +24,10 @@ ceph_mon_fsid: "{{ ceph_fsid }}"
 # Network definitions
 ceph_mon_public_network: "{{ ceph_public_network | default('0.0.0.0/0') }}"
 ceph_mon_cluster_network: "{{ ceph_cluster_network | default(ceph_mon_public_network) }}"
+
+# Make sure we plan to adopt exist cluster
+adopt_exists: true
+
+# This only required when /etc/ceph/ceph.conf are different across hosts.
+adopt_config_seperatly: false
+adopt_image: quay.io/ceph/ceph@sha256:599d706c76d7c3379f61c477b5a61a568bf5b5c6863e0f905dc88d971f129a9b  # ceph v16.2.14

roles/cephadm/meta/main.yml

@@ -0,0 +1,29 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+galaxy_info:
+  author: VEXXHOST, Inc.
+  description: Ansible role for Cephadm
+  license: Apache-2.0
+  min_ansible_version: 5.5.0
+  standalone: false
+  platforms:
+    - name: Ubuntu
+      versions:
+        - focal
+        - jammy
+
+dependencies:
+  - role: vexxhost.containers.docker
+  - role: repository

roles/cephadm/tasks/adoption.yml

@@ -0,0 +1,64 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Prepare host for cephadm
+  ansible.builtin.command: cephadm prepare-host
+
+- name: Wait for prepare host process complete
+  ansible.builtin.command: cephadm ls
+  register: convert_status
+  until: >
+    '"style": "legacy"' not in convert_status.stdout
+  retries: 120
+  delay: 5
+  changed_when: false
+  failed_when: convert_status.rc > 1
+
+- name: Assimilate existing configs
+  ansible.builtin.command: ceph config assimilate-conf -i /etc/ceph/ceph.conf
+  when:
+    - (inventory_hostname == groups[ceph_mon_group][0] and
+      not adopt_config_seperatly) or adopt_config_seperatly
+
+- name: Adopt monitor to cluster
+  ansible.builtin.command:
+    cmd: |
+      cephadm adopt --style legacy --name "mon.{{ item }}"
+  loop: groups[ceph_mon_group]
+
+- name: Adopt manager to cluster
+  ansible.builtin.command:
+    cmd: |
+      cephadm adopt --style legacy --name "mgr.{{ item }}"
+  loop: groups[ceph_mon_group]
+
+- name: Ceph mgr module enable cephadm
+  ansible.builtin.command: ceph mgr module enable cephadm
+
+- name: Ceph orch set backend cephadm
+  ansible.builtin.command: ceph orch set backend cephadm
+
+- name: Ceph cephadm generate-key
+  ansible.builtin.command: ceph cephadm generate-key
+
+- name: Get Ceph cephadm public key
+  ansible.builtin.command: ceph cephadm get-pub-key
+  register: ceph_new_gen_pub_key
+
+- name: Set global var `ceph_ssh_pub_key` from adopt
+  ansible.builtin.set_fact:
+    ceph_ssh_pub_key: "{{ ceph_new_gen_pub_key }}"
+  delegate_to: "{{ item }}"
+  with_items: "{{ ansible_play_hosts }}"
+  run_once: true

roles/cephadm/tasks/bootstrap-ceph.yml

@@ -0,0 +1,129 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Set ceph monitor ip address
+  ansible.builtin.set_fact:
+    ceph_mon_ip_address: "{{ ansible_all_ipv4_addresses | ansible.utils.ipaddr(ceph_mon_public_network) | first }}"
+
+- name: Generate temporary file for ceph cfg
+  become: true
+  ansible.builtin.tempfile:
+    state: file
+    prefix: ceph_cfg
+  register: _ceph_cfg_tempfile
+
+- name: Generate basic configuration file
+  community.general.ini_file:
+    path: "{{ _ceph_cfg_tempfile.path }}"
+    section: global
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: "0640"
+  loop:
+    - option: fsid
+      value: "{{ ceph_mon_fsid }}"
+    - option: public_network
+      value: "{{ ceph_mon_public_network }}"
+    - option: cluster_network
+      value: "{{ ceph_mon_cluster_network }}"
+
+- name: Include extra configuration values
+  ansible.builtin.ini_file:
+    path: "{{ _ceph_cfg_tempfile.path }}"
+    section: "{{ item.section }}"
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: "0640"
+  loop: "{{ ceph_mon_conf_overrides }}"
+
+- name: Run Bootstrap coomand
+  ansible.builtin.command:
+    cmd: cephadm bootstrap --mon-ip {{ ceph_mon_ip_address }} --config {{ _ceph_cfg_tempfile.path }} --allow-overwrite
+  changed_when: false
+
+- name: Validate bootstrap
+  ansible.builtin.command: ceph orch status
+  register: bootstrap_status
+  until: >
+    'Available: Yes' in bootstrap_status.stdout and
+    'Backend: cephadm' in bootstrap_status.stdout
+  retries: 120
+  delay: 5
+  changed_when: false
+  failed_when: bootstrap_status.rc > 1
+
+- name: Load ssh pub key
+  ansible.builtin.slurp:
+    src: /etc/ceph/ceph.pub
+  register: ssh_pub_key
+
+- name: Set global var `ceph_ssh_pub_key`
+  ansible.builtin.set_fact:
+    ceph_ssh_pub_key: "{{ ssh_pub_key['content'] | b64decode }}"
+  delegate_to: "{{ item }}"
+  with_items: "{{ ansible_play_hosts }}"
+  run_once: true
+
+- name: Remove the temporary config file
+  ansible.builtin.file:
+    path: "{{ _ceph_cfg_tempfile.path }}"
+    state: absent
+  when: _ceph_cfg_tempfile.path is defined
+
+- name: Add osd hosts to ceph cluster
+  ansible.builtin.command:
+    ceph orch host add "{{ inventory_hostname_short }}" "{{ item }}" --labels=osd  # yamllint disable-line rule:line-length
+  loop: "{{ groups[ceph_osd_group] | map('extract', hostvars, ['osd_ip']) }}"  # yamllint disable-line rule:line-length
+  changed_when: false
+
+- name: Generate temporary file for osd spec
+  ansible.builtin.tempfile:
+    state: file
+    prefix: ceph_osd_spec
+  register: _ceph_osd_spec_tmp
+  changed_when: false
+
+- name: Write osd spec file
+  become: true
+  ansible.builtin.file:
+    src: "{{ lookup('ansible.builtin.file', ceph_osd_spec_file) }}"
+    dest: "{{ _ceph_osd_spec_tmp.path }}"
+    mode: '0640'
+  when:
+    - ceph_osd_spec_file is defined
+  changed_when: false
+
+- name: Write osd spec file
+  become: true
+  ansible.builtin.template:
+    src: osd_spec.j2
+    dest: "{{ _ceph_osd_spec_tmp.path }}"
+    mode: '0640'
+  changed_when: false
+  when:
+    - ceph_osd_spec_file is not defined
+
+- name: Apply osd spec file
+  ansible.builtin.command:
+    cmd: ceph orch apply -i {{ _ceph_osd_spec_tmp.path }}
+  when:
+    - ceph_osd_spec_file is defined
+  changed_when: false
+
+- name: Remove the temporary config file
+  ansible.builtin.file:
+    path: "{{ _ceph_osd_spec_tmp.path }}"
+    state: absent
+  when: _ceph_osd_spec_tmp.path is defined
+  changed_when: false