From 1d3436c8e075f3d545dcd685f879cc0dcc6fdcb7 Mon Sep 17 00:00:00 2001 From: Todd Gill Date: Sat, 3 Feb 2024 16:30:44 -0500 Subject: [PATCH] feat: add support for user to list snapshots that are part of a set return a json list of VGs that have include a list of the LVs that are in the snapshot set. Signed-off-by: Todd Gill --- README.md | 278 +++++++++++++++++++++++++++++++++++++++- tasks/files/snapshot.py | 133 ++++++++++++++++++- tasks/list.yml | 11 ++ tasks/main.yml | 4 + tests/tests_list.yml | 130 +++++++++++++++++++ 5 files changed, 551 insertions(+), 5 deletions(-) create mode 100644 tasks/list.yml create mode 100644 tests/tests_list.yml diff --git a/README.md b/README.md index 5854086..b210720 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,9 @@ If before running the role, with : This variable is required. snapshot_lvm_snapset_name is a string that will be appended to the name of the LV when the snapshot set is created. It will be used -to identify members of the set. +to identify members of the set. It must be at least one character long and contain +valid characters for use in an LVM volume name. A to Z, a to z, 0 to 9, underscore (_), +hyphen (-), dot (.), and plus (+) are valid characters. If before running the role, the following LVs exist: @@ -152,6 +154,280 @@ be removed by the remove command without snapshot_lvm_verify_only. snapshot_lvm_verify_only is intended to be used to double check that the snapshot or remove command have completed the operation correctly. +### Variables Exported by the Role + +#### snapshot_facts + +Contains volume and mount point information for a given snapset. + +For example: + +```json +{ + "volumes": { + "vg3": [ + { + "lv_uuid": "VY7oRQ-zB1q-DzsP-1y7G-J3gL-ci1e-nQXwAy", + "lv_name": "lv1_vg3", + "lv_full_name": "vg3/lv1_vg3", + "lv_path": "/dev/vg3/lv1_vg3", + "lv_size": "1073741824", + "origin": "", + "origin_size": "1073741824", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "owi-a-s---", + "vg_name": "vg3", + "data_percent": "", + "metadata_percent": "" + }, + { + "lv_uuid": "Yhn7RG-k7pM-ylf9-NNt8-xuGI-WwrF-i0Pf6T", + "lv_name": "lv1_vg3_snapset2", + "lv_full_name": "vg3/lv1_vg3_snapset2", + "lv_path": "/dev/vg3/lv1_vg3_snapset2", + "lv_size": "322961408", + "origin": "lv1_vg3", + "origin_size": "1073741824", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "swi-a-s---", + "vg_name": "vg3", + "data_percent": "0.00", + "metadata_percent": "" + }, + { + "lv_uuid": "NlwbxX-NhwK-IHTj-sV9k-ldZY-Twvj-2SiCVe", + "lv_name": "lv2_vg3", + "lv_full_name": "vg3/lv2_vg3", + "lv_path": "/dev/vg3/lv2_vg3", + "lv_size": "1073741824", + "origin": "", + "origin_size": "1073741824", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "owi-a-s---", + "vg_name": "vg3", + "data_percent": "", + "metadata_percent": "" + }, + { + "lv_uuid": "j0RCzX-OVaA-MGDw-ejHO-Eu35-f4yG-VJL2Kr", + "lv_name": "lv2_vg3_snapset2", + "lv_full_name": "vg3/lv2_vg3_snapset2", + "lv_path": "/dev/vg3/lv2_vg3_snapset2", + "lv_size": "322961408", + "origin": "lv2_vg3", + "origin_size": "1073741824", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "swi-aos---", + "vg_name": "vg3", + "data_percent": "0.66", + "metadata_percent": "" + }, + { + "lv_uuid": "8kfTDY-22SL-4tC7-vTsR-1R63-zVzq-55qEL3", + "lv_name": "lv3_vg3", + "lv_full_name": "vg3/lv3_vg3", + "lv_path": "/dev/vg3/lv3_vg3", + "lv_size": "125829120", + "origin": "", + "origin_size": "125829120", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "owi-a-s---", + "vg_name": "vg3", + "data_percent": "", + "metadata_percent": "" + }, + { + "lv_uuid": "babChm-IzEN-Pf8q-1dxk-BJ9R-3kZb-u91utS", + "lv_name": "lv3_vg3_snapset2", + "lv_full_name": "vg3/lv3_vg3_snapset2", + "lv_path": "/dev/vg3/lv3_vg3_snapset2", + "lv_size": "41943040", + "origin": "lv3_vg3", + "origin_size": "125829120", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "swi-a-s---", + "vg_name": "vg3", + "data_percent": "0.00", + "metadata_percent": "" + } + ], + "vg2": [ + { + "lv_uuid": "8uMuRW-1KCV-8FTJ-frhX-X39o-V15B-1uEC98", + "lv_name": "lv1_vg2", + "lv_full_name": "vg2/lv1_vg2", + "lv_path": "/dev/vg2/lv1_vg2", + "lv_size": "1073741824", + "origin": "", + "origin_size": "1073741824", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "owi-a-s---", + "vg_name": "vg2", + "data_percent": "", + "metadata_percent": "" + }, + { + "lv_uuid": "GGssIK-SHYI-to1m-MhVL-2BDk-PJ8X-dBnL7G", + "lv_name": "lv1_vg2_snapset2", + "lv_full_name": "vg2/lv1_vg2_snapset2", + "lv_path": "/dev/vg2/lv1_vg2_snapset2", + "lv_size": "322961408", + "origin": "lv1_vg2", + "origin_size": "1073741824", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "swi-aos---", + "vg_name": "vg2", + "data_percent": "20.97", + "metadata_percent": "" + }, + { + "lv_uuid": "83A9VM-kVEy-sc60-kF14-gKGb-5Ryj-7yDyEG", + "lv_name": "lv2_vg2", + "lv_full_name": "vg2/lv2_vg2", + "lv_path": "/dev/vg2/lv2_vg2", + "lv_size": "83886080", + "origin": "", + "origin_size": "83886080", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "owi-a-s---", + "vg_name": "vg2", + "data_percent": "", + "metadata_percent": "" + }, + { + "lv_uuid": "6tVL8A-U1x1-qqUt-WFVG-POsG-msFs-ZbbPq1", + "lv_name": "lv2_vg2_snapset2", + "lv_full_name": "vg2/lv2_vg2_snapset2", + "lv_path": "/dev/vg2/lv2_vg2_snapset2", + "lv_size": "29360128", + "origin": "lv2_vg2", + "origin_size": "83886080", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "swi-a-s---", + "vg_name": "vg2", + "data_percent": "0.00", + "metadata_percent": "" + } + ], + "vg1": [ + { + "lv_uuid": "UnN0s0-TauJ-csnN-BgC1-3ocI-p8bE-jz0Hd8", + "lv_name": "lv1_vg1", + "lv_full_name": "vg1/lv1_vg1", + "lv_path": "/dev/vg1/lv1_vg1", + "lv_size": "1073741824", + "origin": "", + "origin_size": "1073741824", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "owi-aos---", + "vg_name": "vg1", + "data_percent": "", + "metadata_percent": "" + }, + { + "lv_uuid": "5Np7N9-H15x-Go96-fIwL-E0GR-4fVB-clLDW2", + "lv_name": "lv1_vg1_snapset2", + "lv_full_name": "vg1/lv1_vg1_snapset2", + "lv_path": "/dev/vg1/lv1_vg1_snapset2", + "lv_size": "322961408", + "origin": "lv1_vg1", + "origin_size": "1073741824", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "swi-a-s---", + "vg_name": "vg1", + "data_percent": "20.97", + "metadata_percent": "" + }, + { + "lv_uuid": "P0LPUQ-CljS-hOEm-U749-yyr9-USE7-1qDc2N", + "lv_name": "lv2_vg1", + "lv_full_name": "vg1/lv2_vg1", + "lv_path": "/dev/vg1/lv2_vg1", + "lv_size": "41943040", + "origin": "", + "origin_size": "41943040", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "owi-a-s---", + "vg_name": "vg1", + "data_percent": "", + "metadata_percent": "" + }, + { + "lv_uuid": "FYIBRe-FDiW-PDUE-3l1y-mLzN-bLEg-qF12cz", + "lv_name": "lv2_vg1_snapset2", + "lv_full_name": "vg1/lv2_vg1_snapset2", + "lv_path": "/dev/vg1/lv2_vg1_snapset2", + "lv_size": "16777216", + "origin": "lv2_vg1", + "origin_size": "41943040", + "pool_lv": "", + "lv_tags": "", + "lv_attr": "swi-a-s---", + "vg_name": "vg1", + "data_percent": "0.00", + "metadata_percent": "" + } + ] + }, + "mounts": { + "/dev/vg3/lv1_vg3": null, + "/dev/vg3/lv1_vg3_snapset2": null, + "/dev/vg3/lv2_vg3": null, + "/dev/vg3/lv2_vg3_snapset2": { + "filesystems": [ + { + "target": "/tmp/mp1", + "source": "/dev/mapper/vg3-lv2_vg3_snapset2", + "fstype": "xfs", + "options": "rw,relatime,seclabel,attr2,inode64,logbufs=8,logbsize=32k,noquota" + } + ] + }, + "/dev/vg3/lv3_vg3": null, + "/dev/vg3/lv3_vg3_snapset2": null, + "/dev/vg2/lv1_vg2": null, + "/dev/vg2/lv1_vg2_snapset2": { + "filesystems": [ + { + "target": "/tmp/production_mnt", + "source": "/dev/mapper/vg2-lv1_vg2_snapset2", + "fstype": "xfs", + "options": "rw,relatime,seclabel,attr2,inode64,logbufs=8,logbsize=32k,noquota" + } + ] + }, + "/dev/vg2/lv2_vg2": null, + "/dev/vg2/lv2_vg2_snapset2": null, + "/dev/vg1/lv1_vg1": { + "filesystems": [ + { + "target": "/mount", + "source": "/dev/mapper/vg1-lv1_vg1", + "fstype": "xfs", + "options": "rw,relatime,seclabel,attr2,inode64,logbufs=8,logbsize=32k,noquota" + } + ] + }, + "/dev/vg1/lv1_vg1_snapset2": null, + "/dev/vg1/lv2_vg1": null, + "/dev/vg1/lv2_vg1_snapset2": null + } +} +``` + ## rpm-ostree See README-ostree.md diff --git a/tasks/files/snapshot.py b/tasks/files/snapshot.py index 9f591fd..68cd6e2 100644 --- a/tasks/files/snapshot.py +++ b/tasks/files/snapshot.py @@ -48,6 +48,7 @@ class SnapshotCommand: REMOVE = "remove" REVERT = "revert" EXTEND = "extend" + LIST = "list" class SnapshotStatus: @@ -130,7 +131,6 @@ def make_handler(path, prefix, level): def run_command(argv, stdin=None): logger.info("Running... %s", " ".join(argv)) - try: proc = subprocess.Popen( argv, stdin=stdin, stdout=subprocess.PIPE, close_fds=True @@ -208,6 +208,73 @@ def lvm_full_report_json(): return lvm_json +def lvm_get_fs_mount_points(block_path): + find_mnt_command = [ + "findmnt", + block_path, + "--json", + "--bytes", + "--verbose", + ] + + rc, output = run_command(find_mnt_command) + + if rc: + return None + try: + lvm_json = json.loads(output) + except ValueError as error: + logger.info(error) + raise LvmBug("'find_mnt_command' decode failed : %s" % error.args[0]) + + return lvm_json + + +def lvm_list_snapset_json(all_lvs, vg_name, lv_name, prefix, suffix): + lvm_json = lvm_full_report_json() + report = lvm_json["report"] + vg_dict = dict() + fs_dict = dict() + lv_list = list() + fs_list = list() + top_level = dict() + + # Revert snapshots + for list_item in report: + # The list contains items that are not VGs + try: + list_item["vg"] + except KeyError: + continue + vg = list_item["vg"][0]["vg_name"] + if vg_name and vg != vg_name: + continue + + for lv_item in list_item["lv"]: + lv_item_name = lv_item["lv_name"] + if lv_name and lv_name != lv_item_name: + continue + + lv_list.append(lv_item) + + if len(lv_list) > 0: + vg_dict[vg] = lv_list + lv_list = [] + + for lv_list in vg_dict.items(): + for lv_item in lv_list[1]: + block_path = lv_item["lv_path"] + fs_mount_points = lvm_get_fs_mount_points(block_path) + if fs_mount_points: + fs_list.append(fs_mount_points) + fs_dict[block_path] = fs_mount_points + fs_list = list() + + top_level["volumes"] = vg_dict + top_level["mounts"] = fs_dict + return json.dumps(top_level, indent=4) + + def get_snapshot_name(lv_name, prefix, suffix): if prefix: prefix_str = prefix @@ -680,6 +747,20 @@ def extend_lvs(vg_name, lv_name, prefix, suffix, required_space): return SnapshotStatus.SNAPSHOT_OK, "" +def list_set(all_lvs, snapset_json): + snapset_name = snapset_json["name"] + lvm_json = lvm_list_snapset_json(all_lvs, None, None, None, snapset_name) + print(lvm_json) + + return SnapshotStatus.SNAPSHOT_OK, "" + + +def list_lvs(all_lvs, vg_name, lv_name, prefix, suffix): + lvm_json = lvm_list_snapset_json(all_lvs, vg_name, lv_name, prefix, suffix) + print(lvm_json) + return SnapshotStatus.SNAPSHOT_OK, "" + + def snapshot_lv(vg_name, lv_name, prefix, suffix, snap_size): snapshot_name = get_snapshot_name(lv_name, prefix, suffix) @@ -939,8 +1020,7 @@ def revert_snapshot_set(snapset_json): vg = list_item["vg"] lv = list_item["lv"] - rc, message = revert_lv( - vg, lv, None, get_snapshot_name(lv, None, snapset_name)) + rc, message = revert_lv(vg, lv, None, get_snapshot_name(lv, None, snapset_name)) if rc != SnapshotStatus.SNAPSHOT_OK: return rc, message @@ -1488,6 +1568,10 @@ def validate_args(args): print("One of --prefix or --suffix is required : ", args.operation) sys.exit(SnapshotStatus.ERROR_CMD_INVALID) + if len(args.suffix) == 0: + print("Snapset name must be provided : ", args.operation) + sys.exit(SnapshotStatus.ERROR_CMD_INVALID) + # not all commands include required_space if hasattr(args, "required_space"): rc, message, _required_space = get_required_space(args.required_space) @@ -1590,6 +1674,8 @@ def validate_snapset_json(cmd, snapset, verify_only): rc, message = validate_json_request(snapset_json, False) elif cmd == SnapshotCommand.REMOVE: rc, message = validate_json_request(snapset_json, False) + elif cmd == SnapshotCommand.LIST: + rc, message = validate_json_request(snapset_json, False) else: rc = SnapshotStatus.ERROR_UNKNOWN_FAILURE message = "validate_snapset_json" @@ -1611,6 +1697,7 @@ def snapshot_cmd(args): ) if args.set_json is None: + validate_args(args) rc, message, required_space = get_required_space(args.required_space) if rc != SnapshotStatus.SNAPSHOT_OK: return rc, message @@ -1694,6 +1781,7 @@ def remove_cmd(args): ) if args.set_json is None: + validate_args(args) if args.all and args.volume_group: print("-all and --volume_group are mutually exclusive: ", args.operation) sys.exit(1) @@ -1813,9 +1901,38 @@ def extend_cmd(args): return rc, message +def list_cmd(args): + logger.info( + "list_cmd: %d %s %s %s %s %s %s", + args.all, + args.operation, + args.volume_group, + args.logical_volume, + args.suffix, + args.prefix, + args.set_json, + ) + + if args.set_json is None: + validate_args(args) + rc, message = list_lvs( + args.all, + args.volume_group, + args.logical_volume, + args.prefix, + args.suffix, + ) + else: + rc, message, snapset_json = validate_snapset_json( + SnapshotCommand.LIST, args.set_json, False + ) + rc, message = list_set(args.all, snapset_json) + + return rc, message + + if __name__ == "__main__": set_up_logging() - # Ensure that we get consistent output for parsing stdout/stderr and that we # are using the lvmdbusd profile. os.environ["LC_ALL"] = "C" @@ -1954,6 +2071,14 @@ def extend_cmd(args): ) extend_parser.set_defaults(func=extend_cmd) + # sub-parser for 'list' + list_parser = subparsers.add_parser( + SnapshotCommand.LIST, + help="List snapshots", + parents=[common_parser], + ) + list_parser.set_defaults(func=list_cmd) + args = parser.parse_args() return_code, display_message = args.func(args) print_result(return_code, display_message) diff --git a/tasks/list.yml b/tasks/list.yml new file mode 100644 index 0000000..48a5383 --- /dev/null +++ b/tasks/list.yml @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: MIT +--- +- name: List snapshots + ansible.builtin.script: "{{ __snapshot_cmd }}" + args: + executable: "{{ __snapshot_python }}" + register: snapshot_cmd + +- name: Set snapshot_facts to the JSON results + set_fact: + snapshot_facts: "{{ snapshot_cmd.stdout | from_json }}" diff --git a/tasks/main.yml b/tasks/main.yml index ce6a116..c260e00 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -34,6 +34,10 @@ ansible.builtin.include_tasks: revert.yml when: snapshot_lvm_action == "revert" +- name: List Snapshot Attributes + ansible.builtin.include_tasks: list.yml + when: snapshot_lvm_action == "list" + - name: Extend Snapshots ansible.builtin.include_tasks: extend.yml when: snapshot_lvm_action == "extend" diff --git a/tests/tests_list.yml b/tests/tests_list.yml new file mode 100644 index 0000000..2d913da --- /dev/null +++ b/tests/tests_list.yml @@ -0,0 +1,130 @@ +--- +- name: Basic snapshot test + hosts: all + tasks: + - name: Run tests + block: + - name: Run the storage role to create test LVs + include_role: + name: fedora.linux_system_roles.storage + + - name: Get unused disks + include_tasks: get_unused_disk.yml + vars: + min_size: "1g" + min_return: 10 + + - name: Set disk lists + set_fact: + disk_list_1: "{{ range(0, 3) | map('extract', unused_disks) | + list }}" + disk_list_2: "{{ range(3, 6) | map('extract', unused_disks) | + list }}" + disk_list_3: "{{ range(6, 10) | map('extract', unused_disks) | + list }}" + + - name: Create LVM logical volumes under volume groups + include_role: + name: fedora.linux_system_roles.storage + vars: + storage_pools: + - name: test_vg1 + disks: "{{ disk_list_1 }}" + volumes: + - name: lv1 + size: "15%" + - name: lv2 + size: "50%" + - name: test_vg2 + disks: "{{ disk_list_2 }}" + volumes: + - name: lv3 + size: "10%" + - name: lv4 + size: "20%" + - name: test_vg3 + disks: "{{ disk_list_3 }}" + volumes: + - name: lv5 + size: "30%" + - name: lv6 + size: "25%" + - name: lv7 + size: "10%" + - name: lv8 + size: "10%" + + - name: Run the snapshot role to create snapshot LVs + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_percent_space_required: 15 + snapshot_lvm_all_vgs: true + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_action: snapshot + + - name: Verify the snapshot LVs are created + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_all_vgs: true + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_verify_only: true + snapshot_lvm_action: check + + - name: List + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_action: list + + - name: Run the snapshot role remove the snapshot LVs + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_action: remove + + - name: Use the snapshot_lvm_verify option to make sure remove is done + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_verify_only: true + snapshot_lvm_action: remove + always: + - name: Remove storage volumes + include_role: + name: fedora.linux_system_roles.storage + vars: + storage_safe_mode: false + storage_pools: + - name: test_vg1 + disks: "{{ disk_list_1 }}" + state: absent + volumes: + - name: lv1 + state: absent + - name: lv2 + state: absent + - name: test_vg2 + disks: "{{ disk_list_2 }}" + state: absent + volumes: + - name: lv3 + state: absent + - name: lv4 + state: absent + - name: test_vg3 + disks: "{{ disk_list_3 }}" + state: absent + volumes: + - name: lv5 + state: absent + - name: lv6 + state: absent + - name: lv7 + state: absent + - name: lv8 + state: absent