diff --git a/library/snapshot.py b/library/snapshot.py index 0a7f31a..e4d4cc6 100644 --- a/library/snapshot.py +++ b/library/snapshot.py @@ -4,18 +4,16 @@ # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import absolute_import, division, print_function -from os.path import join as path_join -import sys -import subprocess -import stat -import re -import os -import math -import logging -import json import argparse - -__metaclass__ = type +import json +import logging +import math +import os +import re +import stat +import subprocess +import sys +from os.path import join as path_join from ansible.module_utils.basic import AnsibleModule @@ -25,7 +23,7 @@ "supported_by": "community", } -DOCUMENTATION = """ +DOCUMENTATION = r""" --- module: snapshot @@ -120,7 +118,8 @@ - Todd Gill (tgill@redhat.com) """ -EXAMPLES = """ + +EXAMPLES = r""" # Create Snapshots of all VGs --- - name: Extend all snapshots @@ -154,7 +153,7 @@ """ -RETURN = """ +RETURN = r""" # Examples of possible return values. msg: description: On success an empty string. On failure a message to @@ -176,13 +175,15 @@ """ +__metaclass__ = type + + logger = logging.getLogger("snapshot-role") LVM_NOTFOUND_RC = 5 MAX_LVM_NAME = 127 CHUNK_SIZE = 65536 DEV_PREFIX = "/dev" -VG_INCLUDE = None # Minimum LVM snapshot size (512MiB) LVM_MIN_SNAPSHOT_SIZE = 512 * 1024**2 @@ -562,7 +563,7 @@ def lvm_get_fs_mount_points(block_path): return mount_list -def vgs_lvs_iterator(vg_name, lv_name, omit_empty_lvs=False): +def vgs_lvs_iterator(vg_name, lv_name, vg_include, omit_empty_lvs=False): """Return an iterator which returns tuples. The first element in the tuple is the vg object matching given vg_name, or all vgs if vg_name is None. The second element is a list of @@ -578,7 +579,7 @@ def vgs_lvs_iterator(vg_name, lv_name, omit_empty_lvs=False): vg and vg["vg_name"] and (not vg_name or vg_name == vg["vg_name"]) - and (not VG_INCLUDE or VG_INCLUDE.search(vg["vg_name"])) + and (not vg_include or vg_include.search(vg["vg_name"])) ): lvs = [ lv @@ -625,6 +626,54 @@ def get_snapshot_name(lv_name, suffix): return lv_name + "_" + suffix_str +def lvm_get_attr(vg_name, lv_name): + lvs_command = ["lvs", "--reportformat", "json", vg_name + "/" + lv_name] + + rc, output = run_command(lvs_command) + + if rc == LVM_NOTFOUND_RC: + return SnapshotStatus.SNAPSHOT_OK, False + + if rc: + return SnapshotStatus.ERROR_LVS_FAILED, None + + try: + lvs_json = json.loads(output) + except ValueError as error: + logger.info(error) + message = "lvm_is_snapshot: json decode failed : %s" % error.args[0] + return SnapshotStatus.ERROR_JSON_PARSER_ERROR, message + + lv_list = lvs_json["report"] + + if len(lv_list) > 1 or len(lv_list[0]["lv"]) > 1: + raise LvmBug("'lvs' returned more than 1 lv '%d'" % rc) + + lv = lv_list[0]["lv"][0] + + lv_attr = lv["lv_attr"] + + if len(lv_attr) == 0: + raise LvmBug("'lvs' zero length attr : '%d'" % rc) + + return SnapshotStatus.SNAPSHOT_OK, lv_attr + + +def lvm_is_thinpool(vg_name, lv_name): + rc, lv_attr = lvm_get_attr(vg_name, lv_name) + + if rc == LVM_NOTFOUND_RC: + return SnapshotStatus.SNAPSHOT_OK, False + + if rc: + return SnapshotStatus.ERROR_LVS_FAILED, None + + if lv_attr[0] == "t": + return SnapshotStatus.SNAPSHOT_OK, True + else: + return SnapshotStatus.SNAPSHOT_OK, False + + def lvm_lv_exists(vg_name, lv_name): vg_exists = False lv_exists = False @@ -698,10 +747,8 @@ def lvm_is_inuse(vg_name, lv_name): return SnapshotStatus.SNAPSHOT_OK, False -def lvm_is_snapshot(vg_name, snapshot_name): - lvs_command = ["lvs", "--reportformat", "json", vg_name + "/" + snapshot_name] - - rc, output = run_command(lvs_command) +def lvm_is_snapshot(vg_name, lv_name): + rc, lv_attr = lvm_get_attr(vg_name, lv_name) if rc == LVM_NOTFOUND_RC: return SnapshotStatus.SNAPSHOT_OK, False @@ -709,25 +756,6 @@ def lvm_is_snapshot(vg_name, snapshot_name): if rc: return SnapshotStatus.ERROR_LVS_FAILED, None - try: - lvs_json = json.loads(output) - except ValueError as error: - logger.info(error) - message = "lvm_is_snapshot: json decode failed : %s" % error.args[0] - return SnapshotStatus.ERROR_JSON_PARSER_ERROR, message - - lv_list = lvs_json["report"] - - if len(lv_list) > 1 or len(lv_list[0]["lv"]) > 1: - raise LvmBug("'lvs' returned more than 1 lv '%d'" % rc) - - lv = lv_list[0]["lv"][0] - - lv_attr = lv["lv_attr"] - - if len(lv_attr) == 0: - raise LvmBug("'lvs' zero length attr : '%d'" % rc) - if lv_attr[0] == "s": return SnapshotStatus.SNAPSHOT_OK, True else: @@ -765,7 +793,13 @@ def revert_lv(vg_name, snapshot_name, check_mode): raise LvmBug("'lvs' failed '%d'" % rc) if lv_exists: - if not lvm_is_snapshot(vg_name, snapshot_name): + rc, is_snapshot = lvm_is_snapshot(vg_name, snapshot_name) + if rc != SnapshotStatus.SNAPSHOT_OK: + return ( + SnapshotStatus.ERROR_VERIFY_COMMAND_FAILED, + "revert_lv: command failed for LV lvm_is_snapshot()", + ) + if not is_snapshot: return ( SnapshotStatus.ERROR_REVERT_FAILED, "LV with name: " + vg_name + "/" + snapshot_name + " is not a snapshot", @@ -796,7 +830,14 @@ def extend_lv_snapshot(vg_name, lv_name, suffix, percent_space_required, check_m changed = False if lv_exists: - if not lvm_is_snapshot(vg_name, snapshot_name): + rc, is_snapshot = lvm_is_snapshot(vg_name, snapshot_name) + if rc != SnapshotStatus.SNAPSHOT_OK: + return ( + SnapshotStatus.ERROR_VERIFY_COMMAND_FAILED, + "extend_lv_snapshot: command failed for LV lvm_is_snapshot()", + changed, + ) + if not is_snapshot: return ( SnapshotStatus.ERROR_EXTEND_NOT_SNAPSHOT, "LV with name: " + vg_name + "/" + snapshot_name + " is not a snapshot", @@ -937,7 +978,13 @@ def snapshot_lv(vg_name, lv_name, suffix, snap_size, check_mode): rc, _vg_exists, lv_exists = lvm_lv_exists(vg_name, snapshot_name) if lv_exists: - if lvm_is_snapshot(vg_name, snapshot_name): + rc, is_snapshot = lvm_is_snapshot(vg_name, snapshot_name) + if rc != SnapshotStatus.SNAPSHOT_OK: + return ( + SnapshotStatus.ERROR_VERIFY_COMMAND_FAILED, + "snapshot_lv: command failed for LV lvm_is_snapshot()", + ) + if is_snapshot: return ( SnapshotStatus.ERROR_ALREADY_EXISTS, "Snapshot of :" + vg_name + "/" + lv_name + " already exists", @@ -1012,27 +1059,6 @@ def check_name_for_snapshot(lv_name, suffix): return SnapshotStatus.SNAPSHOT_OK, "" -def check_lvs(required_space, vg_name, lv_name, suffix): - # Check to make sure all the source vgs/lvs exist - rc, message = verify_source_lvs_exist(vg_name, lv_name) - if rc != SnapshotStatus.SNAPSHOT_OK: - return rc, message - - for vg, lv_list in vgs_lvs_iterator(vg_name, lv_name): - for lv in lv_list: - rc, message = check_name_for_snapshot(lv["lv_name"], suffix) - if rc != SnapshotStatus.SNAPSHOT_OK: - return rc, message - - if check_space_for_snapshots(vg, lv_list, lv_name, required_space): - return ( - SnapshotStatus.ERROR_INSUFFICIENT_SPACE, - "insufficient space for snapshots", - ) - - return SnapshotStatus.SNAPSHOT_OK, "" - - # Verify that the set has been created def check_verify_lvs_set(snapset_json): snapset_name = snapset_json["name"] @@ -1081,11 +1107,11 @@ def check_verify_lvs_set(snapset_json): return SnapshotStatus.SNAPSHOT_OK, "" -def check_verify_lvs_completed(snapshot_all, vg_name, lv_name, suffix): +def check_verify_lvs_completed(snapshot_all, vg_name, lv_name, vg_include, suffix): vg_found = False lv_found = False - for vg, lv_list in vgs_lvs_iterator(vg_name, lv_name): + for vg, lv_list in vgs_lvs_iterator(vg_name, lv_name, vg_include): vg_found = True verify_vg_name = vg["vg_name"] @@ -1541,7 +1567,7 @@ def remove_verify_snapshot_set(snapset_json): return SnapshotStatus.SNAPSHOT_OK, "" -def remove_verify_snapshots(vg_name, lv_name, suffix): +def remove_verify_snapshots(vg_name, lv_name, vg_include, suffix): # if the vg_name and lv_name are supplied, make sure the source is not a snapshot if vg_name and lv_name: rc, is_snapshot = lvm_is_snapshot(vg_name, lv_name) @@ -1556,7 +1582,7 @@ def remove_verify_snapshots(vg_name, lv_name, suffix): "source is a snapshot:" + vg_name + "/" + lv_name, ) - for vg, lv_list in vgs_lvs_iterator(vg_name, lv_name): + for vg, lv_list in vgs_lvs_iterator(vg_name, lv_name, vg_include): verify_vg_name = vg["vg_name"] for lvs in lv_list: @@ -1595,7 +1621,7 @@ def remove_verify_snapshots(vg_name, lv_name, suffix): def get_current_space_state(): vg_size_dict = dict() - for volume_group, lv_list in vgs_lvs_iterator(None, None): + for volume_group, lv_list in vgs_lvs_iterator(None, None, None): vg_name = volume_group["vg_name"] vg_space = VGSpaceState() @@ -1988,7 +2014,7 @@ def validate_umount_args(module_args): return SnapshotStatus.SNAPSHOT_OK, "" -def validate_snapset_args(cmd, module_args): +def validate_snapset_args(cmd, module_args, vg_include): rc, message = validate_general_args(module_args) if rc != SnapshotStatus.SNAPSHOT_OK: @@ -2007,7 +2033,7 @@ def validate_snapset_args(cmd, module_args): if rc != SnapshotStatus.SNAPSHOT_OK: return {"return_code": rc, "errors": [message], "changed": False}, None - rc, message, snapset_dict = get_json_from_args(module_args) + rc, message, snapset_dict = get_json_from_args(module_args, vg_include) return {"return_code": rc, "errors": [message], "changed": False}, snapset_dict @@ -2140,7 +2166,7 @@ def validate_snapset_json(cmd, snapset, verify_only): return {"return_code": rc, "errors": [message], "changed": False}, snapset_dict -def get_json_from_args(module_args): +def get_json_from_args(module_args, vg_include): volume_list = [] args_json = {} cmd = get_command_const(module_args["snapshot_lvm_action"]) @@ -2150,13 +2176,13 @@ def get_json_from_args(module_args): module_args["snapshot_lvm_vg"], module_args["snapshot_lvm_lv"] ) if rc != SnapshotStatus.SNAPSHOT_OK: - return rc, message, "" + return (rc, message, "") if module_args["snapshot_lvm_snapset_name"]: args_json["name"] = module_args["snapshot_lvm_snapset_name"] for vg, lv_list in vgs_lvs_iterator( - module_args["snapshot_lvm_vg"], module_args["snapshot_lvm_lv"] + module_args["snapshot_lvm_vg"], module_args["snapshot_lvm_lv"], vg_include ): vg_str = vg["vg_name"] for lv in lv_list: @@ -2164,8 +2190,26 @@ def get_json_from_args(module_args): if lv["lv_name"].endswith(module_args["snapshot_lvm_snapset_name"]): continue + rc, is_snapshot = lvm_is_snapshot(vg_str, lv["lv_name"]) + if rc != SnapshotStatus.SNAPSHOT_OK: + return ( + SnapshotStatus.ERROR_VERIFY_COMMAND_FAILED, + "get_json_from_args: command failed for LV lvm_is_snapshot()", + None, + ) + + if is_snapshot: continue + rc, is_thinpool = lvm_is_thinpool(vg_str, lv["lv_name"]) + if rc != SnapshotStatus.SNAPSHOT_OK: + return ( + SnapshotStatus.ERROR_VERIFY_COMMAND_FAILED, + "get_json_from_args: command failed for LV lvm_is_thinpool()", + None, + ) + if is_thinpool: + continue volume = {} volume["name"] = ("snapshot : " + vg_str + "/" + lv["lv_name"],) volume["vg"] = vg_str @@ -2317,6 +2361,7 @@ def umount_cmd(module_args, snapset_dict): def run_module(): logger.info("run_module()") + vg_include = None # define available arguments/parameters a user can pass to the module # available arguments/parameters that a user can pass module_args = dict( @@ -2351,16 +2396,16 @@ def run_module(): module.exit_json(**result) if module.params["snapshot_lvm_vg_include"]: - VG_INCLUDE = re.compile(module.params["snapshot_lvm_vg_include"]) + vg_include = re.compile(module.params["snapshot_lvm_vg_include"]) - if module.params["snapshot_lvm_set"]: + if module.params["snapshot_lvm_set"] and len(module.params["snapshot_lvm_set"]) > 3: cmd_result, snapset_dict = validate_snapset_json( get_command_const(module.params["snapshot_lvm_action"]), - module.params["snapshot_lvm_set"].replace("'", '"'), + module.params["snapshot_lvm_set"], False, ) else: - cmd_result, snapset_dict = validate_snapset_args(cmd, module.params) + cmd_result, snapset_dict = validate_snapset_args(cmd, module.params, vg_include) if cmd_result["return_code"] == SnapshotStatus.SNAPSHOT_OK: if cmd == SnapshotCommand.SNAPSHOT: @@ -2399,7 +2444,6 @@ def run_module(): def main(): set_up_logging() - # Ensure that we get consistent output for parsing stdout/stderr and that we # are using the lvmdbusd profile. os.environ["LC_ALL"] = "C" diff --git a/tasks/main.yml b/tasks/main.yml index e7989b8..25954c3 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -19,7 +19,7 @@ snapshot_lvm_all_vgs: "{{ snapshot_lvm_all_vgs | d(false) }}" snapshot_lvm_verify_only: "{{ snapshot_lvm_verify_only | d(false) }}" snapshot_lvm_mount_origin: "{{ snapshot_lvm_mount_origin | d(false) }}" - snapshot_lvm_mountpoint_create: + snapshot_lvm_mountpoint_create: >- "{{ snapshot_lvm_mountpoint_create | d(false) }}" snapshot_lvm_unmount_all: "{{ snapshot_lvm_unmount_all | d(false) }}" snapshot_lvm_percent_space_required: @@ -30,7 +30,7 @@ snapshot_lvm_mount_options: "{{ snapshot_lvm_mount_options | d(omit) }}" snapshot_lvm_fstype: "{{ snapshot_lvm_fstype | d(omit) }}" snapshot_lvm_mountpoint: "{{ snapshot_lvm_mountpoint | d(omit) }}" - snapshot_lvm_set: "{{ snapshot_lvm_set | d(omit) }}" + snapshot_lvm_set: "{{ snapshot_lvm_set | to_json }}" snapshot_lvm_vg_include: "{{ snapshot_lvm_vg_include | d(false) }}" register: snapshot_cmd_raw diff --git a/tests/tests_set_thin_basic.yml b/tests/tests_set_thin_basic.yml new file mode 100644 index 0000000..f975a96 --- /dev/null +++ b/tests/tests_set_thin_basic.yml @@ -0,0 +1,128 @@ +--- +- name: Snapshot a set of logical volumes across different volume groups + hosts: all + vars: + test_disk_min_size: "1g" + test_disk_count: 10 + test_storage_pools: + - name: test_vg1 + disks: "{{ range(0, 6) | map('extract', unused_disks) | list }}" + volumes: + - name: thin_pool_dev + thin_pool_name: thin_pool + thin_pool_size: '3g' + size: "1g" + thin: true + - name: lv2 + thin_pool_name: thin_pool + size: "15" + thin: true + - name: lv3 + thin_pool_name: thin_pool + size: "15%" + thin: true + - name: lv4 + thin_pool_name: thin_pool + size: "15%" + thin: true + - name: test_vg2 + disks: "{{ range(6, 10) | map('extract', unused_disks) | list }}" + volumes: + - name: lv5 + size: "30%" + - name: lv6 + size: "25%" + - name: lv7 + size: "10%" + - name: lv8 + size: "10%" + snapshot_test_set: + name: snapset1 + volumes: + - name: snapshot VG1 LV2 + vg: test_vg1 + lv: lv2 + percent_space_required: 20 + - name: snapshot VG1 LV3 + vg: test_vg1 + lv: lv3 + percent_space_required: 20 + - name: snapshot VG2 LV5 + vg: test_vg2 + lv: lv5 + percent_space_required: 15 + - name: snapshot VG2 LV8 + vg: test_vg2 + lv: lv8 + percent_space_required: 15 + thin_pool: thin_pool + + tasks: + - name: Run tests + block: + - name: Setup + include_tasks: tasks/setup.yml + + - name: Run the snapshot role to create snapshot set of LVs + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_action: snapshot + snapshot_lvm_set: "{{ snapshot_test_set }}" + + - name: Assert changes for create snapset + assert: + that: snapshot_cmd["changed"] + + - name: Run the snapshot role to verify the set of snapshots for the LVs + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_action: check + snapshot_lvm_set: "{{ snapshot_test_set }}" + snapshot_lvm_verify_only: true + + - name: Create snapset again for idempotence + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_action: snapshot + snapshot_lvm_set: "{{ snapshot_test_set }}" + + - name: Assert no changes for create snapset + assert: + that: not snapshot_cmd["changed"] + + - name: Run the snapshot role remove the set + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_action: remove + snapshot_lvm_set: "{{ snapshot_test_set }}" + + - name: Assert changes for remove snapset + assert: + that: snapshot_cmd["changed"] + + - name: Run the snapshot role to verify the set is removed + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_action: remove + snapshot_lvm_set: "{{ snapshot_test_set }}" + snapshot_lvm_verify_only: true + + - name: Remove again to check idempotence + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_action: remove + snapshot_lvm_set: "{{ snapshot_test_set }}" + + - name: Assert no changes for remove snapset + assert: + that: not snapshot_cmd["changed"] + always: + - name: Cleanup + include_tasks: tasks/cleanup.yml + tags: tests::cleanup diff --git a/tests/tests_thin_basic.yml b/tests/tests_thin_basic.yml new file mode 100644 index 0000000..c680a33 --- /dev/null +++ b/tests/tests_thin_basic.yml @@ -0,0 +1,138 @@ +--- +- name: Basic snapshot test + hosts: all + vars: + test_disk_min_size: "1g" + test_disk_count: 10 + test_storage_pools: + - name: test_vg1 + disks: "{{ range(0, 6) | map('extract', unused_disks) | list }}" + volumes: + - name: thin_pool_dev + thin_pool_name: thin_pool + thin_pool_size: '3g' + size: "1g" + thin: true + - name: lv2 + thin_pool_name: thin_pool + size: "15" + thin: true + - name: lv3 + thin_pool_name: thin_pool + size: "15%" + thin: true + - name: lv4 + thin_pool_name: thin_pool + size: "15%" + thin: true + - name: test_vg2 + disks: "{{ range(6, 10) | map('extract', unused_disks) | list }}" + volumes: + - name: lv5 + size: "30%" + - name: lv6 + size: "25%" + - name: lv7 + size: "10%" + - name: lv8 + size: "10%" + tasks: + - name: Run tests + block: + - name: Setup + include_tasks: tasks/setup.yml + + - name: Run the snapshot role to create snapshot LVs + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_percent_space_required: 15 + snapshot_lvm_all_vgs: true + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_action: snapshot + + - name: Assert changes for creation + assert: + that: snapshot_cmd["changed"] + + - name: Verify the snapshot LVs are created + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_all_vgs: true + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_verify_only: true + snapshot_lvm_action: check + + - name: Run the snapshot role again to check idempotence + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_percent_space_required: 15 + snapshot_lvm_all_vgs: true + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_action: snapshot + + - name: Assert no changes for creation + assert: + that: not snapshot_cmd["changed"] + + - name: Verify again to check idempotence + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_all_vgs: true + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_verify_only: true + snapshot_lvm_action: check + + - name: Assert no changes for verify + assert: + that: not snapshot_cmd["changed"] + + - name: Run the snapshot role remove the snapshot LVs + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_action: remove + + - name: Assert changes for removal + assert: + that: snapshot_cmd["changed"] + + - name: Use the snapshot_lvm_verify option to make sure remove is done + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_verify_only: true + snapshot_lvm_action: remove + + - name: Remove again to check idempotence + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_action: remove + + - name: Assert no changes for remove + assert: + that: not snapshot_cmd["changed"] + + - name: Verify remove again to check idempotence + include_role: + name: linux-system-roles.snapshot + vars: + snapshot_lvm_snapset_name: snapset1 + snapshot_lvm_verify_only: true + snapshot_lvm_action: remove + + - name: Assert no changes for remove verify + assert: + that: not snapshot_cmd["changed"] + + always: + - name: Cleanup + include_tasks: tasks/cleanup.yml + tags: tests::cleanup