version 1.9.18

Changelog

@@ -1,3 +1,10 @@
+Version 1.9.18
+---------
+ * virtnbdbackup: treat direct attached lun and block devices just like
+ regular raw disks: backup will work but only in backup mode copy and
+ of course full provisioned backup of these block devices is created
+ if option `--raw` is enabled. (#111)
+
 Version 1.9.17
 ---------
  * virtnbdrestore: fix restore fails with IndexError if CDROM or LUN devices

README.md

@@ -278,14 +278,23 @@ with qcow(v3) based disk images. If you are using older image versions, you can
 only create `copy` backups, or consider converting the images to a newer
 format using `qemu-img`.
 
-By default `virtnbdbackup` will exclude all disks with format `raw`. This
-behavior can be changed if option  `--raw` is specified, raw disks will then be
-included during a `full` backup. This of course means that no thin provisioned
-backup is created for these particular disks.
+By default `virtnbdbackup` will exclude all disks with format `raw` aswell
+as direct attached (passthrough) disks such as LVM or ZVOL and ISCSI
+volumes. These type of disks do not support storing checkpoint/bitmap
+metadata.
+
+This behavior can be changed if option `--raw` is specified, raw disks will
+then be included during a `full` backup.  This of course means that no thin
+provisioned backup is created for these particular disks.
 
 During restore, these files can be copied "as is" from the backup folder and
 must not be processed using `virtnbdrestore`.
 
+`Note:`
+> The backup data for raw disks will only be crash consistant, be aware
+> that this might result in inconsistent filesystems after restoring!
+
+
 # Backup Examples
 
 ## Local backups

libvirtnbdbackup/__init__.py

@@ -15,4 +15,4 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
 
-__version__ = "1.9.17"
+__version__ = "1.9.18"

libvirtnbdbackup/virt/client.py

@@ -31,6 +31,7 @@
     connectionFailed,
     startBackupFailed,
 )
+from libvirtnbdbackup.virt.fs import freeze, thaw
 from libvirtnbdbackup import output
 from libvirtnbdbackup.output.exceptions import OutputException
 
@@ -319,13 +320,15 @@ def adjustDomainConfig(
 
             device = disk.get("device")
             driver = disk.xpath("driver")[0].get("type")
-            if device in ("lun", "cdrom", "floppy"):
-                log.info("Removing [%s] device from vm config", device)
+
+            if self._isOptical(device, dev):
+                log.info("Removing device [%s], type [%s] from vm config", dev, device)
                 disk.getparent().remove(disk)
                 continue
-            if driver == "raw" and args.raw is False:
+
+            if self._isRaw(args, driver, device):
                 log.warning(
-                    "Removing raw disk [%s] from vm config.",
+                    "Removing raw disk [%s] from vm config, use --raw to copy as is.",
                     dev,
                 )
                 disk.getparent().remove(disk)
@@ -382,6 +385,51 @@ def _getDiskPathByVolume(self, disk: _Element) -> Union[str, None]:
 
         return diskPath
 
+    @staticmethod
+    def _isOptical(device: list, dev: str) -> bool:
+        """Check if device is cdrom or floppy"""
+        if device in ("cdrom", "floppy"):
+            log.info("Skipping attached [%s] device: [%s].", device, dev)
+            return True
+
+        return False
+
+    @staticmethod
+    def _isLun(args: Namespace, device: list, dev: str) -> bool:
+        """Check if device is direct attached LUN"""
+        if device == "lun" and args.raw is False:
+            log.warning(
+                "Skipping direct attached lun [%s], use option --raw to include",
+                dev,
+            )
+            return True
+
+        return False
+
+    @staticmethod
+    def _isBlock(args, disk: _Element, dev: str) -> bool:
+        """Check if device is direct attached block type device"""
+        if disk.xpath("target")[0].get("type") == "block" and args.raw is False:
+            log.warning(
+                "Block device [%s] excluded by default, use option --raw to include.",
+                dev,
+            )
+            return True
+
+        return False
+
+    @staticmethod
+    def _isRaw(args, diskFormat: str, dev: str) -> bool:
+        """Check if disk has RAW disk format"""
+        if diskFormat == "raw" and args.raw is False:
+            log.warning(
+                "Raw disk [%s] excluded by default, use option --raw to include.",
+                dev,
+            )
+            return True
+
+        return False
+
     def getDomainDisks(self, args: Namespace, vmConfig: str) -> List[DomainDisk]:
         """Parse virtual machine configuration for disk devices, filter
         all non supported devices
@@ -395,52 +443,38 @@ def getDomainDisks(self, args: Namespace, vmConfig: str) -> List[DomainDisk]:
 
         for disk in tree.xpath("devices/disk"):
             dev = disk.xpath("target")[0].get("dev")
+            device = disk.get("device")
+            diskFormat = disk.xpath("driver")[0].get("type")
 
             if excludeList is not None and dev in excludeList:
                 log.warning("Excluding disk [%s] from backup as requested", dev)
                 continue
 
-            # ignore attached lun or direct access block devices
-            if disk.xpath("target")[0].get("type") == "block":
-                log.warning(
-                    "Device [%s] does not support changed block tracking, skipping.",
-                    dev,
-                )
+            if self._isBlock(args, disk, dev):
                 continue
-
-            device = disk.get("device")
-            if device == "lun":
-                log.warning(
-                    "Skipping direct attached lun [%s].",
-                    dev,
-                )
+            if self._isLun(args, device, dev):
                 continue
-            if device in ("cdrom", "floppy"):
-                log.info("Skipping attached [%s] device: [%s].", device, dev)
+            if self._isOptical(device, dev):
                 continue
-
-            # ignore disk which use raw format, they do not support CBT
-            diskFormat = disk.xpath("driver")[0].get("type")
-            if diskFormat == "raw" and args.raw is False:
-                log.warning(
-                    "Raw disk [%s] excluded by default, use option --raw to include.",
-                    dev,
-                )
+            if self._isRaw(args, diskFormat, dev):
                 continue
 
+            diskPath = None
             diskType = disk.get("type")
             if diskType == "volume":
                 log.debug("Disk [%s]: volume notation", dev)
                 diskPath = self._getDiskPathByVolume(disk)
             elif diskType == "file":
-                log.debug("Disk [%s]: file notation")
+                log.debug("Disk [%s]: file notation", dev)
                 diskPath = disk.xpath("source")[0].get("file")
             elif diskType == "block":
-                log.warning(
-                    "Skipping direct attached block device [%s].",
-                    dev,
-                )
-                continue
+                if args.raw is False:
+                    log.warning(
+                        "Skipping direct attached block device [%s], use option --raw to include.",
+                        dev,
+                    )
+                    continue
+                diskPath = disk.xpath("source")[0].get("dev")
             else:
                 log.error("Unable to detect disk volume type for disk [%s]", dev)
                 continue
@@ -544,41 +578,19 @@ def _createCheckpointXml(
             # because it is not supported. Backup will only be crash
             # consistent. If we would like to create a consistent
             # backup, we would have to create an snapshot for these
-            # kind of disks.
+            # kind of disks, example:
+            # virsh checkpoint-create-as vm4 --diskspec sdb
+            # error: unsupported configuration:  \
+            # checkpoint for disk sdb unsupported for storage type raw
+            # See also:
+            # https://lists.gnu.org/archive/html/qemu-devel/2021-03/msg07448.html
             if disk.format != "raw":
                 ElementTree.SubElement(disks, "disk", {"name": disk.target})
 
         xml = self._indentXml(top)
 
         return xml
 
-    @staticmethod
-    def fsFreeze(domObj: libvirt.virDomain, mountpoints: None) -> bool:
-        """Attempt to freeze domain filesystems using qemu guest agent"""
-        log.debug("Attempting to freeze filesystems.")
-        try:
-            if mountpoints is not None:
-                frozen = domObj.fsFreeze(mountpoints.split(","))
-            else:
-                frozen = domObj.fsFreeze()
-            log.info("Freezed [%s] filesystems.", frozen)
-            return True
-        except libvirt.libvirtError as errmsg:
-            log.warning(errmsg)
-            return False
-
-    @staticmethod
-    def fsThaw(domObj: libvirt.virDomain) -> bool:
-        """Thaw freeze filesystems"""
-        log.debug("Attempting to thaw filesystems.")
-        try:
-            thawed = domObj.fsThaw()
-            log.info("Thawed [%s] filesystems.", thawed)
-            return True
-        except libvirt.libvirtError as errmsg:
-            log.warning(errmsg)
-            return False
-
     def startBackup(
         self,
         args: Namespace,
@@ -596,7 +608,7 @@ def startBackup(
             checkpointXml = self._createCheckpointXml(
                 diskList, args.cpt.parent, args.cpt.name
             )
-        freezed = self.fsFreeze(domObj, args.freeze_mountpoint)
+        freezed = freeze(domObj, args.freeze_mountpoint)
         try:
             log.debug("Starting backup job via libvirt API.")
             domObj.backupBegin(backupXml, checkpointXml)
@@ -612,7 +624,7 @@ def startBackup(
             # check if filesystem is freezed and thaw
             # in case creating checkpoint fails.
             if freezed is True:
-                self.fsThaw(domObj)
+                thaw(domObj)
 
     @staticmethod
     def _checkpointExists(

libvirtnbdbackup/virt/fs.py

@@ -0,0 +1,47 @@
+"""
+    Copyright (C) 2023  Michael Ablassmeier <[email protected]>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+import logging
+import libvirt
+
+log = logging.getLogger("fs")
+
+
+def freeze(domObj: libvirt.virDomain, mountpoints: None) -> bool:
+    """Attempt to freeze domain filesystems using qemu guest agent"""
+    log.debug("Attempting to freeze filesystems.")
+    try:
+        if mountpoints is not None:
+            frozen = domObj.fsFreeze(mountpoints.split(","))
+        else:
+            frozen = domObj.fsFreeze()
+        log.info("Freezed [%s] filesystems.", frozen)
+        return True
+    except libvirt.libvirtError as errmsg:
+        log.warning(errmsg)
+        return False
+
+
+def thaw(domObj: libvirt.virDomain) -> bool:
+    """Thaw freezed filesystems"""
+    log.debug("Attempting to thaw filesystems.")
+    try:
+        thawed = domObj.fsThaw()
+        log.info("Thawed [%s] filesystems.", thawed)
+        return True
+    except libvirt.libvirtError as errmsg:
+        log.warning(errmsg)
+        return False

man/virtnbdbackup.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.48.1.
-.TH VIRTNBDBACKUP "1" "February 2023" "virtnbdbackup 1.9.17" "User Commands"
+.TH VIRTNBDBACKUP "1" "March 2023" "virtnbdbackup 1.9.18" "User Commands"
 .SH NAME
 virtnbdbackup \- backup utility for libvirt
 .SH DESCRIPTION
@@ -48,7 +48,7 @@ Backup only disk with target dev name (\fB\-i\fR vda)
 Exclude disk(s) with target dev name (\fB\-x\fR vda,vdb)
 .TP
 \fB\-f\fR SOCKETFILE, \fB\-\-socketfile\fR SOCKETFILE
-Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.7802\/\fP)
+Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.9649\/\fP)
 .TP
 \fB\-n\fR, \fB\-\-noprogress\fR
 Disable progress bar

man/virtnbdmap.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.48.1.
-.TH VIRTNBDMAP "1" "February 2023" "virtnbdmap 1.9.17" "User Commands"
+.TH VIRTNBDMAP "1" "March 2023" "virtnbdmap 1.9.18" "User Commands"
 .SH NAME
 virtnbdmap \- map virtnbdbackup image files to nbd devices
 .SH DESCRIPTION

man/virtnbdrestore.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.48.1.
-.TH VIRTNBDRESTORE "1" "February 2023" "virtnbdrestore 1.9.17" "User Commands"
+.TH VIRTNBDRESTORE "1" "March 2023" "virtnbdrestore 1.9.18" "User Commands"
 .SH NAME
 virtnbdrestore \- restore utility for libvirt
 .SH DESCRIPTION
@@ -40,7 +40,7 @@ Process only disk matching target dev name. (default: None)
 Disable progress bar
 .TP
 \fB\-f\fR SOCKETFILE, \fB\-\-socketfile\fR SOCKETFILE
-Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.7807\/\fP)
+Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.9654\/\fP)
 .TP
 \fB\-r\fR, \fB\-\-raw\fR
 Copy raw images as is during restore. (default: False)