Allow adding MSI capability via vfu_pci_add_capability (nutanix#758)

Signed-off-by: Florian Freudiger <[email protected]>

include/libvfio-user.h

@@ -981,6 +981,7 @@ vfu_pci_get_config_space(vfu_ctx_t *vfu_ctx);
  * Certain standard capabilities are handled entirely within the library:
  *
  * PCI_CAP_ID_EXP (pxcap)
+ * PCI_CAP_ID_MSI (msicap)
  * PCI_CAP_ID_MSIX (msixcap)
  * PCI_CAP_ID_PM (pmcap)
  *

include/pci_caps/msi.h

@@ -39,33 +39,38 @@
 extern "C" {
 #endif
 
+/* Message Control for MSI */
 struct mc {
-    unsigned int msie:1;
-    unsigned int mmc:3;
-    unsigned int mme:3;
-    unsigned int c64:1;
-    unsigned int pvm:1;
-    unsigned int res1:7;
+    unsigned int msie:1;    /* RW */
+    unsigned int mmc:3;     /* RO */
+    unsigned int mme:3;     /* RW */
+    unsigned int c64:1;     /* RO */
+    unsigned int pvm:1;     /* RO */
+    unsigned int res1:7;    /* not implemented, extended message data control */
 } __attribute__ ((packed));
 _Static_assert(sizeof(struct mc) == 0x2, "bad MC size");
 
+/* Message Address for MSI */
 struct ma {
-    unsigned int res1:2;
-    unsigned int addr:30;
+    unsigned int res1:2;    /* read must return 0, write has no effect */
+    unsigned int addr:30;   /* RW */
 } __attribute__ ((packed));
 _Static_assert(sizeof(struct ma) == 0x4, "bad MA size");
 
+#define VFIO_USER_PCI_CAP_MSI_SIZEOF (0x18)
+
 struct msicap {
     struct cap_hdr hdr;
     struct mc mc;
     struct ma ma;
-    uint32_t mua;
-    uint16_t md;
-    uint16_t padding;
-    uint32_t mmask;
-    uint32_t mpend;
+    uint32_t mua;           /* RW */
+    uint16_t md;            /* RW */
+    uint16_t padding;       /* not implemented, extended message data */
+    uint32_t mmask;         /* RW */
+    uint32_t mpend;         /* RO */
 }  __attribute__ ((packed));
-_Static_assert(sizeof(struct msicap) == 0x18, "bad MSICAP size");
+_Static_assert(sizeof(struct msicap) == VFIO_USER_PCI_CAP_MSI_SIZEOF,
+        "bad MSICAP size");
 _Static_assert(offsetof(struct msicap, hdr) == 0, "bad offset");
 
 #ifdef __cplusplus

lib/pci_caps.c

@@ -99,6 +99,8 @@ cap_size(vfu_ctx_t *vfu_ctx, void *data, bool extended)
             return PCI_PM_SIZEOF;
         case PCI_CAP_ID_EXP:
             return VFIO_USER_PCI_CAP_EXP_SIZEOF;
+        case PCI_CAP_ID_MSI:
+            return VFIO_USER_PCI_CAP_MSI_SIZEOF;
         case PCI_CAP_ID_MSIX:
             return PCI_CAP_MSIX_SIZEOF;
         case PCI_CAP_ID_VNDR:
@@ -166,6 +168,67 @@ cap_write_pm(vfu_ctx_t *vfu_ctx, struct pci_cap *cap, char * buf,
     return ERROR_INT(EINVAL);
 }
 
+static ssize_t
+cap_write_msi(vfu_ctx_t *vfu_ctx, struct pci_cap *cap, char *buf,
+               size_t count, loff_t offset)
+{
+    struct msicap *msi = cap_data(vfu_ctx, cap);
+    struct msicap new_msi = *msi;
+
+    memcpy((char *)&new_msi + offset - cap->off, buf, count);
+
+    if (msi->mc.msie != new_msi.mc.msie) {
+        msi->mc.msie = new_msi.mc.msie;
+        vfu_log(vfu_ctx, LOG_DEBUG, "%s MSI",
+                msi->mc.msie ? "enable" : "disable");
+    }
+
+    if (msi->mc.mme != new_msi.mc.mme) {
+        if (new_msi.mc.mme > 5) {
+            vfu_log(vfu_ctx, LOG_ERR,
+                    "MSI cannot have more than 32 interrupt vectors");
+            return ERROR_INT(EINVAL);
+        }
+
+        if (new_msi.mc.mme > msi->mc.mmc) {
+            vfu_log(vfu_ctx, LOG_ERR,
+                    "MSI cannot have more interrupt vectors"
+                    " in MME than defined in MMC");
+            return ERROR_INT(EINVAL);
+        }
+        msi->mc.mme = new_msi.mc.mme;
+
+        vfu_log(vfu_ctx, LOG_DEBUG,
+                "MSI Updated Multiple Message Enable count");
+    }
+
+    if (msi->ma.addr != new_msi.ma.addr) {
+        msi->ma.addr = new_msi.ma.addr;
+        vfu_log(vfu_ctx, LOG_DEBUG,
+                "MSI Message Address set to %x", msi->ma.addr << 2);
+    }
+
+    if (msi->mua != new_msi.mua) {
+        msi->mua = new_msi.mua;
+        vfu_log(vfu_ctx, LOG_DEBUG,
+                "MSI Message Upper Address set to %x", msi->mua);
+    }
+
+    if (msi->md != new_msi.md) {
+        msi->md = new_msi.md;
+        vfu_log(vfu_ctx, LOG_DEBUG,
+                "MSI Message Data set to %x", msi->md);
+    }
+
+    if (msi->mmask != new_msi.mmask) {
+        msi->mmask = new_msi.mmask;
+        vfu_log(vfu_ctx, LOG_DEBUG,
+                "MSI Mask Bits set to %x", msi->mmask);
+    }
+
+    return count;
+}
+
 static ssize_t
 cap_write_msix(vfu_ctx_t *vfu_ctx, struct pci_cap *cap, char *buf,
                size_t count, loff_t offset)
@@ -682,6 +745,10 @@ vfu_pci_add_capability(vfu_ctx_t *vfu_ctx, size_t pos, int flags, void *data)
             cap.name = "PCI Express";
             cap.cb = cap_write_px;
             break;
+        case PCI_CAP_ID_MSI:
+            cap.name = "MSI";
+            cap.cb = cap_write_msi;
+            break;
         case PCI_CAP_ID_MSIX:
             cap.name = "MSI-X";
             cap.cb = cap_write_msix;

test/py/libvfio_user.py

@@ -70,6 +70,7 @@
 
 PCI_CAP_ID_PM = 0x1
 PCI_CAP_ID_VNDR = 0x9
+PCI_CAP_ID_MSI = 0x5
 PCI_CAP_ID_MSIX = 0x11
 PCI_CAP_ID_EXP = 0x10
 
@@ -83,6 +84,12 @@
 
 PCI_EXT_CAP_VNDR_HDR_SIZEOF = 8
 
+# MSI registers
+PCI_MSI_FLAGS = 2  # Message Control offset
+PCI_MSI_ADDRESS_LO = 4  # Message Address offset
+PCI_MSI_FLAGS_ENABLE = 0x0001  # MSI enable
+PCI_CAP_MSI_SIZEOF = 24  # size of MSI registers
+
 # MSI-X registers
 PCI_MSIX_FLAGS = 2  # Message Control
 PCI_MSIX_TABLE = 4  # Table offset

test/py/test_pci_caps.py

@@ -342,6 +342,64 @@ def test_pci_cap_write_px(mock_quiesce, mock_reset):
                          expect=errno.EINVAL)
 
 
+def test_pci_cap_write_msi():
+    setup_pci_dev(realize=True)
+    sock = connect_client(ctx)
+
+    # Set MMC to 100b (16 interrupt vectors)
+    mmc = 0b00001000
+
+    # Bad MME with 101b (32 interrupt vectors), over MMC
+    mme_bad = 0b01010000
+    # Test MME with 100b (16 interrupt vectors)
+    mme_good = 0b01000000
+
+    # Test if capability is placed at right offset
+    pos = vfu_pci_add_capability(ctx, pos=0, flags=0,
+                                 data=struct.pack("ccHIIIII",
+                                                  to_byte(PCI_CAP_ID_MSI),
+                                                  b'\0', mmc, 0, 0, 0, 0, 0))
+    assert pos == cap_offsets[0]
+
+    offset = vfu_pci_find_capability(ctx, False, PCI_CAP_ID_MSI)
+
+    # Test if write fails as expected
+    # as MME is out of bounds, 111b is over the max of 101b (32 vectors)
+    data = b'\xff\xff'
+    write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX,
+                 offset=offset + PCI_MSI_FLAGS,
+                 count=len(data), data=data, expect=errno.EINVAL)
+
+    # Test if write fails as expected
+    # as MME is over MMC, 101b (32 vectors) > 100b (16 vectors)
+    data = to_bytes_le(mme_bad, 2)
+    write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX,
+                 offset=offset + PCI_MSI_FLAGS,
+                 count=len(data), data=data, expect=errno.EINVAL)
+
+    # Test good write, MSI Enable + good MME
+    data = to_bytes_le(PCI_MSI_FLAGS_ENABLE | mme_good, 2)
+    write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX,
+                 offset=offset + PCI_MSI_FLAGS,
+                 count=len(data), data=data)
+
+    size_before_flags = PCI_CAP_MSI_SIZEOF - PCI_MSI_FLAGS
+    size_after_flags = PCI_CAP_MSI_SIZEOF - PCI_MSI_ADDRESS_LO
+
+    # reset
+    data = size_before_flags * b'\x00'
+    write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX,
+                 offset=offset + PCI_MSI_FLAGS,
+                 count=len(data), data=data)
+
+    # Check if MMC is still present after reset (since it is RO)
+    expected = (to_bytes_le(PCI_CAP_ID_MSI) + b'\x00' +
+                to_bytes_le(mmc, 2) + (size_after_flags * b'\x00'))
+    payload = read_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX, offset=offset,
+                          count=len(expected))
+    assert expected == payload
+
+
 def test_pci_cap_write_msix():
     setup_pci_dev(realize=True)
     sock = connect_client(ctx)