Goal: I want to pass the PCI-E device 04:00.0 to KVM, and keep the device 03:00.0 working on the host (it's got the host OS). proxmox:~# dmesg | grep -e DMAR -e IOMMU ACPI: DMAR 00000000bf79e0c0 00118 (v01 AMI OEMDMAR 00000001 MSFT 00000097) Intel-IOMMU: enabled DMAR: Host address width 39 DMAR: DRHD base: 0x000000fbffe000 flags: 0x1 IOMMU fbffe000: ver 1:0 cap c90780106f0462 ecap f020f6 DMAR: RMRR base: 0x000000000ec000 end: 0x000000000effff DMAR: RMRR base: 0x000000bf7ec000 end: 0x000000bf7fffff DMAR: ATSR flags: 0x0 IOMMU 0xfbffe000: using Queued invalidation IOMMU: Setting RMRR: IOMMU: Setting identity map for device 0000:00:1d.0 [0xbf7ec000 - 0xbf800000] IOMMU: Setting identity map for device 0000:00:1d.1 [0xbf7ec000 - 0xbf800000] IOMMU: Setting identity map for device 0000:00:1d.2 [0xbf7ec000 - 0xbf800000] IOMMU: Setting identity map for device 0000:00:1d.7 [0xbf7ec000 - 0xbf800000] IOMMU: Setting identity map for device 0000:00:1a.0 [0xbf7ec000 - 0xbf800000] IOMMU: Setting identity map for device 0000:00:1a.1 [0xbf7ec000 - 0xbf800000] IOMMU: Setting identity map for device 0000:00:1a.2 [0xbf7ec000 - 0xbf800000] IOMMU: Setting identity map for device 0000:00:1a.7 [0xbf7ec000 - 0xbf800000] IOMMU: Setting identity map for device 0000:00:1d.0 [0xec000 - 0xf0000] IOMMU: Setting identity map for device 0000:00:1d.1 [0xec000 - 0xf0000] IOMMU: Setting identity map for device 0000:00:1d.2 [0xec000 - 0xf0000] IOMMU: Setting identity map for device 0000:00:1d.7 [0xec000 - 0xf0000] IOMMU: Setting identity map for device 0000:00:1a.0 [0xec000 - 0xf0000] IOMMU: Setting identity map for device 0000:00:1a.1 [0xec000 - 0xf0000] IOMMU: Setting identity map for device 0000:00:1a.2 [0xec000 - 0xf0000] IOMMU: Setting identity map for device 0000:00:1a.7 [0xec000 - 0xf0000] IOMMU: Prepare 0-16MiB unity mapping for LPC IOMMU: Setting identity map for device 0000:00:1f.0 [0x0 - 0x1000000] proxmox:~# grep iommu /boot/grub/menu.lst kernel /vmlinuz-2.6.32-6-pve root=/dev/mapper/pve-root ro intel_iommu=on proxmox:~# lspci | grep -i 3ware 03:00.0 RAID bus controller: 3ware Inc 9750 SAS2/SATA-II RAID PCIe (rev 05) 04:00.0 RAID bus controller: 3ware Inc 9750 SAS2/SATA-II RAID PCIe (rev 05) proxmox:~# lspci -n | tail -n 6 02:00.0 0300: 10de:0659 (rev a1) 03:00.0 0104: 13c1:1010 (rev 05) 04:00.0 0104: 13c1:1010 (rev 05) 05:00.0 0200: 8086:10d3 06:00.0 0200: 8086:10d3 07:08.0 0c00: 104c:8023 proxmox:~# cat /sys/bus/pci/devices/0000:03:00.0/irq 16 proxmox:~# cat /sys/bus/pci/devices/0000:04:00.0/irq 16 proxmox:~# echo "13c1 1010" > /sys/bus/pci/drivers/pci-stub/new_id proxmox:~# echo "0000:04:00.0" > /sys/bus/pci/drivers/3w-sas/unbind proxmox:~# echo "0000:04:00.0" > /sys/bus/pci/drivers/pci-stub/bind proxmox:~# echo "13c1 1010" > /sys/bus/pci/drivers/pci-stub/remove_id proxmox:~# /usr/bin/kvm -name FreeNAS_dgroup -smp sockets=2,cores=1 -nodefaults -boot menu=on -vga cirrus -tdf -k en-us -drive file=/var/lib/vz/images/102/vm-102-disk-1.raw,if=ide,index=0,cache=none,boot=on -m 12288 -netdev type=tap,id=vlan1d0,ifname=tap102i1d0,script=/var/lib/qemu-server/bridge-vlan -device rtl8139,romfile=,mac=5E:6A:83:F4:31:D1,netdev=vlan1d0 -id 102 -cpuunits 1000 -device pci-assign,host=04:00.0 Failed to assign device "(null)" : Operation not permitted kvm: -device pci-assign,host=04:00.0: Device 'pci-assign' could not be initialized proxmox:~# strace /usr/bin/kvm -name FreeNAS_dgroup -smp sockets=2,cores=1 -nodefaults -boot menu=on -vga cirrus -tdf -k en-us -drive file=/var/lib/vz/images/102/vm-102-disk-1.raw,if=ide,index=0,cache=none,boot=on -m 12288 -netdev type=tap,id=vlan1d0,ifname=tap102i1d0,script=/var/lib/qemu-server/bridge-vlan -device rtl8139,romfile=,mac=5E:6A:83:F4:31:D1,netdev=vlan1d0 -id 102 -cpuunits 1000 -device pci-assign,host=04:00.0 brk(0x42c7000) = 0x42c7000 brk(0x42eb000) = 0x42eb000 access("/usr/share/kvm/bios.bin", R_OK) = 0 open("/usr/share/kvm/bios.bin", O_RDONLY) = 13 lseek(13, 0, SEEK_END) = 131072 close(13) = 0 mmap(NULL, 139264, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835f0000 madvise(0x7ff2835f1000, 131072, 0xc /* MADV_??? */) = 0 ioctl(3, 0xae03, 0x10) = 1 access("/usr/share/kvm/bios.bin", R_OK) = 0 open("/usr/share/kvm/bios.bin", O_RDONLY) = 13 lseek(13, 0, SEEK_END) = 131072 mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835cf000 lseek(13, 0, SEEK_SET) = 0 read(13, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 131072) = 131072 close(13) = 0 ioctl(4, 0x4020ae46, 0x7fff1918f7c0) = 0 mmap(NULL, 139264, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835ad000 madvise(0x7ff2835ae000, 131072, 0xc /* MADV_??? */) = 0 ioctl(3, 0xae03, 0x10) = 1 ioctl(4, 0x4020ae46, 0x7fff1918f7c0) = 0 ioctl(4, 0x4020ae46, 0x7fff1918f7c0) = 0 ioctl(3, 0xae03, 0x19) = 1024 access("/usr/share/kvm/extboot.bin", R_OK) = 0 open("/usr/share/kvm/extboot.bin", O_RDONLY) = 13 lseek(13, 0, SEEK_END) = 1536 lseek(13, 0, SEEK_SET) = 0 read(13, "U\252\3fP\0361\300\216\330\376\6u\4f\241d\0f\243\0\2\307\6d\0$\0\214\16f\0\37"..., 1536) = 1536 close(13) = 0 access("/usr/share/kvm/vapic.bin", R_OK) = 0 open("/usr/share/kvm/vapic.bin", O_RDONLY) = 13 lseek(13, 0, SEEK_END) = 9216 lseek(13, 0, SEEK_SET) = 0 read(13, "U\252\22\6\16\0071\300\271\0\20\215>\0\3\374\363\253\7\270 \0\347~\313\17\37\200\0\0\0\0k"..., 9216) = 9216 close(13) = 0 brk(0x431d000) = 0x431d000 mmap(NULL, 16785408, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7e1a2000 madvise(0x7fef7e1a3000, 16777216, 0xc /* MADV_??? */) = 0 mremap(0x7fef7f1a4000, 3149824, 3153920, MREMAP_MAYMOVE) = 0x7fef7dea0000 ioctl(3, 0xae03, 0x10) = 1 ioctl(4, 0x4010ae67, 0x7fff1918f7d0) = 0 mmap(NULL, 1232896, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f378000 access("/usr/share/kvm/vgabios-cirrus.bin", R_OK) = 0 open("/usr/share/kvm/vgabios-cirrus.bin", O_RDONLY) = 13 lseek(13, 0, SEEK_END) = 35840 close(13) = 0 madvise(0x42fe000, 65536, 0xc /* MADV_??? */) = 0 ioctl(3, 0xae03, 0x10) = 1 open("/usr/share/kvm/vgabios-cirrus.bin", O_RDONLY) = 13 lseek(13, 0, SEEK_END) = 35840 lseek(13, 0, SEEK_SET) = 0 read(13, "U\252F\351!\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\17\1\0\0\0\0IBM"..., 35840) = 35840 close(13) = 0 brk(0x4340000) = 0x4340000 open("/etc/localtime", O_RDONLY) = 13 fstat(13, {st_mode=S_IFREG|0644, st_size=327, ...}) = 0 fstat(13, {st_mode=S_IFREG|0644, st_size=327, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835ac000 read(13, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\3\0\0\0\3\0\0\0\0\0"..., 4096) = 327 lseek(13, -197, SEEK_CUR) = 130 read(13, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0\0"..., 4096) = 197 close(13) = 0 munmap(0x7ff2835ac000, 4096) = 0 mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff28358c000 mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff28356b000 mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f357000 mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f336000 pread(9, "\3741\300\216\300\216\330\216\320\274\0|\211\346\277\0\6\271\0\1\363\245\211\375\261\10\363\253\376E\362\351\0"..., 512, 0) = 512 ioctl(11, 0x4090ae82, 0x7fff1918f7a0) = 0 ioctl(11, 0x41a0ae8d, 0x7fff1918f680) = 0 ioctl(11, 0x4138ae84, 0x7fff1918f860) = 0 ioctl(11, 0x4008ae89, 0x7fff1918f1f0) = 53 ioctl(11, 0x4004ae99, 0x7fff1918f850) = 0 ioctl(11, 0x4400ae8f, 0x7fff1918f430) = 0 ioctl(11, 0x4040aea0, 0x7fff1918f860) = 0 ioctl(12, 0x4090ae82, 0x7fff1918f7a0) = 0 ioctl(12, 0x41a0ae8d, 0x7fff1918f680) = 0 ioctl(12, 0x4138ae84, 0x7fff1918f860) = 0 ioctl(12, 0x4008ae89, 0x7fff1918f1f0) = 53 ioctl(12, 0x4004ae99, 0x7fff1918f850) = 0 ioctl(12, 0x4400ae8f, 0x7fff1918f430) = 0 ioctl(12, 0x4040aea0, 0x7fff1918f860) = 0 open("/sys/bus/pci/devices/0000:04:00.0/config", O_RDWR) = 13 read(13, "\301\23\20\20\3\0\20\0\5\0\4\1@\0\0\0\1\310\0\0\4\300\271\373\0\0\0\0\4\0\274\373\0"..., 256) = 256 open("/sys/bus/pci/devices/0000:04:00.0/resource", O_RDONLY) = 14 fstat(14, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f335000 read(14, "0x000000000000c800 0x000000000000"..., 4096) = 741 open("/sys/bus/pci/devices/0000:04:00.0/resource0", O_RDWR) = 15 open("/sys/bus/pci/devices/0000:04:00.0/resource1", O_RDWR) = 16 open("/sys/bus/pci/devices/0000:04:00.0/resource3", O_RDWR) = 17 close(14) = 0 munmap(0x7fef7f335000, 4096) = 0 open("/sys/bus/pci/devices/0000:04:00.0/vendor", O_RDONLY) = 14 fstat(14, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f335000 read(14, "0x13c1\n"..., 4096) = 7 read(14, ""..., 4096) = 0 close(14) = 0 munmap(0x7fef7f335000, 4096) = 0 open("/sys/bus/pci/devices/0000:04:00.0/device", O_RDONLY) = 14 fstat(14, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f335000 read(14, "0x1010\n"..., 4096) = 7 read(14, ""..., 4096) = 0 close(14) = 0 munmap(0x7fef7f335000, 4096) = 0 stat("/sys/bus/pci/devices/0000:04:00.0/physfn/", 0x7fff1918f4c0) = -1 ENOENT (No such file or directory) pread(15, 0x7fff1918f6ec, 3, 0) = -1 EINVAL (Invalid argument) mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_SHARED, 16, 0) = 0x7fef7f332000 ioctl(3, 0xae03, 0x10) = 1 mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_SHARED, 17, 0) = 0x7fef7f2f2000 ioctl(3, 0xae03, 0x10) = 1 pread(13, "\20"..., 1, 6) = 1 pread(13, "P"..., 1, 52) = 1 pread(13, "\1"..., 1, 80) = 1 pread(13, "h"..., 1, 81) = 1 pread(13, "\20"..., 1, 104) = 1 pread(13, "\320"..., 1, 105) = 1 pread(13, "\3"..., 1, 208) = 1 pread(13, "\250"..., 1, 209) = 1 pread(13, "\5"..., 1, 168) = 1 pread(13, "\20"..., 1, 6) = 1 pread(13, "P"..., 1, 52) = 1 pread(13, "\1"..., 1, 80) = 1 pread(13, "h"..., 1, 81) = 1 pread(13, "\20"..., 1, 104) = 1 pread(13, "\320"..., 1, 105) = 1 pread(13, "\3"..., 1, 208) = 1 pread(13, "\250"..., 1, 209) = 1 pread(13, "\5"..., 1, 168) = 1 pread(13, "\0"..., 1, 169) = 1 pread(13, "\20"..., 1, 6) = 1 pread(13, "P"..., 1, 52) = 1 pread(13, "\1"..., 1, 80) = 1 pread(13, "\20"..., 1, 6) = 1 pread(13, "P"..., 1, 52) = 1 pread(13, "\1"..., 1, 80) = 1 pread(13, "h"..., 1, 81) = 1 pread(13, "\20"..., 1, 104) = 1 pread(13, "\20"..., 1, 6) = 1 pread(13, "P"..., 1, 52) = 1 pread(13, "\1"..., 1, 80) = 1 pread(13, "h"..., 1, 81) = 1 pread(13, "\20"..., 1, 104) = 1 pread(13, "\320"..., 1, 105) = 1 pread(13, "\3"..., 1, 208) = 1 pread(13, "\250"..., 1, 209) = 1 pread(13, "\5"..., 1, 168) = 1 pread(13, "\0"..., 1, 169) = 1 pread(13, "\20"..., 1, 6) = 1 pread(13, "P"..., 1, 52) = 1 pread(13, "\1"..., 1, 80) = 1 pread(13, "h"..., 1, 81) = 1 pread(13, "\20"..., 1, 104) = 1 pread(13, "\320"..., 1, 105) = 1 pread(13, "\3"..., 1, 208) = 1 pread(13, "\20"..., 1, 6) = 1 pread(13, "P"..., 1, 52) = 1 pread(13, "\1"..., 1, 80) = 1 pread(13, "h"..., 1, 81) = 1 pread(13, "\20"..., 1, 104) = 1 pread(13, "\320"..., 1, 105) = 1 pread(13, "\3"..., 1, 208) = 1 pread(13, "\250"..., 1, 209) = 1 pread(13, "\5"..., 1, 168) = 1 pread(13, "\0"..., 1, 169) = 1 ioctl(3, 0xae03, 0x2f) = 0 ioctl(3, 0xae03, 0x12) = 1 ioctl(4, 0xffffffff8040ae69, 0x7fff1918f6f0) = -1 EPERM (Operation not permitted) write(2, "Failed to assign device \"(null)\" "..., 59Failed to assign device "(null)" : Operation not permitted ) = 59 close(15) = 0 munmap(0x7fef7f332000, 16384) = 0 close(16) = 0 munmap(0x7fef7f2f2000, 262144) = 0 close(17) = 0 close(13) = 0 write(2, "kvm:"..., 4kvm:) = 4 write(2, " -device"..., 8 -device) = 8 write(2, " pci-assign,host=04:00.0"..., 24 pci-assign,host=04:00.0) = 24 write(2, ": "..., 2: ) = 2 write(2, "Device 'pci-assign' could not be "..., 44Device 'pci-assign' could not be initialized) = 44 write(2, "\n"..., 1 ) = 1 exit_group(1) = ? proxmox:~#