Xen SRIOV config on sles11sp2

env: sles11sp2 xen

1.BIOS Configuration：

打开vt-x ，vt-d 技术

2.检查硬件是否支持SRIOV

dom0-93:/opt # lspci -vvk -s 0b:00.1 | grep SR-IOV

Capabilities: [160] Single Root I/O Virtualization (SR-IOV)

3.Modifying modprobe.conf

append as follows to /etc/modprobe.conf ：

options igb max_vfs=7

blacklist igbvf

options loop max_loop=128

4.Modifying grub.conf, add iommu=1

kernel /boot/xen.gz vga=mode-0x317 iommu=1

5.reboot machine, after rebooting

dom0-93:/opt # lspci | grep Eth
03:00.0 Ethernet controller: Broadcom Corporation NetXtreme II BCM5708 Gigabit Ethernet (rev 12)
05:00.0 Ethernet controller: Broadcom Corporation NetXtreme II BCM5708 Gigabit Ethernet (rev 12)
0b:00.0 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01)
0b:00.1 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01)
0c:10.0 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:10.1 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:10.4 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:10.5 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:11.0 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:11.1 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:11.4 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:11.5 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:12.0 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:12.1 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:12.4 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:12.5 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:13.0 Ethernet controller: Intel Corporation Device 1520 (rev 01)
0c:13.1 Ethernet controller: Intel Corporation Device 1520 (rev 01)

modifying grub.conf

title Xen -- SUSE Linux Enterprise Server 11 SP2 - 3.0.13-0.27
    root (hd0,0)
    kernel /boot/xen.gz vga=mode-0x317 iommu=1
    module /boot/vmlinuz-3.0.13-0.27-xen root=/dev/disk/by-id/cciss-3600508b100104839535656314933001b-part1 resume=/dev/disk/by-id/cciss-3600508b100104839535656314933001b-part2 splash=silent showopts vga=0x317 pciback.hide=(0c:10.0)(0c:10.1)(0c:10.4)(0c:10.5)(0c:11.0)(0c:11.1)(0c:11.4)(0c:11.5)(0c:12.0)(0c:12.1)(0c:12.4)(0c:12.5)(0c:13.0)(0c:13.1)
    module /boot/initrd-3.0.13-0.27-xen

6.reboot machine, Assigning a Virtual Function to a Virtual Machine

1）modifying vm.conf , add pci setting.

dom0-93:/etc/xen/vm # cat sles10sp4r2
name="sles10sp4r2"
description="None"
uuid="a847c466-adeb-704f-4ec5-c6fc0bdb96aa"
memory=1024
maxmem=1024
vcpus=4
on_poweroff="destroy"
on_reboot="restart"
on_crash="destroy"
localtime=0
keymap="en-us"
builder="linux"
bootloader="/usr/lib/xen/boot/domUloader.py"
bootargs="--entry=xvda2:/boot/vmlinuz-xen,/boot/initrd-xen"
#extra=" swiotlb=force "
extra=" " (加不加swiotlb=force都行)
disk=[ 'phy:/dev/domu/sles10sp4,xvda,w', ]
vif=[ 'mac=00:16:3e:5f:5e:f5,bridge=br1', 'mac=00:16:3e:5f:5e:f6,bridge=br2', ]
vfb=['type=vnc,vncunused=1']
pci=['0c:10.0']

2)start vm

3)in domU,

vsles10sp4:~ # lspci
00:00.0 Ethernet controller: Intel Corporation Device 1520 (rev 01)

vsles10sp4:~ # dmesg | grep -i pci
PCI-DMA: Using software bounce buffering for IO (SWIOTLB)
PCI: setting up Xen PCI frontend stub
PCI: System does not support PCI
PCI: System does not support PCI
pcifront pci-0: Installing PCI frontend
pcifront pci-0: Creating PCI Frontend Bus 0000:00

swiotlb :

For DMA access to work for pci devices in a PV guest it is required to enable the swiotlb in the domU (guest VM) kernel. (for some explanation about the swiotlb see http://lwn.net/Articles/91870/)

Add this option to your domU (guest VM) kernel boot options (cmdline arguments) list :

swiotlb=force

NOTE: This option is only required for older PV guest kernel versions (linux-2.6.18-xen, RHEL5, SLES10/11, Debian etch/lenny).

upstream kernel.org (pvops) Linux domU kernels requires just this option:

iommu=soft

4) in domU,install the igbvf driver

vsles10sp4:/opt/igbvf-1.1.5/src # make
make -C /lib/modules/2.6.16.60-0.85.1-xen/build SUBDIRS=/opt/igbvf-1.1.5/src modules
make[1]: Entering directory `/usr/src/linux-2.6.16.60-0.85.1-obj/x86_64/xen'
make -C ../../../linux-2.6.16.60-0.85.1 O=../linux-2.6.16.60-0.85.1-obj/x86_64/xen modules
Updating /opt/igbvf-1.1.5/src/Makefile.xen
CC [M] /opt/igbvf-1.1.5/src/netdev.o
CC [M] /opt/igbvf-1.1.5/src/ethtool.o
CC [M] /opt/igbvf-1.1.5/src/param.o
CC [M] /opt/igbvf-1.1.5/src/e1000_vf.o
CC [M] /opt/igbvf-1.1.5/src/kcompat.o
CC [M] /opt/igbvf-1.1.5/src/e1000_mbx.o
LD [M] /opt/igbvf-1.1.5/src/igbvf.o
Building modules, stage 2.
MODPOST
CC      /opt/igbvf-1.1.5/src/igbvf.mod.o
LD [M] /opt/igbvf-1.1.5/src/igbvf.ko
make[1]: Leaving directory `/usr/src/linux-2.6.16.60-0.85.1-obj/x86_64/xen'
vsles10sp4:/opt/igbvf-1.1.5/src # insmod igbvf.ko
vsles10sp4:/opt/igbvf-1.1.5/src # lspci -vk
00:00.0 Ethernet controller: Intel Corporation Device 1520 (rev 01)
        Subsystem: Intel Corporation Device 0002
        Flags: bus master, fast devsel, latency 0
        Memory at fdc00000 (32-bit, non-prefetchable) [size=16K]
        Memory at fdc20000 (32-bit, non-prefetchable) [size=16K]
        Capabilities: [70] MSI-X: Enable+ Mask- TabSize=3
        Capabilities: [a0] Express Endpoint, MSI 00
        Capabilities: [100] Advanced Error Reporting
        Capabilities: [150] Alternative Routing-ID Interpretation (ARI)
        Capabilities: [1a0] #17
        Capabilities: [1d0] Access Control Services
        Kernel driver in use: igbvf

vsles10sp4:/opt/igbvf-1.1.5/src # ifconfig
eth0      Link encap:Ethernet HWaddr 00:16:3E:5F:5E:F5
          inet addr:10.6.193.92 Bcast:10.6.193.255 Mask:255.255.255.0
          inet6 addr: fe80::216:3eff:fe5f:5ef5/64 Scope:Link
          UP BROADCAST NOTRAILERS RUNNING MULTICAST MTU:1500 Metric:1
          RX packets:162968 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39709 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:124790243 (119.0 Mb) TX bytes:2657146 (2.5 Mb)

eth1      Link encap:Ethernet HWaddr 00:16:3E:5F:5E:F6
          inet addr:192.168.1.81 Bcast:192.168.1.255 Mask:255.255.255.0
          inet6 addr: fe80::216:3eff:fe5f:5ef6/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
          RX packets:13865 errors:0 dropped:0 overruns:0 frame:0
          TX packets:336 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:2921773 (2.7 Mb) TX bytes:20036 (19.5 Kb)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1 Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING MTU:16436 Metric:1
          RX packets:10 errors:0 dropped:0 overruns:0 frame:0
          TX packets:10 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:644 (644.0 b) TX bytes:644 (644.0 b)

vsles10sp4:/opt/igbvf-1.1.5/src # ifconfig -a
eth0      Link encap:Ethernet HWaddr 00:16:3E:5F:5E:F5
          inet addr:10.6.193.92 Bcast:10.6.193.255 Mask:255.255.255.0
          inet6 addr: fe80::216:3eff:fe5f:5ef5/64 Scope:Link
          UP BROADCAST NOTRAILERS RUNNING MULTICAST MTU:1500 Metric:1
          RX packets:162988 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39709 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:124791876 (119.0 Mb) TX bytes:2657146 (2.5 Mb)

eth1      Link encap:Ethernet HWaddr 00:16:3E:5F:5E:F6
          inet addr:192.168.1.81 Bcast:192.168.1.255 Mask:255.255.255.0
          inet6 addr: fe80::216:3eff:fe5f:5ef6/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
          RX packets:13866 errors:0 dropped:0 overruns:0 frame:0
          TX packets:336 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:2921833 (2.7 Mb) TX bytes:20036 (19.5 Kb)

eth2      Link encap:Ethernet HWaddr 22:BA:70:FE:1A:4D
          BROADCAST MULTICAST MTU:1500 Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)

sit0      Link encap:IPv6-in-IPv4
          NOARP MTU:1480 Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)

trouble-shooting tools:

dom0-93:/sys/bus/pci/drivers/igb/0000:0b:00.0/virtfn0 # xm pci-list-assignable-devices
0000:0c:10.1
0000:0c:10.4
0000:0c:10.5
0000:0c:11.0
0000:0c:11.1
0000:0c:11.4
0000:0c:11.5
0000:0c:12.0
0000:0c:12.1
0000:0c:12.4
0000:0c:12.5
0000:0c:13.0
0000:0c:13.1
dom0-93:/sys/bus/pci/drivers/igb/0000:0b:00.0/virtfn0 # xm pci-list 5
Device
0000:0c:10.0

todaygood

Xen SRIOV config on sles11sp2