IcnHg

Aug 20, 2024
Technology

Modification on grub default parameters:

GRUB_CMDLINE_LINUX_DEFAULT="quiet splash iommu=pt amd_iommu=on initcall_blacklist=sysfb_init pcie_acs_override=downstream,multifunction"

Remove some items in nova:

$ vim /etc/nova/nova.conf
[libvirt]
virt_type = kvm
cpu_mode = host-passthrough
qemu_envs = QEMU_AUDIO_DRV=pa,QEMU_PA_SERVER=/run/user/1000/pulse/native
#qemu_envs = QEMU_AUDIO_DRV=pa
#qemu_args = -set,device.hostdev0.x-igd-opregion=on, -set,device.hostdev0.x-igd-gms=2, -set,device.hostdev0.romfile=/usr/share/OVMF/adls-1085.rom, -set,device.hostdev0.multifunction=on

Modification on libvirt hooks:

$ vim /etc/libvirt/hook/vfio-startup
......
# 1. find the graphical card and its associated devices in the same pci slot, record its pci id
# 1.1 find vgaid via lspci 
vgaid=`lspci | grep -i vga | awk {'print $1'}`
# 1.2 find iommu file in /sys 
vgaiommufile=`find /sys/kernel/iommu_groups/ -type l | grep $vgaid`
# 1.3 retrieve the iommu fold, which contains other device which should be processed in next step
vgaiommudir=`dirname $vgaiommufile`
# 1.4 record the system device driver in dic devicedriver
declare -A devicedriver
for i in `ls $vgaiommudir`; do devicedriver[$i]=`lspci -v -s $i | grep 'Kernel driver in use' | awk -F ': ' {'print $2'}`;done
# 1.5 record the pciid in dic pciid
declare -A pciid
for i in `ls $vgaiommudir`; do pciid[$i]=`lspci -n -s $i | awk -F ' ' {'print $3'}`;done
# 1.6 write devicedriver and pciid in /tmp
rm -f /tmp/devicedriver
rm -f /tmp/pciid
for i in `echo "${!devicedriver[@]}"`; do echo $i "${devicedriver[$i]}">>/tmp/devicedriver;done
for i in `echo "${!pciid[@]}"`; do echo $i "${pciid[$i]}">>/tmp/pciid;done

# 2.  unbind the current activated device driver mode
# example: echo "0000:06:00.0" > /sys/bus/pci/drivers/amdgpu/unbind
while IFS= read -r line; do
  pci_no=$(echo "$line" | cut -d' ' -f1)
  driver_name=$(echo "$line" | cut -d' ' -f2)
  echo $pci_no > /sys/bus/pci/drivers/$driver_name/unbind
done < /tmp/devicedriver

# 3. echo new_id to vfio-pci
# example echo $igd_id > /sys/bus/pci/drivers/vfio-pci/new_id
while IFS= read -r line; do
  pci_id=$(echo "$line" | cut -d' ' -f2)
  echo $pci_id > /sys/bus/pci/drivers/vfio-pci/new_id
done < /tmp/pciid

# Old Logical for gen10/gen12
#igd_id="8086 $(lspci -n|grep '0:02.0'|cut -d ':' -f4|cut -c 1-4)"
#echo 0000:00:02.0 > /sys/bus/pci/drivers/i915/unbind
#if ! lsmod | grep "vfio_pci" &> /dev/null ; then
#    modprobe vfio-pci
#fi
#echo $igd_id > /sys/bus/pci/drivers/vfio-pci/new_id
#echo $usb_id > /sys/bus/pci/drivers/vfio-pci/new_id

Also change teardown hook:

$ vim /etc/libvirt/hook/vfio-teardown.sh


# 1. unbind all of the vfio-pci items
# Example:  echo $igd_id > /sys/bus/pci/drivers/vfio-pci/remove_id
while IFS= read -r line; do
  pci_id=$(echo "$line" | cut -d' ' -f2)
  echo $pci_id > /sys/bus/pci/drivers/vfio-pci/remove_id
done < /tmp/pciid
# 2. re-bind system device driver
# example: echo "0000:06:00.0" > /sys/bus/pci/drivers/radeon/unbind
while IFS= read -r line; do
  pci_no=$(echo "$line" | cut -d' ' -f1)
  driver_name=$(echo "$line" | cut -d' ' -f2)
  echo $pci_no > /sys/bus/pci/drivers/$driver_name/bind
done < /tmp/devicedriver

Change the nova’s libvirt items:

$ vim /usr/lib/python2.7/dist-packages/nova/virt/libvirt/driver.py
        else:
            with open('/tmp/pciid', 'r') as f:
                for line in f:
                    #print(line.strip().split(' ')[0])
                    pci = vconfig.LibvirtConfigGuestHostdevPCI()
                    pci_addr = (line.strip().split(' ')[0])
                    pci.address = pci_addr
                    dbs, sep, pci.function = pci_addr.partition('.')
                    pci.domain, pci.bus, pci.slot = dbs.split(':')
                    if CONF.libvirt.virt_type in ('xen', 'parallels',):
                        pci.managed = 'no'
                    if CONF.libvirt.virt_type in ('kvm', 'qemu'):
                        pci.managed = 'yes'
                    guest.add_device(pci)

$ vim /usr/lib/python2.7/dist-packages/nova/virt/libvirt/config.py

    def _format_os(self, root):
        os = etree.Element("os")
        type_node = self._text_node("type", self.os_type)
        type_node.set("machine", "pc-q35-8.0")

# rm -f /usr/lib/python2.7/dist-packages/nova/virt/libvirt/driver.pyc 
# rm -f /usr/lib/python2.7/dist-packages/nova/virt/libvirt/config.pyc

But this will cause the libvirt won’t startup, cause it will need /tmp/pciid file available before creating vms, so we have to change the items for generating pciids:

$ sudo vim /bin/detectgpu.sh

#!/bin/bash
# 1. find the graphical card and its associated devices in the same pci slot, record its pci id
# 1.1 find vgaid via lspci 
vgaid=`lspci | grep -i vga | awk {'print $1'}`
# 1.2 find iommu file in /sys 
vgaiommufile=`find /sys/kernel/iommu_groups/ -type l | grep $vgaid`
# 1.3 retrieve the iommu fold, which contains other device which should be processed in next step
vgaiommudir=`dirname $vgaiommufile`
# 1.4 record the system device driver in dic devicedriver
declare -A devicedriver
for i in `ls $vgaiommudir`; do devicedriver[$i]=`lspci -v -s $i | grep 'Kernel driver in use' | awk -F ': ' {'print $2'}`;done
# 1.5 record the pciid in dic pciid
declare -A pciid
for i in `ls $vgaiommudir`; do pciid[$i]=`lspci -n -s $i | awk -F ' ' {'print $3'}`;done
# 1.6 write devicedriver and pciid in /tmp
rm -f /tmp/devicedriver
rm -f /tmp/pciid
for i in `echo "${!devicedriver[@]}"`; do echo $i "${devicedriver[$i]}">>/tmp/devicedriver;done
for i in `echo "${!pciid[@]}"`; do echo $i "${pciid[$i]}">>/tmp/pciid;done

Now remove items in hooks’s vfio-startup.sh.

Add this script in crontab:

crontab -e
added:   
@reboot /bin/detectgpu.sh

Finally we will get the amd 520 running on hygon machine.

很惭愧，就做了一点微小的工作

IcnHg