gravatar

很惭愧，就做了一点微小的工作

Dash

OnBuildLibvirt

Jun 16, 2023
Technology

System libs:

libunistring-devel

gnutls >=3.6.0 is required for building libvirtd, thus:

[root@buildgen10 gnutls-3.6.0]# ./configure --prefix=/usr
  *** Libnettle 3.1 was not found.

Nettle:

 wget https://ftp.gnu.org/gnu/nettle/nettle-3.1.1.tar.gz
 tar xzvf nettle-3.1.1.tar.gz 
 cd nettle-3.1.1
 ./configure --prefix=/usr
 make
 make install

Rebuild gnutls:

./configure --prefix=/usr
make -j8
pkcs11_privkey.c:335:32: error: storage size of 'rsa_pss_params' isn't known
  struct ck_rsa_pkcs_pss_params rsa_pss_params;

https://www.frytea.com/archives/546/

RockyLinux91SRIOV

Jun 12, 2023
Technology

Kernel

Install:

tar xzvf linux-intel-lts.tar.gz
yum install -y gcc openssl-devel bc rpm-build pciutils flex bison elfutils-libelf-devel
cd linux-intel-lts
make distclean
cp ./kernel-config/x86_64_defconfig .config
echo "" | make ARCH=x86_64 olddefconfig
update-crypto-policies --set LEGACY
reboot
make rpm-pkg -j8

Replace the repository:

sed -e 's|^mirrorlist=|#mirrorlist=|g' \
    -e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.ustc.edu.cn/rocky|g' \
    -i.bak \
    /etc/yum.repos.d/rocky-extras.repo \
    /etc/yum.repos.d/rocky.repo
yum makecache

Install kernel:

cp i915/* /lib/firmware/i915/
rpm -ivh kernel-5.15.85-1.x86_64.rpm kernel-headers-5.15.85-1.x86_64.rpm
vi /etc/default/grub
Add "i915.enable_guc=0x7 udmabuf.list_limit=8192 intel_iommu=on i915.force_probe=*" to 
grub2-mkconfig -o /boot/efi/EFI/rocky/grub.cfg
reboot

Examine via:

[root@localhost ~]# uname -r
5.15.85
[root@localhost ~]# dmesg | grep GuC
[    5.176714] i915 0000:00:02.0: [drm] GuC error state capture buffer maybe too small: 2097152 < 2163708 (min = 721236)
[    5.179124] i915 0000:00:02.0: [drm] GuC firmware i915/tgl_guc_70.bin version 70.5.1
[    5.182137] i915 0000:00:02.0: [drm] GuC submission enabled
[    5.182138] i915 0000:00:02.0: [drm] GuC SLPC enabled
[    5.182455] i915 0000:00:02.0: [drm] GuC RC: enabled
[root@localhost ~]# dmesg | grep HuC
[    5.179129] i915 0000:00:02.0: [drm] HuC firmware i915/tgl_huc.bin version 7.9.3
[    5.181869] i915 0000:00:02.0: [drm] HuC authenticated
[root@localhost ~]# dmesg | grep SR-IOV
[    5.060282] i915 0000:00:02.0: Running in SR-IOV PF mode

Qemu/libvirt/virt-manager

Install following packages for using qemu:

# yum install -y virt-manager qemu-kvm
# systemctl enable libvirtd
# systemctl start libvirtd
# /usr/libexec/qemu-kvm --version
QEMU emulator version 7.2.0 (qemu-kvm-7.2.0-14.el9_2)
Copyright (c) 2003-2022 Fabrice Bellard and the QEMU Project developers
# yum groupinstall -y "Server with GUI"

sriovpreproductdev

Jun 8, 2023
Technology

Kernel

CentOS 7.6.1810, minimal installtion.

Enable keep cache:

# vi /etc/yum.conf
...
keepcache=1
...

Replace i915 firmware:

mv /lib/firmware/i915/ /lib/firmware/i915.back
tar xzvf materials/i915.tar.gz -C /lib/firmware/

Install Kernel:

# rpm -ivh materials/kernel-5.15.85+-1.x86_64.rpm
# vi /boot/grub2/grubenv
saved_entry=CentOS Linux (5.15.85+) 7 (Core)
# vi /etc/default/grub
Add "i915.enable_guc=0x7 udmabuf.list_limit=8192 intel_iommu=on i915.force_probe=*" to 
GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb quiet i915.enable_guc=0x7 udmabuf.list_limit=8192 intel_iommu=on i915.force_probe=*"
# grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
# reboot

Check the status:

[root@i3 ~]# dmesg | grep SR-IOV
[    1.025680] i915 0000:00:02.0: Running in SR-IOV PF mode
[root@i3 ~]# dmesg | grep GuC
[    1.157064] i915 0000:00:02.0: [drm] GuC error state capture buffer maybe too small: 2097152 < 2163708 (min = 721236)
[    1.161653] i915 0000:00:02.0: [drm] GuC firmware i915/tgl_guc_70.bin version 70.5.1
[    1.166302] i915 0000:00:02.0: [drm] GuC submission enabled
[    1.166304] i915 0000:00:02.0: [drm] GuC SLPC enabled
[    1.166591] i915 0000:00:02.0: [drm] GuC RC: enabled
[root@i3 ~]# dmesg | grep HuC
[    1.161660] i915 0000:00:02.0: [drm] HuC firmware i915/tgl_huc.bin version 7.9.3
[    1.166031] i915 0000:00:02.0: [drm] HuC authenticated

Qemu

Install following packages:

# yum install -y libvirt libvirt-python libguestfs-tools virt-install pciutils gcc gcc-c++ python3 git glib2-devel pixman-devel zlib-devel libusb-devel libusb libusbx-devel pulseaudio-libs-devel libcap-ng-devel libattr-devel spice-server-devel usbredir-devel centos-release-scl  unzip OVMF
# yum install -y devtoolset-8-gcc-c++
# cp materials/ninja /usr/bin
# chmod 777 /usr/bin/ninja

Build Qemu:

# tar xJvf qemu-6.2.0.tar.xz
# scl enable devtoolset-8 bash
# mkdir /opt/local
# cd qemu-6.2.0
# scl enable devtoolset-8 bash
# ./configure --target-list=x86_64-softmmu --enable-debug --disable-docs --disable-virglrenderer --prefix=/opt/local --enable-virtfs --enable-libusb --disable-debug-tcg --audio-drv-list=pa  --enable-spice --enable-usb-redir
# make -j8
# make install
# /opt/local/bin/qemu-system-x86_64 --version
QEMU emulator version 6.2.0
Copyright (c) 2003-2021 Fabrice Bellard and the QEMU Project developers

start and enable libvirtd :

# systemctl enable libvirtd
# systemctl start libvirrtd

Add following lines to /etc/rc.local:

## start of create vf
sudo modprobe i2c-algo-bit
sudo modprobe video
echo '0' | sudo tee -a /sys/bus/pci/devices/0000\:00\:02.0/sriov_drivers_autoprobe > /dev/null
echo 7 | sudo tee -a /sys/class/drm/card0/device/sriov_numvfs > /dev/null
echo '1' | sudo tee -a /sys/bus/pci/devices/0000\:00\:02.0/sriov_drivers_autoprobe > /dev/null
sudo modprobe vfio-pci
vendor=$(cat /sys/bus/pci/devices/0000:00:02.0/iommu_group/devices/0000:00:02.0/vendor)
device=$(cat /sys/bus/pci/devices/0000:00:02.0/iommu_group/devices/0000:00:02.0/device)
echo $vendor $device  | sudo tee -a /sys/bus/pci/drivers/vfio-pci/new_id
## endof create vf

Create virtual machine

Define win10 machine:

virsh define materials/wwin10.xml

Transfer the image to machine:

# scp /var/lib/libvirt/images/winpure.qcow2 root@192.168.1.120:/var/lib/libvirt/images/

Failed, no display output.

r8168

Get the driver source code from: https://www.realtek.com/en/component/zoo/category/network-interface-controllers-10-100-1000m-gigabit-ethernet-pci-express-software,

rpm -ivh kernel-devel-5.15.85+-1.x86_64.rpm
yum remove kernel-headers
rpm -ivh kernel-headers-5.15.85+-1.x86_64.rpm
yum install -y centos-release-scl
yum install devtoolset-7
scl enable devtoolset-7 bash
cd r8168-8.051.02/src/
make
modprobe r8168

Add modprobe r8168 to `/etc/rc.local

build qemu7.1

On centos76, build/install via:

scl enable devtoolset-11 bash
yum install -y git glib2-devel pixman-devel zlib-devel libusb-devel libusb libusbx-devel pulseaudio-libs-devel libcap-ng-devel libattr-devel spice-server-devel usbredir-devel python3 bzip2 
 ./configure --enable-modules --target-list=x86_64-softmmu --enable-debug --disable-docs --disable-virglrenderer --prefix=/opt/local --enable-virtfs --enable-libusb --disable-debug-tcg --audio-drv-list=pa
make -j8
make install

WorkingTipsOnGen12CentOS

Jun 7, 2023
Technology

CentOS7.6.1810,minimum installation.

# yum makecache
# yum install -y vim git gcc libevent-devel unzip flex bison
# vim /etc/selinux/config
SELINUX=disabled
# systemctl disable firewalld

Install gcc-7:

yum install centos-release-scl
yum install devtoolset-7-gcc-c++
scl enable devtoolset-7 bash

Install kernel related dependencies:

yum install -y devtoolset-7-elfutils-libelf-devel.x86_64  openssl-devel bc rpm-build pciutils

Clone the kernel code:

# mkdir Code
# cd Code
# git clone https://github.com/intel/linux-intel-lts.git
# cd linux-intel-lts
# git checkout gawougoweugowugo
# cp ~/kernel-config.zip .
# unzip kernel-config.zip
# cp ./kernel-config/x86_64_defconfig .config
# echo "" | make ARCH=x86_64 olddefconfig
# make ARCH=x86_64 -j16 LOCALVERSION=-lts2021-iotg -j8

Verification

Make sure you have the latest firmware(copy from the ubuntu):

# mkdir /lib/firmware/i915/backup
# mv /lib/firmware/i915/* /lib/firmware/i915/backup
# cp -ar /mnt/lib/firmware/i915/* /lib/firmware/i915/

Install the minimal system, install the built kernel rpms, then edit the grub:

# vim /etc/default/grub
......
GRUB_CMDLINE_LINUX="crashkernel=auto rhgb quiet i915.enable_guc=0x7 udmabuf.list_limit=8192 intel_iommu=on i915.force_probe=*"
......
# grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
# cat /boot/grub2/grubenv 
    # GRUB Environment Block
    #saved_entry=CentOS Linux (3.10.0-957.el7.x86_64) 7 (Core)
    saved_entry=CentOS Linux (5.15.85+) 7 (Core)
# reboot

Check the status of SR-IOV:

# dmesg | grep SR-IOV
[    1.643471] i915 0000:00:02.0: Running in SR-IOV PF mode
# dmesg | grep GuC
[    1.844856] i915 0000:00:02.0: [drm] GuC error state capture buffer maybe too small: 2097152 < 2557128 (min = 852376)
[    1.846949] i915 0000:00:02.0: [drm] GuC firmware i915/tgl_guc_70.bin version 70.5.1
[    1.850099] i915 0000:00:02.0: [drm] GuC submission enabled
[    1.850099] i915 0000:00:02.0: [drm] GuC SLPC enabled
[    1.850430] i915 0000:00:02.0: [drm] GuC RC: enabled
# dmesg | grep HuC
[    1.846952] i915 0000:00:02.0: [drm] HuC firmware i915/tgl_huc.bin version 7.9.3
[    1.849623] i915 0000:00:02.0: [drm] HuC authenticated

Install necessary packages:

# yum install -y virt-manager
# yum groupinstall "GNOME Desktop"

Upgrade qemu:

# yum install centos-release-qemu-ev
# yum makecache
# yum install -y qemu-kvm-ev qemu-img-ev
# /usr/libexec/qemu-kvm --version
QEMU emulator version 2.12.0 (qemu-kvm-ev-2.12.0-44.1.el7_8.1)
Copyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers

Create vf:

[root@cs76 ~]# lspci | grep -i vga
0000:00:02.0 VGA compatible controller: Intel Corporation Device 4680 (rev 0c)
[root@cs76 ~]# chmod 777 createvf.sh 
.[root@cs76 ~]# ./createvf.sh 
0x8086 0x4680
[root@cs76 ~]# lspci | grep -i vga
0000:00:02.0 VGA compatible controller: Intel Corporation Device 4680 (rev 0c)
0000:00:02.1 VGA compatible controller: Intel Corporation Device 4680 (rev 0c)
0000:00:02.2 VGA compatible controller: Intel Corporation Device 4680 (rev 0c)
0000:00:02.3 VGA compatible controller: Intel Corporation Device 4680 (rev 0c)
0000:00:02.4 VGA compatible controller: Intel Corporation Device 4680 (rev 0c)
0000:00:02.5 VGA compatible controller: Intel Corporation Device 4680 (rev 0c)
0000:00:02.6 VGA compatible controller: Intel Corporation Device 4680 (rev 0c)
0000:00:02.7 VGA compatible controller: Intel Corporation Device 4680 (rev 0c)

Install some management tools:

yum install libvirt libvirt-python libguestfs-tools virt-install -y

switch to qemu 6.2

build ninja:

yum install -y gcc gcc-c++ python3 git
git clone https://github.com/ninja-build/ninja.git
cd ninja
git checkout release
./configure.py --bootstrap
cp ninja /usr/bin

Install qemu build dependencies:

yum install -y glib2-devel pixman-devel zlib-devel libusb-devel libusb libusbx-devel pulseaudio-libs-devel libcap-ng-devel libattr-devel spice-server-devel usbredir-devel

Since qemu 6.2 requires gcc>7.6? Install gcc 8 for building:

yum install devtoolset-8-gcc-c++
scl enable devtoolset-8 bash
./configure --target-list=x86_64-softmmu --enable-debug --disable-docs --disable-virglrenderer --prefix=/opt/local --enable-virtfs --enable-libusb --disable-debug-tcg --audio-drv-list=pa  --enable-spice --enable-usb-redir
make -j8
make install

Change from default qemu to /opt/local/bin/qemu-system-x86_64, then start the machine.
You may also need virsh or other tools:

yum install -y libvirt libvirt-python libguestfs-tools virt-install -y

Gen12GPUPassthrough

May 8, 2023
Technology

Ubuntu 22.04.2 desktop cdrom installation on Gen12 machine.

sudo apt update -y && sudo apt upgrade -y
sudo apt install -y openssh-server virt-manager qemu vim

Edit the grub and initramfs-tools, and vfio:

$ cat /etc/default/grub | grep CMDLINE_LINUX_DEFAULT
GRUB_CMDLINE_LINUX_DEFAULT="quiet  intel_iommu=on intel_iommu=pt kvm.ignore_msrs=1 video=efifb:off,vesafb:off"
$ cat /etc/initramfs-tools/modules
vfio
vfio_iommu_type1
vfio_pci
vfio_virqfd
$  echo "options vfio-pci disable_vga=1" > /etc/modprobe.d/vfio.conf
$ sudo update-grub2 && sudo update-initramfs -u -k all
$ sudo reboot

Build ovmf:

$ cp /home/idv/xxxxxxxxx/VBT/Vbt.bin OvmfPkg/Vbt/Vbt.bin 
$ cp /home/idv/xxxxxxxxx/./IntelGopDriver/RELEASE_VS2015x86/X64/IntelGopDriver.efi OvmfPkg/IntelGop/IntelGopDriver.efi
$ pwd
/home/idv/Code/Intel_edk2/edk2_gen12
Build the ovmf(follow the guideline)

Install an ubuntu22.04 vm in host machine, with qxl/spice.

glmark2: score 1711

Install win11(shift+f10), use regedit for editing:

/images/2023_05_08_11_23_09_794x480.jpg

Fallback and continue install:

/images/2023_05_08_11_24_16_769x553.jpg