gravatar

很惭愧，就做了一点微小的工作

Dash

PreProductionCentOS76Libvirtd

Jun 19, 2023
Technology

System Information

Basic info:

[root@tl ~]# cat /etc/redhat-release 
CentOS Linux release 7.6.1810 (Core) 
[root@tl ~]# uname -r
3.10.0-957.el7.x86_64
[root@tl ~]# lscpu |grep "Model name"
Model name:            12th Gen Intel(R) Core(TM) i3-12100
[root@tl ~]# free -m
              total        used        free      shared  buff/cache   available
Mem:          64005         482       63299           9         223       63023
Swap:          2043           0        2043

Kernel

Install kernel and configure its default startup:

[root@tl materials]# cp -ar i915/* /lib/firmware/i915/
cp: overwrite ‘/lib/firmware/i915/bxt_dmc_ver1_07.bin’? y
cp: overwrite ‘/lib/firmware/i915/bxt_dmc_ver1.bin’? y
cp: overwrite ‘/lib/firmware/i915/bxt_guc_ver8_7.bin’? y
......
# rpm -ivh kernel-*.rpm

WorkingTipsOnLibvirtBuild

Jun 19, 2023
Technology

Steps

libvirt-7.5.0:

yum makecache
yum install -y epel-release
yum install -y meson gcc qemu-kvm wget virt-install python36-docutils python-docutils glib2-devel gnutls-devel libxml2-devel libtirpc-devel
meson --version
0.55.1
wget https://libvirt.org/sources/libvirt-7.5.0.tar.xz --no-check-certificate
tar xJvf libvirt-7.5.0.tar.xz
cd libvirt-7.5.0
meson build --prefix=/usr
ninja -C build
#ninja -C build install

libvirt-8.5.0:

yum install -y centos-release-scl
yum install devtoolset-11 -y
scl enable devtoolset-11 bash
gcc --version
gcc (GCC) 11.2.1 20220127 (Red Hat 11.2.1-9)
meson build --prefix=/usr
ninja -C build

From 8.6.0:

meson.build:986:0: ERROR: Invalid version of dependency, need 'gnutls' ['>=3.6.0'] found '3.3.29'.

From 9.1.0, Later will have meson version check:

meson.build:1:0: ERROR: Meson version is 0.55.1 but project requires >= 0.56.0

product env

build and install via:

yum install -y gnutls-devel
yum install -y python36-docutils python-docutils glib2-devel gnutls-devel libxml2-devel libtirpc-devel
meson build --prefix=/opt/local/
ninja -C build
ninja -C build install

Change configuration files:

vim /usr/lib/systemd/system/libvirtd.servic
/usr/bin/libvirtd --> /opt/local/bin/libvirtd
reboot
/opt/local/bin/virsh  list --all

Rebuild

Rebuild with full support:

yum install -y netcf-devel libpciaccess-devel systemd-devel
meson build -Dsystem=true -Ddriver_qemu=enabled -Ddriver_interface=enabled -Ddriver_libvirtd=enabled -Ddriver_remote=enabled -Ddriver_network=enabled --prefix=/usr
ninja -C build install

Edit the qemu configuration:

# vim /etc/libvirt/qemu.conf
cgroup_device_acl = [
    "/dev/null", "/dev/full", "/dev/zero",
    "/dev/random", "/dev/urandom",
    "/dev/ptmx", "/dev/kvm", "/dev/udmabuf"
]
user = "root"
group = "root"

OnBuildLibvirt

Jun 16, 2023
Technology

System libs:

libunistring-devel

gnutls >=3.6.0 is required for building libvirtd, thus:

[root@buildgen10 gnutls-3.6.0]# ./configure --prefix=/usr
  *** Libnettle 3.1 was not found.

Nettle:

 wget https://ftp.gnu.org/gnu/nettle/nettle-3.1.1.tar.gz
 tar xzvf nettle-3.1.1.tar.gz 
 cd nettle-3.1.1
 ./configure --prefix=/usr
 make
 make install

Rebuild gnutls:

./configure --prefix=/usr
make -j8
pkcs11_privkey.c:335:32: error: storage size of 'rsa_pss_params' isn't known
  struct ck_rsa_pkcs_pss_params rsa_pss_params;

https://www.frytea.com/archives/546/

RockyLinux91SRIOV

Jun 12, 2023
Technology

Kernel

Install:

tar xzvf linux-intel-lts.tar.gz
yum install -y gcc openssl-devel bc rpm-build pciutils flex bison elfutils-libelf-devel
cd linux-intel-lts
make distclean
cp ./kernel-config/x86_64_defconfig .config
echo "" | make ARCH=x86_64 olddefconfig
update-crypto-policies --set LEGACY
reboot
make rpm-pkg -j8

Replace the repository:

sed -e 's|^mirrorlist=|#mirrorlist=|g' \
    -e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.ustc.edu.cn/rocky|g' \
    -i.bak \
    /etc/yum.repos.d/rocky-extras.repo \
    /etc/yum.repos.d/rocky.repo
yum makecache

Install kernel:

cp i915/* /lib/firmware/i915/
rpm -ivh kernel-5.15.85-1.x86_64.rpm kernel-headers-5.15.85-1.x86_64.rpm
vi /etc/default/grub
Add "i915.enable_guc=0x7 udmabuf.list_limit=8192 intel_iommu=on i915.force_probe=*" to 
grub2-mkconfig -o /boot/efi/EFI/rocky/grub.cfg
reboot

Examine via:

[root@localhost ~]# uname -r
5.15.85
[root@localhost ~]# dmesg | grep GuC
[    5.176714] i915 0000:00:02.0: [drm] GuC error state capture buffer maybe too small: 2097152 < 2163708 (min = 721236)
[    5.179124] i915 0000:00:02.0: [drm] GuC firmware i915/tgl_guc_70.bin version 70.5.1
[    5.182137] i915 0000:00:02.0: [drm] GuC submission enabled
[    5.182138] i915 0000:00:02.0: [drm] GuC SLPC enabled
[    5.182455] i915 0000:00:02.0: [drm] GuC RC: enabled
[root@localhost ~]# dmesg | grep HuC
[    5.179129] i915 0000:00:02.0: [drm] HuC firmware i915/tgl_huc.bin version 7.9.3
[    5.181869] i915 0000:00:02.0: [drm] HuC authenticated
[root@localhost ~]# dmesg | grep SR-IOV
[    5.060282] i915 0000:00:02.0: Running in SR-IOV PF mode

Qemu/libvirt/virt-manager

Install following packages for using qemu:

# yum install -y virt-manager qemu-kvm
# systemctl enable libvirtd
# systemctl start libvirtd
# /usr/libexec/qemu-kvm --version
QEMU emulator version 7.2.0 (qemu-kvm-7.2.0-14.el9_2)
Copyright (c) 2003-2022 Fabrice Bellard and the QEMU Project developers
# yum groupinstall -y "Server with GUI"

sriovpreproductdev

Jun 8, 2023
Technology

Kernel

CentOS 7.6.1810, minimal installtion.

Enable keep cache:

# vi /etc/yum.conf
...
keepcache=1
...

Replace i915 firmware:

mv /lib/firmware/i915/ /lib/firmware/i915.back
tar xzvf materials/i915.tar.gz -C /lib/firmware/

Install Kernel:

# rpm -ivh materials/kernel-5.15.85+-1.x86_64.rpm
# vi /boot/grub2/grubenv
saved_entry=CentOS Linux (5.15.85+) 7 (Core)
# vi /etc/default/grub
Add "i915.enable_guc=0x7 udmabuf.list_limit=8192 intel_iommu=on i915.force_probe=*" to 
GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb quiet i915.enable_guc=0x7 udmabuf.list_limit=8192 intel_iommu=on i915.force_probe=*"
# grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
# reboot

Check the status:

[root@i3 ~]# dmesg | grep SR-IOV
[    1.025680] i915 0000:00:02.0: Running in SR-IOV PF mode
[root@i3 ~]# dmesg | grep GuC
[    1.157064] i915 0000:00:02.0: [drm] GuC error state capture buffer maybe too small: 2097152 < 2163708 (min = 721236)
[    1.161653] i915 0000:00:02.0: [drm] GuC firmware i915/tgl_guc_70.bin version 70.5.1
[    1.166302] i915 0000:00:02.0: [drm] GuC submission enabled
[    1.166304] i915 0000:00:02.0: [drm] GuC SLPC enabled
[    1.166591] i915 0000:00:02.0: [drm] GuC RC: enabled
[root@i3 ~]# dmesg | grep HuC
[    1.161660] i915 0000:00:02.0: [drm] HuC firmware i915/tgl_huc.bin version 7.9.3
[    1.166031] i915 0000:00:02.0: [drm] HuC authenticated

Qemu

Install following packages:

# yum install -y libvirt libvirt-python libguestfs-tools virt-install pciutils gcc gcc-c++ python3 git glib2-devel pixman-devel zlib-devel libusb-devel libusb libusbx-devel pulseaudio-libs-devel libcap-ng-devel libattr-devel spice-server-devel usbredir-devel centos-release-scl  unzip OVMF
# yum install -y devtoolset-8-gcc-c++
# cp materials/ninja /usr/bin
# chmod 777 /usr/bin/ninja

Build Qemu:

# tar xJvf qemu-6.2.0.tar.xz
# scl enable devtoolset-8 bash
# mkdir /opt/local
# cd qemu-6.2.0
# scl enable devtoolset-8 bash
# ./configure --target-list=x86_64-softmmu --enable-debug --disable-docs --disable-virglrenderer --prefix=/opt/local --enable-virtfs --enable-libusb --disable-debug-tcg --audio-drv-list=pa  --enable-spice --enable-usb-redir
# make -j8
# make install
# /opt/local/bin/qemu-system-x86_64 --version
QEMU emulator version 6.2.0
Copyright (c) 2003-2021 Fabrice Bellard and the QEMU Project developers

start and enable libvirtd :

# systemctl enable libvirtd
# systemctl start libvirrtd

Add following lines to /etc/rc.local:

## start of create vf
sudo modprobe i2c-algo-bit
sudo modprobe video
echo '0' | sudo tee -a /sys/bus/pci/devices/0000\:00\:02.0/sriov_drivers_autoprobe > /dev/null
echo 7 | sudo tee -a /sys/class/drm/card0/device/sriov_numvfs > /dev/null
echo '1' | sudo tee -a /sys/bus/pci/devices/0000\:00\:02.0/sriov_drivers_autoprobe > /dev/null
sudo modprobe vfio-pci
vendor=$(cat /sys/bus/pci/devices/0000:00:02.0/iommu_group/devices/0000:00:02.0/vendor)
device=$(cat /sys/bus/pci/devices/0000:00:02.0/iommu_group/devices/0000:00:02.0/device)
echo $vendor $device  | sudo tee -a /sys/bus/pci/drivers/vfio-pci/new_id
## endof create vf

Create virtual machine

Define win10 machine:

virsh define materials/wwin10.xml

Transfer the image to machine:

# scp /var/lib/libvirt/images/winpure.qcow2 root@192.168.1.120:/var/lib/libvirt/images/

Failed, no display output.

r8168

Get the driver source code from: https://www.realtek.com/en/component/zoo/category/network-interface-controllers-10-100-1000m-gigabit-ethernet-pci-express-software,

rpm -ivh kernel-devel-5.15.85+-1.x86_64.rpm
yum remove kernel-headers
rpm -ivh kernel-headers-5.15.85+-1.x86_64.rpm
yum install -y centos-release-scl
yum install devtoolset-7
scl enable devtoolset-7 bash
cd r8168-8.051.02/src/
make
modprobe r8168

Add modprobe r8168 to `/etc/rc.local

build qemu7.1

On centos76, build/install via:

scl enable devtoolset-11 bash
yum install -y git glib2-devel pixman-devel zlib-devel libusb-devel libusb libusbx-devel pulseaudio-libs-devel libcap-ng-devel libattr-devel spice-server-devel usbredir-devel python3 bzip2 
 ./configure --enable-modules --target-list=x86_64-softmmu --enable-debug --disable-docs --disable-virglrenderer --prefix=/opt/local --enable-virtfs --enable-libusb --disable-debug-tcg --audio-drv-list=pa
make -j8
make install