Running trs + xen on AVA

EFI / Bootloader

To avoid having to setup the boot options, I enter the EFI Boot menu by pressing Escape during the AVA booting process, then select “Boot Maintenance Manager” → “Boot from file” and select “xen.efi” from the disk where the TRS image is flashed.

Creating the Xen guest

How to change the QEMU invocation command flags for a guest

Using the device_model_override option inside a guest .cfg file we can change the QEMU binary that runs for a guest. The following Python3 script can be used as a device_model_override binary and launch the QEMU build of your choice with the command flags you want.

Custom command flags for QEMU go to the hard-coded global variable guest_device_args list in the script’s source code.
QEMU Binary path is hard-coded in the QEMU_BINARY_PATH global variable.
Environment flags can be set by adding os.environ["ENV_VAR"] = "VALUE" lines.
stdout/stderr are redirected to the path in the hard-coded LOG_FILE global variable.

Save the following source code as qemu_device_model.py, set executable permissions with chmod +x qemu_device_model.py and override the xen device model parameter in the guest .cfg file:

device_model_version="qemu-xen"
device_model_override="/home/root/qemu_device_model.py"

This will call the script with the same arguments it’d otherwise pass to the QEMU binary.

The source code:

#!/usr/bin/env python3

# SPDX-License-Identifier: GPL-3.0-or-later
# <manos.pitsidianakis@linaro.org>

import sys, signal, os
from subprocess import Popen

""" Usage

Save as `qemu_device_model.py`, set as executable
(`chmod +x qemu_device_model.py`) and add in xen guest .cfg:

device_model_version="qemu-xen"
device_model_override="/home/root/qemu_device_model.py"

"""


def handler_exit_signals(signum, frame):
    raise SystemExit(0)


signal.signal(signal.SIGINT, handler_exit_signals)
signal.signal(signal.SIGTERM, handler_exit_signals)

#os.environ["LD_LIBRARY_PATH"] = "/usr/local/lib/aarch64-linux-gnu"
QEMU_BINARY_PATH = "/usr/bin/qemu-system-aarch64"
LOG_FILE = "/home/root/qemu_log.txt"

args = sys.argv[1:]
dom_id = None
name = None

# print("args are:",args)

i = 0
try:
    while i < len(args):
        if args[i].startswith("-xen-domid"):
            dom_id = args[i + 1]
            i += 2
            continue

        if args[i].startswith("-name"):
            name = args[i + 1]
            i += 2
            continue

        i += 1
except IndexError as exc:
    print(exc, "for i =", i)
    print("Invalid command line arguments: ", args)
    sys.exit(1)

if name is None:
    name = "trs-vm"

if dom_id is None:
    print("No -xen-domid found in arguments:\nArguments were:", args)
    sys.exit(1)

libxl_cmd_socket = "/var/run/xen/qmp-libxl-{}".format(dom_id)
libxenstat_cmd_socket = "/var/run/xen/qmp-libxenstat-{}".format(dom_id)

# Create new argv with flags that include the domid and guest name first, then
# add the device flags

xen_guest_specific_args = [
    "-name",
    name,
    "-xen-domid",
    dom_id,
    "-chardev",
    "socket,id=libxl-cmd,path={},server=on,wait=off".format(libxl_cmd_socket),
    "-chardev",
    "socket,id=libxenstat-cmd,path={},server=on,wait=off".format(libxenstat_cmd_socket),
    "-mon",
    "chardev=libxl-cmd,mode=control",
    "-mon",
    "chardev=libxenstat-cmd,mode=control",
    "-nodefaults",
    "-no-user-config",
    "-xen-attach",
    "-no-shutdown",
    "-smp",
    "4,maxcpus=4",
    "-machine",
    "xenpvh",
    "-m",
    "6144",
]

guest_device_args = [
    "-vga",
    "std",
    "-vnc",
    "none",
    "-display",
    "sdl,gl=on",
    "-device",
    "virtio-gpu-pci,disable-legacy=on,iommu_platform=on,xres=1080,yres=600",
    "-global",
    "virtio-mmio.force-legacy=false",
    "-device",
    "virtio-mouse-pci,disable-legacy=on,iommu_platform=on",
    "-device",
    "virtio-keyboard-pci,disable-legacy=on,iommu_platform=on",
    "-device",
    "virtio-snd-pci,audiodev=snd0,disable-legacy=on,iommu_platform=on",
    "-audiodev",
    "alsa,id=snd0,out.dev=default",
]

new_args = [QEMU_BINARY_PATH] + xen_guest_specific_args + guest_device_args

# print("new args are:",new_args)

with open(LOG_FILE, "a") as log, Popen(new_args, stdout=log, stderr=log) as proc:
    try:
        proc.wait()
    except SystemExit:
        proc.kill()
    sys.exit(proc.returncode)

Create guest

xl create /path/to/guest.cfg

Inspect guest domain state:

xl list

Sometimes, QEMU doesn’t exit cleanly and the guest shows up as (null) . Issue pkill qemu and it should go away.

Attach to guest console

xl console -t pv -n 0 GUEST_NAME

where GUEST_NAME is the value of "name" in the guest configuration.

Example guest `.cfg` file

name = "trs-vm"
memory = 6144
vcpus = 4

# extra linux cmdline parameters

extra = " earlyprintk=xenboot console=hvc0 rw fb=false root=/dev/xvda1 modules_load=virtio_snd,virtio_gpu r
d.modules_load=virtio_snd,virtio_gpu"

# guest files on host storage

kernel = "/home/root/vmlinuz-6.5.13"
ramdisk = "/home/root/initrd.img-6.5.13"

# disks will show up as /dev/xvd{a,b}:
disk = ['format=qcow2, vdev=xvda, access=rw, backendtype=qdisk, target=/home/roo
t/debian-13-nocloud-arm64-daily-20240108-1620.qcow2', 'format=qcow2, vdev=xvdb, a
ccess=rw, backendtype=qdisk, target=/home/root/storage.qcow2']

# device parameters

device_model_version="qemu-xen"
device_model_override="/home/root/qemu_device_model.py"
virtio_qemu_domid = 0
xen_platform_pci = 1
gfx_passthru =1
vif = ['mac=00:16:3E:74:34:32,script=vif-bridge,bridge=xenbr0']

# Manpage xl.cfg(5) says:
# 
#   the type field must be set to "virtio,device" or "virtio,device<N>", 
#   where "N" is the virtio device id in hexadecimal format,
#   without the "0x" prefix and all in lower case,
#   like "virtio,device1a" for the file system device.
#
# https://xenbits.xen.org/docs/unstable/man/xl.cfg.5.html
# 
# the `bdf` field is the PCI ID the guest sees for the specific VIRTIO device.

virtio = [ "backend=0,type=virtio,device19,transport=pci,bdf=00:04.0,backend_typ
e=qemu,grant_usage=true", "backend=0,type=virtio,device10,transport=pci,bdf=00:0
5.0,backend_type=qemu,grant_usage=true", "backend=0,type=virtio,device,transport
=pci,bdf=00:03.0,backend_type=qemu,grant_usage=true" ]

# ## Type value         ID 	Virtio Device
#                     	0 	reserved (invalid)
# type=virtio,device1 	1 	network card
# type=virtio,device2 	2 	block device
# type=virtio,device3 	3 	console
# type=virtio,device4 	4 	entropy source
# type=virtio,device5 	5 	memory ballooning (traditional)
# type=virtio,device6 	6 	ioMemory
# type=virtio,device7 	7 	rpmsg
# type=virtio,device8 	8 	SCSI host
# type=virtio,device9 	9 	9P transport
# type=virtio,devicea 	10 	mac80211 wlan
# type=virtio,deviceb 	11 	rproc serial
# type=virtio,devicec 	12 	virtio CAIF
# type=virtio,deviced 	13 	memory balloon
#                         	14 	(none)
#                         	15 	(none)
# type=virtio,device10	16 	GPU device
# type=virtio,device11	17 	Timer/Clock device
# type=virtio,device12	18 	Input device
# type=virtio,device13	19 	Socket device
# type=virtio,device14	20 	Crypto device
# type=virtio,device15	21 	Signal Distribution Module
# type=virtio,device16	22 	pstore device
# type=virtio,device17	23 	IOMMU device
# type=virtio,device18	24 	Memory device
# type=virtio,device19	25 	Audio device
# type=virtio,device1a	26 	file system device
# type=virtio,device1b	27 	PMEM device
# type=virtio,device1c	28 	RPMB device
# type=virtio,device1d	29 	mac80211 hwsim wireless simulation device
# type=virtio,device1e	30 	Video encoder device
# type=virtio,device1f	31 	Video decoder device
# type=virtio,device20	32 	SCMI device
# type=virtio,device21	33 	NitroSecureModule
# type=virtio,device22	34 	I2C adapter
# type=virtio,device23	35 	Watchdog
# type=virtio,device24	36 	CAN device
# type=virtio,device25	38 	Parameter Server
# type=virtio,device26	39 	Audio policy device
# type=virtio,device27	40 	Bluetooth device
# type=virtio,device28	41 	GPIO device
# type=virtio,device29	42 	RDMA device