How to override VM UEFI boot in LXD 5.3 and use seabios?

Sorry I’ve tried research this but there is to much I’m missing so need to ask…

Just trying to run the Dell OpenManage 3.9 QCOW2 image as a VM in LXD 5.3-924be6a.
Started here: Instance configuration - LXD documentation
Then couldn’t find a nice list of what a qemu.conf would look like.

Found that ps -ef revealed a lot, so installed qemu-kvm and configured 2 VMs the same apart from:

  1. a VM using BIOS boot
  2. a VM using UEFI boot

then used ps -ef to see the difference.

The BIOS boot

/usr/bin/qemu-system-x86_64
-name guest=dellome,debug-threads=on
-S
-object {"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-3-dellome/master-key.aes"}
-machine pc-q35-6.2,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram
-accel kvm
-cpu host,migratable=on
-m 4096
-object {"qom-type":"memory-backend-ram","id":"pc.ram","size":4294967296}
-overcommit mem-lock=off
-smp 4,sockets=4,cores=1,threads=1
-uuid 29cebb17-fd0f-438c-b34d-d9fad2b9e3df
-no-user-config
-nodefaults
-chardev socket,id=charmonitor,fd=33,server=on,wait=off
-mon chardev=charmonitor,id=monitor,mode=control
-rtc base=utc,driftfix=slew
-global kvm-pit.lost_tick_policy=delay
-no-hpet
-no-shutdown
-global ICH9-LPC.disable_s3=1
-global ICH9-LPC.disable_s4=1
-boot strict=on
-device pcie-root-port,port=16,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2
-device pcie-root-port,port=17,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1
-device pcie-root-port,port=18,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2
-device pcie-root-port,port=19,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3
-device pcie-root-port,port=20,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4
-device pcie-root-port,port=21,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5
-device pcie-root-port,port=22,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6
-device pcie-root-port,port=23,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7
-device pcie-root-port,port=24,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x3
-device pcie-root-port,port=25,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1
-device pcie-root-port,port=26,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2
-device pcie-root-port,port=27,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3
-device pcie-root-port,port=28,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4
-device pcie-root-port,port=29,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5
-device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0
-device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0
-blockdev {"driver":"file","filename":"/var/lib/libvirt/images/openmanage_enterprise.qcow2","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}
-blockdev {"node-name":"libvirt-1-format","read-only":false,"driver":"qcow2","file":"libvirt-1-storage","backing":null}
-device virtio-blk-pci,bus=pci.4,addr=0x0,drive=libvirt-1-format,id=virtio-disk0,bootindex=1
-netdev tap,fd=34,id=hostnet0,vhost=on,vhostfd=36
-device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:bc:0d:ff,bus=pci.1,addr=0x0
-chardev pty,id=charserial0
-device isa-serial,chardev=charserial0,id=serial0
-chardev socket,id=charchannel0,fd=32,server=on,wait=off
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0
-chardev spicevmc,id=charchannel1,name=vdagent
-device virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel1,name=com.redhat.spice.0
-device usb-tablet,id=input0,bus=usb.0,port=1
-audiodev {"id":"audio1","driver":"spice"}
-spice port=5900,addr=127.0.0.1,disable-ticketing=on,image-compression=off,seamless-migration=on
-device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1
-device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b
-device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0,audiodev=audio1
-chardev spicevmc,id=charredir0,name=usbredir
-device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2
-chardev spicevmc,id=charredir1,name=usbredir
-device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3
-device virtio-balloon-pci,id=balloon0,bus=pci.5,addr=0x0
-object {"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"}
-device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.6,addr=0x0
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
-msg timestamp=on

The UEFI boot

/usr/bin/qemu-system-x86_64
-name guest=dellome,debug-threads=on
-S
-object {"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-4-dellome/master-key.aes"}
-blockdev {"driver":"file","filename":"/usr/share/OVMF/OVMF_CODE_4M.ms.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}
-blockdev {"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}
-blockdev {"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/dellome_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}
-blockdev {"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}
-machine pc-q35-6.2,usb=off,vmport=off,smm=on,dump-guest-core=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram
-accel kvm
-cpu host,migratable=on
-m 4096
-object {"qom-type":"memory-backend-ram","id":"pc.ram","size":4294967296}
-overcommit mem-lock=off
-smp 4,sockets=4,cores=1,threads=1
-uuid c5924e21-9f92-426c-bd4a-a7466965cb1b
-no-user-config
-nodefaults
-chardev socket,id=charmonitor,fd=33,server=on,wait=off
-mon chardev=charmonitor,id=monitor,mode=control
-rtc base=utc,driftfix=slew
-global kvm-pit.lost_tick_policy=delay
-no-hpet
-no-shutdown
-global ICH9-LPC.disable_s3=1
-global ICH9-LPC.disable_s4=1
-boot strict=on
-device pcie-root-port,port=16,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2
-device pcie-root-port,port=17,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1
-device pcie-root-port,port=18,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2
-device pcie-root-port,port=19,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3
-device pcie-root-port,port=20,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4
-device pcie-root-port,port=21,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5
-device pcie-root-port,port=22,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6
-device pcie-root-port,port=23,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7
-device pcie-root-port,port=24,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x3
-device pcie-root-port,port=25,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1
-device pcie-root-port,port=26,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2
-device pcie-root-port,port=27,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3
-device pcie-root-port,port=28,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4
-device pcie-root-port,port=29,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5
-device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0
-device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0
-blockdev {"driver":"file","filename":"/var/lib/libvirt/images/openmanage_enterprise.qcow2","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}
-blockdev {"node-name":"libvirt-1-format","read-only":false,"driver":"qcow2","file":"libvirt-1-storage","backing":null}
-device virtio-blk-pci,bus=pci.4,addr=0x0,drive=libvirt-1-format,id=virtio-disk0,bootindex=1
-netdev tap,fd=34,id=hostnet0,vhost=on,vhostfd=36
-device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:f6:71:bc,bus=pci.1,addr=0x0
-chardev pty,id=charserial0
-device isa-serial,chardev=charserial0,id=serial0
-chardev socket,id=charchannel0,fd=32,server=on,wait=off
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0
-chardev spicevmc,id=charchannel1,name=vdagent
-device virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel1,name=com.redhat.spice.0
-device usb-tablet,id=input0,bus=usb.0,port=1
-audiodev {"id":"audio1","driver":"spice"}
-spice port=5900,addr=127.0.0.1,disable-ticketing=on,image-compression=off,seamless-migration=on
-device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1
-device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b
-device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0,audiodev=audio1
-chardev spicevmc,id=charredir0,name=usbredir
-device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2
-chardev spicevmc,id=charredir1,name=usbredir
-device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3
-device virtio-balloon-pci,id=balloon0,bus=pci.5,addr=0x0
-object {"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"}
-device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.6,addr=0x0
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
-msg timestamp=on

The XML in virt-manager for BIOS boot:

    <os>
    <type arch="x86_64" machine="q35">hvm</type>
    <boot dev="hd"/>
  </os>

The XML in virt-manager for UEFI boot:

  <os firmware="efi">
    <type arch="x86_64" machine="q35">hvm</type>
    <boot dev="hd"/>
  </os>

So I guesses that I would need to override the [machine] section based on…
BIOS boot

-machine pc-q35-6.2,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram

UEFI boot

-machine pc-q35-6.2,usb=off,vmport=off,smm=on,dump-guest-core=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram

and then added the following to the LXD VM instance config:

  raw.qemu.conf: |-
    [machine]
    smm = ""
    pflash0 = ""
    pflash1 = ""

and it attempted to boot UEFI :crazy_face:

The LXD VM qemu.conf:

> cat /var/snap/lxd/common/lxd/logs/dellome/qemu.conf
# Machine
[machine]
graphics = "off"
type = "q35"
accel = "kvm"
usb = "off"

[global]
driver = "ICH9-LPC"
property = "disable_s3"
value = "1"

[global]
driver = "ICH9-LPC"
property = "disable_s4"
value = "1"

[boot-opts]
strict = "on"

# Memory
[memory]
size = "3072M"

# CPU
[smp-opts]
cpus = "4"
sockets = "1"
cores = "4"
threads = "1"

[object "mem0"]
qom-type = "memory-backend-memfd"
size = "3072M"
share = "on"

[numa]
type = "node"
nodeid = "0"
memdev = "mem0"

# Firmware (read only)
[drive]
file = "/snap/lxd/current/share/qemu/OVMF_CODE.fd"
if = "pflash"
format = "raw"
unit = "0"
readonly = "on"

# Firmware settings (writable)
[drive]
file = "/dev/fd/3"
if = "pflash"
format = "raw"
unit = "1"

# Qemu control
[chardev "monitor"]
backend = "socket"
path = "/var/snap/lxd/common/lxd/logs/dellome/qemu.monitor"
server = "on"
wait = "off"

[mon]
chardev = "monitor"
mode = "control"

# Console
[chardev "console"]
backend = "socket"
path = "/var/snap/lxd/common/lxd/logs/dellome/qemu.console"
server = "on"
wait = "off"

[device "qemu_pcie0"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.0"
chassis = "0"
multifunction = "on"

# Balloon driver
[device "qemu_balloon"]
driver = "virtio-balloon-pci"
bus = "qemu_pcie0"
addr = "00.0"
multifunction = "on"

# Random number generator
[object "qemu_rng"]
qom-type = "rng-random"
filename = "/dev/urandom"

[device "dev-qemu_rng"]
driver = "virtio-rng-pci"
bus = "qemu_pcie0"
addr = "00.1"
rng = "qemu_rng"

# Input
[device "qemu_keyboard"]
driver = "virtio-keyboard-pci"
bus = "qemu_pcie0"
addr = "00.2"

# Input
[device "qemu_tablet"]
driver = "virtio-tablet-pci"
bus = "qemu_pcie0"
addr = "00.3"

# Vsock
[device "qemu_vsock"]
driver = "vhost-vsock-pci"
bus = "qemu_pcie0"
addr = "00.4"
guest-cid = "6"

# Virtual serial bus
[device "dev-qemu_serial"]
driver = "virtio-serial-pci"
bus = "qemu_pcie0"
addr = "00.5"

# LXD serial identifier
[chardev "qemu_serial-chardev"]
backend = "ringbuf"
size = "16B"

[device "qemu_serial"]
driver = "virtserialport"
name = "org.linuxcontainers.lxd"
chardev = "qemu_serial-chardev"
bus = "dev-qemu_serial.0"

# Spice agent
[chardev "qemu_spice-chardev"]
backend = "spicevmc"
name = "vdagent"

[device "qemu_spice"]
driver = "virtserialport"
name = "com.redhat.spice.0"
chardev = "qemu_spice-chardev"
bus = "dev-qemu_serial.0"

# Spice folder
[chardev "qemu_spicedir-chardev"]
backend = "spiceport"
name = "org.spice-space.webdav.0"

[device "qemu_spicedir"]
driver = "virtserialport"
name = "org.spice-space.webdav.0"
chardev = "qemu_spicedir-chardev"
bus = "dev-qemu_serial.0"

# USB controller
[device "qemu_usb"]
driver = "qemu-xhci"
bus = "qemu_pcie0"
addr = "00.6"
p2 = "8"
p3 = "8"

[chardev "qemu_spice-usb-chardev1"]
backend = "spicevmc"
name = "usbredir"

[device "qemu_spice-usb1"]
driver = "usb-redir"
chardev = "qemu_spice-usb-chardev1"

[chardev "qemu_spice-usb-chardev2"]
backend = "spicevmc"
name = "usbredir"

[device "qemu_spice-usb2"]
driver = "usb-redir"
chardev = "qemu_spice-usb-chardev2"

[chardev "qemu_spice-usb-chardev3"]
backend = "spicevmc"
name = "usbredir"

[device "qemu_spice-usb3"]
driver = "usb-redir"
chardev = "qemu_spice-usb-chardev3"

[device "qemu_pcie1"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.1"
chassis = "1"

# SCSI controller
[device "qemu_scsi"]
driver = "virtio-scsi-pci"
bus = "qemu_pcie1"
addr = "00.0"

[device "qemu_pcie2"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.2"
chassis = "2"

# Config drive (9p)
[fsdev "qemu_config"]
fsdriver = "local"
security_model = "none"
readonly = "on"
path = "/var/snap/lxd/common/lxd/devices/dellome/config.mount"

[device "dev-qemu_config-drive-9p"]
driver = "virtio-9p-pci"
bus = "qemu_pcie2"
addr = "00.0"
multifunction = "on"
mount_tag = "config"
fsdev = "qemu_config"

# Config drive (virtio-fs)
[chardev "qemu_config"]
backend = "socket"
path = "/var/snap/lxd/common/lxd/logs/dellome/virtio-fs.config.sock"

[device "dev-qemu_config-drive-virtio-fs"]
driver = "vhost-user-fs-pci"
bus = "qemu_pcie2"
addr = "00.1"
tag = "config"
chardev = "qemu_config"

[device "qemu_pcie3"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.3"
chassis = "3"

# GPU
[device "qemu_gpu"]
driver = "virtio-vga"
bus = "qemu_pcie3"
addr = "00.0"

[device "qemu_pcie4"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.4"
chassis = "4"

[device "qemu_pcie5"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.5"
chassis = "5"

[device "qemu_pcie6"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.6"
chassis = "6"

[device "qemu_pcie7"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.7"
chassis = "7"

[device "qemu_pcie8"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "2.0"
chassis = "8"
multifunction = "on"

So I’m missing some settings that need to be overwritten but I don’t know what they are, so if you could point me in the right direction please.

Thanks

There is the raw.qemu setting for passing command line options to qemu too, which may help.

There’s an example of setting the machine type here:

There’s also a comment about using a non-UEFI bios here:

Thanks for your suggestions.

I’ve event tried setting the q35 version from 6.2 down to 2.6 like this:

raw.qemu: -machine pc-q35-2.6,usb=off,vmport=off,smm=off,dump-guest-core=off -drive
    file=/var/snap/lxd/common/lxd/logs/dellome/openmanage_enterprise.qcow2,if=virtio

But just keep getting this UEFI boot message

When I compare the command line that qemu-kvm runs between the BIOS boot vs the UEFI VM, there is very little at the top…

will keep scratching.

Interesting development after reading this:

The pflash value tells QEMU that the specified files are parallel flash images (ROM firmware).

So when using lxc init {image} {instance}... the pflash stuff isn’t added to the /var/snap/lxd/common/lxd/logs/dellome1/qemu.conf until the machine is started for the first time, which look like this:

# Firmware (read only)
[drive]
file = "/snap/lxd/current/share/qemu/OVMF_CODE.fd"
if = "pflash"
format = "raw"
unit = "0"
readonly = "on"

# Firmware settings (writable)
[drive]
file = "/dev/fd/3"
if = "pflash"
format = "raw"
unit = "1"

Based on the docs to override qemu.conf, tried to use a section index to override:

So my instance config now contains:

  raw.qemu.conf: |-
    [drive][0]
    [drive][1]

and when I start the VM, LXD errors

...: Process exited with non-zero value 1
Try `lxc info --show-log dellome1` for more info

and in there, sadly :sob:

qemu: could not load PC BIOS 'bios-256k.bin'

That’s most likely because we just don’t have that file in our build of QEMU.
You may be able to pass in a full bios path to an external version of that binary.

Thanks,

Well I’ve certainly gone the real long way around this one, but that error message lead me to this new feature:

In the end I just used raw.qemu instead of raw.qemu.conf.

So all good now with the Dell OME and the Citrix vADC images, which is about all I need because I just doing have the time to get them to work with UEFI and then maintain them. Only thing left to do is making the binary more accessible on the file system to instances rather than putting in the logs dir, but I have bumped into some topics on that.

Thanks so much for the great help and have a wonderful evening!

1 Like

Thanks! For posterity’s sake please could toy show the output of "lxc config show " to show how you achieved it. Thanks

Sure, but I still haven’t figured out how to make the binary available in a common directory for multiple instances (well only 2 in my case), so just manually copying the binary into the /var/snap/lxd/common/lxd/logs/{instanceName}/ dir. Here are the steps so far…

Installed seabios package on the host:

sudo apt install seabios

Which installs the binaries to /usr/share/seabios but due to namespaces we cannot just do this in the instance config:

raw.qemu: -bios /usr/share/seabios/bios-256k.bin

As said at the moment, just manually copying the bios-256k.bin into the /var/snap/lxd/common/lxd/logs/{instanceName}/ dir, and then in the instance config, set this for my Dell OpenManage Enterprise appliance (dellome1):

raw.qemu: -bios /var/snap/lxd/common/lxd/logs/dellome1/bios-256k.bin

That’s all I needed to do with that one because it’s a Centos7 based appliance.

From my local test env:


lxc config show dellome1 --expanded 
architecture: x86_64
config:
  image.architecture: amd64
  image.description: Dell OpenManage Enterprise 3.9
  image.name: dell-openmanage-3.9
  image.os: centos
  image.release: 3.9.0
  image.serial: "20220628_07:42"
  image.variant: default
  limits.cpu: "4"
  limits.memory: 8GiB
  raw.qemu: -bios /var/snap/lxd/common/lxd/logs/dellome1/bios-256k.bin
  volatile.base_image: 240803a3c4a639ec082f17bd40d586eb74e7ad549ad4987a122b3b3396626b8b
  volatile.cloud-init.instance-id: 19dfc2b2-78a3-4d0e-851c-e72995d111c8
  volatile.eth0.hwaddr: 00:16:3e:c4:6b:73
  volatile.last_state.power: STOPPED
  volatile.uuid: 65199a00-d052-48a2-8dd7-23c13a8acb2f
  volatile.vsock_id: "16"
devices:
  eth0:
    name: eth0
    network: lxdbr0
    type: nic
  root:
    path: /
    pool: default
    type: disk
ephemeral: false
profiles:
- default
stateful: false
description: ""

For the Citrix vADC which is a freeBSD based appliance, the BIOS is now fine but I’m running into the disks issue, for which there are other forum posts I need to look at, like the pfSense ones.

It would be good to have it look at that seabios package’s directory somehow (some sort of bind mount into the instance namespace via the config/profile) rather than the manual copy, which will not help with HA or moves, but that’s not going to be much an issue in my scenario. Even better would be some storage in the DB, bit like dkr Secrets so this stuff (BIOS, certs, scripts, configs, etc) can be reused injected, and clustered… not sure it’s the right thing though for LXD as I’ll probably end up doing something with Ansible or the likes of.

For the disks issue with the freeBSD based Citrix vADC appliance, added -machine pc-q35-2.6 as seen in many freeBSD / pfSense topics here, to instance config raw.qemu and that was it. Not tested how new we can go with this image but 2.6 works.

Would be nice is there is a better way of mapping / mounting the BIOS file to instance namespace thingy, rather than copying it to the instance logs, so suggestions welcome thanks.

Citrix vADC instance config in my test env:

lxc config show citrixns1 --expanded

architecture: x86_64
config:
  image.architecture: amd64
  image.description: Citrix vADC (NSVPX) 13.1-12.51
  image.name: citrx-vadc-13.1-12.51
  image.os: freebsd
  image.release: 13.1-12.51
  image.serial: "20220628_07:42"
  image.variant: default
  limits.cpu: "4"
  limits.memory: 4GiB
  raw.qemu: -bios /var/snap/lxd/common/lxd/logs/citrixns1/bios-256k.bin -machine pc-q35-2.6
  volatile.base_image: ba3f131c49b21f4699e01ea47952a352b1bcf06972d8a965cdf3e1ef885d4640
  volatile.cloud-init.instance-id: f486526e-c781-4e76-854b-0928c177cdd5
  volatile.eth0.hwaddr: 00:16:3e:4e:ea:81
  volatile.last_state.power: STOPPED
  volatile.uuid: 9f211343-660f-4ff8-a71e-b38116215d2e
  volatile.vsock_id: "17"
devices:
  eth0:
    name: eth0
    network: lxdbr0
    type: nic
  root:
    path: /
    pool: default
    type: disk
ephemeral: false
profiles:
- default
stateful: false
description: ""

Thanks.

The PR that @stgraber added to allow that mentions:

This allows for booting LXD off of an external firmware file or directly
into a specific kernel. Such firmware or kernel will likely need a
matching raw.apparmor entry so that QEMU can actually access them.

Also, you can reference the host’s filesystem outside of the snap package using /var/lib/snapd/hostfs.

So something like this should work:

lxc config set <instance> raw.qemu=" -bios /var/lib/snapd/hostfs/path/to/bios-256k.bin -machine pc-q35-2.6"
printf "/var/lib/snapd/hostfs/path/to/bios-256k.bin r," | lxc config set <instance> raw.apparmor -

See for more examples of using raw.apparmor:

Perfect, thank you so much for that very useful info and worked out perfectly!

Did a bind mount from /usr/share/seabios to /var/lib/snap/hostfs/usr/share/seabios, did updated the instance conf with the new path and the apparmor setting, so now looks like this for the freeBSD based Citrix vADC:

lxc config show citrixns1 --expanded
architecture: x86_64
config:
  image.architecture: amd64
  image.description: Citrix vADC (NSVPX) 13.1-12.51
  image.name: citrx-vadc-13.1-12.51
  image.os: freebsd
  image.release: 13.1-12.51
  image.serial: "20220628_07:42"
  image.variant: default
  limits.cpu: "4"
  limits.memory: 4GiB
  raw.apparmor: /var/lib/snapd/hostfs/usr/share/seabios/bios-256k.bin r,
  raw.qemu: -bios /var/lib/snapd/hostfs/usr/share/seabios/bios-256k.bin -machine pc-q35-2.6
  volatile.base_image: ba3f131c49b21f4699e01ea47952a352b1bcf06972d8a965cdf3e1ef885d4640
  volatile.cloud-init.instance-id: f486526e-c781-4e76-854b-0928c177cdd5
  volatile.eth0.host_name: tap3a82a88a
  volatile.eth0.hwaddr: 00:16:3e:4e:ea:81
  volatile.last_state.power: RUNNING
  volatile.uuid: 9f211343-660f-4ff8-a71e-b38116215d2e
  volatile.vsock_id: "17"
devices:
  eth0:
    name: eth0
    network: lxdbr0
    type: nic
  root:
    path: /
    pool: default
    type: disk
ephemeral: false
profiles:
- default
stateful: false
description: ""

Back in business with the migration now. Most things going on standard LXD Linux images, or a Windows Server one that are UEFI enabled, but just needed this Citrix BIOS boot one out of the way because it’s very important. Phew!

Great! Although you shouldn’t need to do the bind mount, that path should already be able to access the host filesystem.

Oh I see… no I didn’t need to bind mount. Thanks for pointing that out.

Have a good evening what’s left of it :slight_smile:

1 Like