Lxd 4.3: VMs no longer start

1

After an update to 4.3 VMs no longer start, here’s the log.

Error: Failed to run: /snap/lxd/current/bin/lxd forklimits limit=memlock:unlimited:unlimited -- /snap/lxd/16044/bin/qemu-system-x86_64 -S -name pod-890-1000-00 -uuid <uuid> -daemonize -cpu host -nographic -serial chardev:console -nodefaults -no-reboot -no-user-config -sandbox on,obsolete=deny,elevateprivileges=allow,spawn=deny,resourcecontrol=deny -readconfig /var/snap/lxd/common/lxd/logs/<company>-staging2_pod-890-1000-00/qemu.conf -pidfile /var/snap/lxd/common/lxd/logs/<company>-staging2_pod-`Preformatted text`890-1000-00/qemu.pid -D /var/snap/lxd/common/lxd/logs/<company>-staging2_pod-890-1000-00/qemu.log -chroot /var/snap/lxd/common/lxd/virtual-machines/<company>-staging2_pod-890-1000-00 -smbios type=2,manufacturer=Canonical Ltd.,product=LXD -runas lxd: : exit status 1
Try `lxc info --show-log pod-890-1000-00` for more info
ubuntu@aa1-cptef101-n1:~$ lxc info --show-log --project <company>-staging2 pod-890-1000-00
Name: pod-890-1000-00
Location: aa1-cptef101-n1
Remote: unix://
Architecture: x86_64
Created: 2020/06/18 23:54 UTC
Status: Stopped
Type: virtual-machine
Profiles: stagingtwo-virtual-vault-890
Snapshots:
  snap0 (taken at 2020/06/19 01:15 UTC) (stateless)
  pre-common-yml (taken at 2020/06/30 18:42 UTC) (stateless)
  phase3 (taken at 2020/06/30 23:34 UTC) (stateless)

Log:

qemu-system-x86_64:/var/snap/lxd/common/lxd/logs/<company>-staging2_pod-890-1000-00/qemu.conf:226: Parameter 'locking' is unexpected
2

Can you show the qemu.conf for that VM?

3

And lxc config show --expanded NAME for the VM.

I know what the problem is but need to track down exactly what code path needs to skip the file.locking feature.

4

QEMU Config:
# Machine
[machine]
graphics = “off”
type = “q35”
accel = “kvm”
usb = “off”
graphics = “off”

[global]
driver = "ICH9-LPC"
property = "disable_s3"
value = "1"

[global]
driver = "ICH9-LPC"
property = "disable_s4"
value = "1"
[boot-opts]
strict = "on"

# Console
[chardev "console"]
backend = "pty"

# Graphical console
[spice]
unix = "on"
addr = "/var/snap/lxd/common/lxd/logs/<company>-staging2_pod-890-1000-00/qemu.spice"
disable-ticketing = "on"

# Memory
[memory]
size = "15258M"

# CPU
[smp-opts]
cpus = "8"
sockets = "1"
cores = "8"
threads = "1"








# Firmware (read only)
[drive]
file = "/snap/lxd/current/share/qemu/OVMF_CODE.fd"
if = "pflash"
format = "raw"
unit = "0"
readonly = "on"

# Firmware settings (writable)
[drive]
file = "/var/snap/lxd/common/lxd/virtual-machines/<company>-staging2_pod-890-1000-00/qemu.nvram"
if = "pflash"
format = "raw"
unit = "1"

# Qemu control
[chardev "monitor"]
backend = "socket"
path = "/var/snap/lxd/common/lxd/logs/<company>-staging2_pod-890-1000-00/qemu.monitor"
server = "on"
wait = "off"

[mon]
chardev = "monitor"
mode = "control"

[device "qemu_pcie0"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.0"
chassis = "0"
multifunction = "on"

# Balloon driver
[device "qemu_balloon"]
driver = "virtio-balloon-pci"
bus = "qemu_pcie0"
addr = "00.0"

multifunction = "on"

# Random number generator
[object "qemu_rng"]
qom-type = "rng-random"
filename = "/dev/urandom"

[device "dev-qemu_rng"]
driver = "virtio-rng-pci"
bus = "qemu_pcie0"
addr = "00.1"

rng = "qemu_rng"


# Input
[device "qemu_keyboard"]
driver = "virtio-keyboard-pci"
bus = "qemu_pcie0"
addr = "00.2"



# Input
[device "qemu_tablet"]
driver = "virtio-tablet-pci"
bus = "qemu_pcie0"
addr = "00.3"



# Vsock
[device "qemu_vsock"]
driver = "vhost-vsock-pci"
bus = "qemu_pcie0"
addr = "00.4"

guest-cid = "71"


# LXD serial identifier
[device "dev-qemu_serial"]
driver = "virtio-serial-pci"
bus = "qemu_pcie0"
addr = "00.5"



[chardev "qemu_serial-chardev"]
backend = "ringbuf"
size = "16B"

[device "qemu_serial"]
driver = "virtserialport"
name = "org.linuxcontainers.lxd"
chardev = "qemu_serial-chardev"
bus = "dev-qemu_serial.0"

[device "qemu_pcie1"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.1"
chassis = "1"


# SCSI controller
[device "qemu_scsi"]
driver = "virtio-scsi-pci"
bus = "qemu_pcie1"
addr = "00.0"



[device "qemu_pcie2"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.2"
chassis = "2"


# Config drive
[fsdev "qemu_config"]
fsdriver = "local"
security_model = "none"
readonly = "on"
path = "/var/snap/lxd/common/lxd/virtual-machines/<company>-staging2_pod-890-1000-00/config"

[device "dev-qemu_config"]
driver = "virtio-9p-pci"
bus = "qemu_pcie2"
addr = "00.0"

mount_tag = "config"
fsdev = "qemu_config"
multifunction = "on"

[device "qemu_pcie3"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.3"
chassis = "3"


# GPU
[device "qemu_gpu"]
driver = "virtio-vga"
bus = "qemu_pcie3"
addr = "00.0"



[device "qemu_pcie4"]
driver = "pcie-root-port"
bus = "pcie.0"
addr = "1.4"
chassis = "4"


# Network card ("eth0" device)
[netdev "lxd_eth0"]
type = "tap"
vhost = "on"
ifname = "tap6a5eb9bd"
script = "no"
downscript = "no"

[device "dev-lxd_eth0"]
driver = "virtio-net-pci"
bus = "qemu_pcie4"
addr = "00.0"

netdev = "lxd_eth0"
mac = "00:16:3e:4d:81:a2"
bootindex = "1"


# audit drive
[drive "lxd_audit"]
file = "rbd:lxd-virtualvaultfs-01/disk_pod-890-1000-00_audit:id=lxd:conf=/etc/ceph/ceph.conf"
format = "raw"
if = "none"
cache = "none"
aio = "native"
discard = "on"
file.locking = "off"

[device "dev-lxd_audit"]
driver = "scsi-hd"
bus = "qemu_scsi.0"
channel = "0"
scsi-id = "4"
lun = "1"
drive = "lxd_audit"
bootindex = "4"


# disk_a drive
[drive "lxd_disk_a"]
file = "rbd:lxd-virtualvaultfs-01/disk_pod-890-1000-00_a:id=lxd:conf=/etc/ceph/ceph.conf"
format = "raw"
if = "none"
cache = "none"
aio = "native"
discard = "on"
file.locking = "off"

[device "dev-lxd_disk_a"]
driver = "scsi-hd"
bus = "qemu_scsi.0"
channel = "0"
scsi-id = "5"
lun = "1"
drive = "lxd_disk_a"
bootindex = "5"


# disk_b drive
[drive "lxd_disk_b"]
file = "rbd:lxd-virtualvaultfs-01/disk_pod-890-1000-00_b:id=lxd:conf=/etc/ceph/ceph.conf"
format = "raw"
if = "none"
cache = "none"
aio = "native"
discard = "on"
file.locking = "off"

[device "dev-lxd_disk_b"]
driver = "scsi-hd"
bus = "qemu_scsi.0"
channel = "0"
scsi-id = "6"
lun = "1"
drive = "lxd_disk_b"
bootindex = "6"


# disk_c drive
[drive "lxd_disk_c"]
file = "rbd:lxd-virtualvaultfs-01/disk_pod-890-1000-00_c:id=lxd:conf=/etc/ceph/ceph.conf"
format = "raw"
if = "none"
cache = "none"
aio = "native"
discard = "on"
file.locking = "off"

[device "dev-lxd_disk_c"]
driver = "scsi-hd"
bus = "qemu_scsi.0"
channel = "0"
scsi-id = "2"
lun = "1"
drive = "lxd_disk_c"
bootindex = "2"


# disk_d drive
[drive "lxd_disk_d"]
file = "rbd:lxd-virtualvaultfs-01/disk_pod-890-1000-00_d:id=lxd:conf=/etc/ceph/ceph.conf"
format = "raw"
if = "none"
cache = "none"
aio = "native"
discard = "on"
file.locking = "off"

[device "dev-lxd_disk_d"]
driver = "scsi-hd"
bus = "qemu_scsi.0"
channel = "0"
scsi-id = "3"
lun = "1"
drive = "lxd_disk_d"
bootindex = "3"


# root drive
[drive "lxd_root"]
file = "/dev/rbd8"
format = "raw"
if = "none"
cache = "none"
aio = "native"
discard = "on"
file.locking = "off"

[device "dev-lxd_root"]
driver = "scsi-hd"
bus = "qemu_scsi.0"
channel = "0"
scsi-id = "0"
lun = "1"
drive = "lxd_root"
bootindex = "0"

Config:
architecture: x86_64
config:
image.architecture: amd64
image.description: Debian stretch amd64 (20200618_16:58)
image.os: Debian
image.release: stretch
image.serial: “20200618_16:58”
image.type: disk-kvm.img
limits.cpu: “8”
limits.memory: 16GB
security.secureboot: “false”
volatile.base_image: 214c4284fea0055f1563e75a54dc349259428d43ea5e8502a78092b63d3852ec
volatile.eth0.hwaddr: 00:16:3e:4d:81:a2
volatile.last_state.power: RUNNING
volatile.vm.uuid:
devices:
audit:
ceph.cluster_name: ceph
ceph.user_name: lxd
source: ceph:lxd-virtualvaultfs-01/disk_pod-890-1000-00_audit
type: disk
disk_a:
ceph.cluster_name: ceph
ceph.user_name: lxd
source: ceph:lxd-virtualvaultfs-01/disk_pod-890-1000-00_a
type: disk
disk_b:
ceph.cluster_name: ceph
ceph.user_name: lxd
source: ceph:lxd-virtualvaultfs-01/disk_pod-890-1000-00_b
type: disk
disk_c:
ceph.cluster_name: ceph
ceph.user_name: lxd
source: ceph:lxd-virtualvaultfs-01/disk_pod-890-1000-00_c
type: disk
disk_d:
ceph.cluster_name: ceph
ceph.user_name: lxd
source: ceph:lxd-virtualvaultfs-01/disk_pod-890-1000-00_d
type: disk
eth0:
name: eth0
nictype: bridged
parent: br4
type: nic
root:
path: /
pool: ceph
size: 1500GB
type: disk
ephemeral: false
profiles:
- stagingtwo-virtual-vault-890
stateful: false
description: “”

5

https://github.com/lxc/lxd/pull/7634 will take care of that issue.

6

That is merged now.

7

How long until this hits stable in snap?

8

I just pushed the cherry-pick now, will hit candidate in the next hour and should be in stable in the next 2-3 hours.