Ok, so maybe I’m pushing it, I don’t know . But in my current set up I have a Vagrant spawned VM, configured as a LXD/LXC hypervisor (on Ubuntu 20.04), in which I have an Ubuntu 20.04 container, also with LXD installed, and now I want to run Debian 10 container IN that container. Not sure if it matters but all containers are configured to use “routed” nictype. This nested hypervisor itself works just fine, but I cannot start Debian 10 container in it. Error I’m getting is:
I’m using the “dir” filesystem all the way up.
root@hypervisor-nested:~# lxc start test-profile-1
Error: Failed to run: /snap/lxd/current/bin/lxd forkstart test-profile-1 /var/snap/lxd/common/lxd/containers /var/snap/lxd/common/lxd/logs/test-profile-1/lxc.conf:
Try `lxc info --show-log test-profile-1` for more info
luken@lxd-hypervisor:~$ lxc exec hypervisor-nested -- bash
root@hypervisor-nested:~# lxc start test-profile-1
Error: Failed to run: /snap/lxd/current/bin/lxd forkstart test-profile-1 /var/snap/lxd/common/lxd/containers /var/snap/lxd/common/lxd/logs/test-profile-1/lxc.conf:
Try `lxc info --show-log test-profile-1` for more info
root@hypervisor-nested:~# lxc info --show-log test-profile-1
Name: test-profile-1
Location: none
Remote: unix://
Architecture: x86_64
Created: 2021/02/24 21:32 UTC
Status: Stopped
Type: container
Profiles: default, test-profile-1
Log:
lxc test-profile-1 20210224213608.238 WARN cgfsng - cgroups/cgfsng.c:mkdir_eexist_on_last:1126 - File exists - Failed to create directory "/sys/fs/cgroup/cpuset//lxc.monitor.test-profile-1"
lxc test-profile-1 20210224213608.245 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_create:1142 - File exists - The /sys/fs/cgroup/unified//lxc.payload.test-profile-1 cgroup already existed
lxc test-profile-1 20210224213608.245 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_create:1142 - File exists - The /sys/fs/cgroup/unified//lxc.payload.test-profile-1-1 cgroup already existed
lxc test-profile-1 20210224213608.245 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_create:1142 - File exists - The /sys/fs/cgroup/unified//lxc.payload.test-profile-1-2 cgroup already existed
lxc test-profile-1 20210224213608.246 WARN cgfsng - cgroups/cgfsng.c:mkdir_eexist_on_last:1126 - File exists - Failed to create directory "/sys/fs/cgroup/cpuset//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.262 WARN cgfsng - cgroups/cgfsng.c:fchowmodat:1547 - No such file or directory - Failed to fchownat(17, memory.oom.group, 999900000, 0, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW )
lxc test-profile-1 20210224213608.269 ERROR utils - utils.c:__safe_mount_beneath_at:1106 - Function not implemented - Failed to open 30(dev)
lxc test-profile-1 20210224213608.270 ERROR utils - utils.c:safe_mount:1204 - Permission denied - Failed to mount "proc" onto "/var/snap/lxd/common/lxc//proc"
lxc test-profile-1 20210224213608.270 ERROR conf - conf.c:lxc_mount_auto_mounts:697 - Permission denied - Failed to mount "proc" on "/var/snap/lxd/common/lxc//proc" with flags 14
lxc test-profile-1 20210224213608.270 ERROR conf - conf.c:lxc_setup:3346 - Failed to setup first automatic mounts
lxc test-profile-1 20210224213608.270 ERROR start - start.c:do_start:1218 - Failed to setup container "test-profile-1"
lxc test-profile-1 20210224213608.270 ERROR sync - sync.c:__sync_wait:36 - An error occurred in another process (expected sequence number 5)
lxc test-profile-1 20210224213608.270 ERROR lxccontainer - lxccontainer.c:wait_on_daemonized_start:860 - Received container state "ABORTING" instead of "RUNNING"
lxc test-profile-1 20210224213608.271 ERROR start - start.c:__lxc_start:1999 - Failed to spawn container "test-profile-1"
lxc test-profile-1 20210224213608.271 WARN start - start.c:lxc_abort:1013 - No such process - Failed to send SIGKILL via pidfd 30 for process 8672
lxc test-profile-1 20210224213608.331 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/unified//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.331 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/unified//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.331 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/systemd//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.331 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/systemd//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.331 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/freezer//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.331 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/freezer//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.331 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/hugetlb//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.331 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/hugetlb//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.331 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/net_cls,net_prio//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.331 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/net_cls,net_prio//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/blkio//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/blkio//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/cpuset//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/cpuset//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/pids//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/pids//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/rdma//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/rdma//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/cpu,cpuacct//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/cpu,cpuacct//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/perf_event//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/perf_event//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/devices//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/devices//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN utils - utils.c:lxc_rm_rf:1843 - Permission denied - Failed to delete "/sys/fs/cgroup/memory//lxc.payload.test-profile-1-3"
lxc test-profile-1 20210224213608.333 WARN cgfsng - cgroups/cgfsng.c:cgroup_tree_remove:939 - Failed to destroy "/sys/fs/cgroup/memory//lxc.payload.test-profile-1-3"
lxc 20210224213608.334 WARN commands - commands.c:lxc_cmd_rsp_recv:126 - Connection reset by peer - Failed to receive response for command "get_state"
root@hypervisor-nested:~# cat /var/snap/lxd/common/lxd/logs/test-profile-1/lxc.conf
lxc.log.file = /var/snap/lxd/common/lxd/logs/test-profile-1/lxc.log
lxc.log.level = warn
lxc.console.buffer.size = auto
lxc.console.size = auto
lxc.console.logfile = /var/snap/lxd/common/lxd/logs/test-profile-1/console.log
lxc.mount.auto = proc:rw sys:rw cgroup:mixed
lxc.autodev = 1
lxc.pty.max = 1024
lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file,optional 0 0
lxc.mount.entry = /dev/net/tun dev/net/tun none bind,create=file,optional 0 0
lxc.mount.entry = /proc/sys/fs/binfmt_misc proc/sys/fs/binfmt_misc none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/fs/fuse/connections sys/fs/fuse/connections none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/fs/pstore sys/fs/pstore none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/kernel/config sys/kernel/config none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/kernel/debug sys/kernel/debug none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/kernel/security sys/kernel/security none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/kernel/tracing sys/kernel/tracing none rbind,create=dir,optional 0 0
lxc.mount.entry = /dev/mqueue dev/mqueue none rbind,create=dir,optional 0 0
lxc.include = /snap/lxd/current/lxc/config//common.conf.d/
lxc.arch = linux64
lxc.hook.version = 1
lxc.hook.pre-start = /proc/5093/exe callhook /var/snap/lxd/common/lxd "default" "test-profile-1" start
lxc.hook.stop = /snap/lxd/current/bin/lxd callhook /var/snap/lxd/common/lxd "default" "test-profile-1" stopns
lxc.hook.post-stop = /snap/lxd/current/bin/lxd callhook /var/snap/lxd/common/lxd "default" "test-profile-1" stop
lxc.tty.max = 0
lxc.uts.name = test-profile-1
lxc.mount.entry = /var/snap/lxd/common/lxd/devlxd dev/lxd none bind,create=dir 0 0
lxc.apparmor.profile = lxd-test-profile-1_</var/snap/lxd/common/lxd>
lxc.seccomp.profile = /var/snap/lxd/common/lxd/security/seccomp/test-profile-1
lxc.idmap = u 0 100000 999900000
lxc.idmap = g 0 100000 999900000
lxc.mount.auto = shmounts:/var/snap/lxd/common/lxd/shmounts/test-profile-1:/dev/.lxd-mounts
lxc.net.0.name = eth0
lxc.net.0.type = veth
lxc.net.0.flags = up
lxc.net.0.veth.mode = router
lxc.net.0.veth.pair = veth513227bd
lxc.net.0.l2proxy = 1
lxc.net.0.link = eth0
lxc.net.0.ipv4.address = 192.168.7.240/32
lxc.net.0.ipv4.gateway = 169.254.0.1
lxc.rootfs.path = dir:/var/snap/lxd/common/lxd/containers/test-profile-1/rootfs
root@hypervisor-nested:~# lxc profile show test-profile-1
config: {}
description: 'Test profile #1'
devices:
eth0:
ipv4.address: 192.168.7.240
nictype: routed
parent: eth0
type: nic
name: test-profile-1
used_by:
- /1.0/instances/test-profile-1
Any idea what may be wrong here? Is there anything special that has to be done to run nested lxd/lxc hypervisor that I don’t know about?