Rootfs is "full"

Hi,

I have two containers where there’s a mismatch between what df -h shows as used and what du -h / shows. Both containers have more or less the same configuration, as they are based on the same “base image”. One difference though is that they use different storage backends, one with LVM and the second one uses ceph RBD.
It’s something I’ve experienced before with other containers, and restarting them will free up the space that df claims as used.

The host is running LXD 4.10.

$ lxc config show compute-small-1

architecture: x86_64
config:
  image.build_ver: 10f16957b38b3df1cb525889296835c4868c3d4661a7fcd1040d78da1c364379
  image.name: base_image
  image.os: ubuntu
  image.version: "20.04"
  security.nesting: "true"
  volatile.base_image: 895aaf899f8e4a4298368f310c9ab64186b06f75bc6a29233282a392dfb555d8
  volatile.eth0.host_name: veth27e5afbf
  volatile.eth0.hwaddr: 00:16:3e:55:7e:47
  volatile.eth1.host_name: vethd873fd69
  volatile.eth1.hwaddr: 00:16:3e:1f:ad:4b
  volatile.idmap.base: "0"
  volatile.idmap.current: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":65536},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":65536}]'
  volatile.idmap.next: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":65536},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":65536}]'
  volatile.last_state.idmap: '[]'
  volatile.last_state.power: RUNNING
  volatile.uuid: da2fc426-4a9b-4b0a-b3a3-093640e4017e
devices: {}
ephemeral: false
profiles:
- ceph
- vlan
stateful: false
description: ""
root@compute-small-1:~# df -h
Filesystem                                                             Size  Used Avail Use% Mounted on
/var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs   12G   12G     0 100% /
none                                                                   492K  4.0K  488K   1% /dev
udev                                                                    63G     0   63G   0% /dev/tty
tmpfs                                                                  100K     0  100K   0% /dev/lxd
tmpfs                                                                  100K     0  100K   0% /dev/.lxd-mounts
tmpfs                                                                   63G     0   63G   0% /dev/shm
tmpfs                                                                   13G  1.8M   13G   1% /run
tmpfs                                                                  5.0M     0  5.0M   0% /run/lock
tmpfs                                                                   63G     0   63G   0% /sys/fs/cgroup
root@compute-small-1:~# du -hs /
du: cannot read directory '/proc/sys/fs/binfmt_misc': Permission denied
du: cannot read directory '/proc/tty/driver': Permission denied
du: cannot access '/proc/62828/task/62828/fd/4': No such file or directory
du: cannot access '/proc/62828/task/62828/fdinfo/4': No such file or directory
du: cannot access '/proc/62828/fd/3': No such file or directory
du: cannot access '/proc/62828/fdinfo/3': No such file or directory
du: cannot read directory '/dev/.lxd-mounts': Permission denied
du: cannot read directory '/dev/.lxc/sys/kernel': Permission denied
du: cannot read directory '/dev/.lxc/sys/power': Permission denied
du: cannot read directory '/dev/.lxc/sys/class': Permission denied
du: cannot read directory '/dev/.lxc/sys/devices': Permission denied
du: cannot read directory '/dev/.lxc/sys/dev': Permission denied
du: cannot read directory '/dev/.lxc/sys/hypervisor': Permission denied
du: cannot read directory '/dev/.lxc/sys/fs': Permission denied
du: cannot read directory '/dev/.lxc/sys/bus': Permission denied
du: cannot read directory '/dev/.lxc/sys/firmware': Permission denied
du: cannot read directory '/dev/.lxc/sys/block': Permission denied
du: cannot read directory '/dev/.lxc/sys/module': Permission denied
du: cannot read directory '/dev/.lxc/proc/fs': Permission denied
du: cannot read directory '/dev/.lxc/proc/bus': Permission denied
du: cannot read directory '/dev/.lxc/proc/irq': Permission denied
du: cannot read directory '/dev/.lxc/proc/spl': Permission denied
du: cannot read directory '/dev/.lxc/proc/sys': Permission denied
du: cannot read directory '/dev/.lxc/proc/tty': Permission denied
du: cannot read directory '/dev/.lxc/proc/acpi': Permission denied
du: cannot read directory '/dev/.lxc/proc/scsi': Permission denied
du: cannot read directory '/dev/.lxc/proc/driver': Permission denied
du: cannot read directory '/dev/.lxc/proc/sysvipc': Permission denied
du: cannot read directory '/dev/.lxc/proc/pressure': Permission denied
du: cannot read directory '/dev/.lxc/proc/1': Permission denied
du: cannot read directory '/dev/.lxc/proc/208': Permission denied
du: cannot read directory '/dev/.lxc/proc/210': Permission denied
du: cannot read directory '/dev/.lxc/proc/213': Permission denied
du: cannot read directory '/dev/.lxc/proc/214': Permission denied
du: cannot read directory '/dev/.lxc/proc/219': Permission denied
du: cannot read directory '/dev/.lxc/proc/221': Permission denied
du: cannot read directory '/dev/.lxc/proc/230': Permission denied
du: cannot read directory '/dev/.lxc/proc/236': Permission denied
du: cannot read directory '/dev/.lxc/proc/269': Permission denied
du: cannot read directory '/dev/.lxc/proc/62820': Permission denied
du: cannot read directory '/dev/.lxc/proc/62828': Permission denied
du: cannot read directory '/dev/.lxc/proc/2089092': Permission denied
du: cannot read directory '/dev/.lxc/proc/3572609': Permission denied
du: cannot read directory '/dev/.lxc/proc/3573152': Permission denied
du: cannot read directory '/dev/.lxc/proc/3573159': Permission denied
du: cannot read directory '/sys/kernel/tracing': Permission denied
du: cannot read directory '/sys/kernel/slab': Permission denied
du: cannot read directory '/sys/kernel/debug': Permission denied
du: cannot read directory '/sys/kernel/config': Permission denied
du: cannot read directory '/sys/fs/pstore': Permission denied
du: cannot read directory '/sys/fs/fuse/connections/719': Permission denied
du: cannot read directory '/sys/fs/fuse/connections/147': Permission denied
du: cannot read directory '/sys/fs/fuse/connections/52': Permission denied
3.4G    /

At first I thought it might be that had open file handles to deleted files, but I’ve tried stopping everything but the essentials.

root@compute-small-1:~# ps aux
USER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root           1  0.0  0.0 170580 12760 ?        Ss    2021   2:12 /lib/systemd/systemd --system --deserialize 22
root         208  0.0  0.0   8536  2936 ?        Ss    2021   0:17 /usr/sbin/cron -f
message+     210  0.0  0.0   7804  4960 ?        Ss    2021   0:10 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only
root         213  0.0  0.0  29668 15556 ?        Ss    2021   0:03 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers
syslog       214  0.0  0.0 154864  4344 ?        Ssl   2021   0:05 /usr/sbin/rsyslogd -n -iNONE
root         219  0.0  0.0  16804  6548 ?        Ss    2021   0:17 /lib/systemd/systemd-logind
daemon       221  0.0  0.0   3792  2200 ?        Ss    2021   0:00 /usr/sbin/atd -f
root         230  0.0  0.0   7352  2176 console  Ss+   2021   0:00 /sbin/agetty -o -p -- \u --noclear --keep-baud console 115200,38400,9600 linux
root         236  0.0  0.0  12176  5660 ?        Ss    2021   0:00 sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
root         269  0.0  0.0 236440  3760 ?        Ssl   2021   0:00 /usr/lib/policykit-1/polkitd --no-debug
root       62820  0.0  0.0   8960  3960 ?        Ss   14:54   0:00 bash
root       62830  0.0  0.0  10616  3236 ?        R+   14:58   0:00 ps aux
root     2089092  0.0  0.0 241052  6428 ?        Ssl   2021   2:17 /usr/lib/accountsservice/accounts-daemon
root     3572609  0.0  0.0  21608  4648 ?        Ss   Jan13   0:03 /lib/systemd/systemd-udevd
systemd+ 3573152  0.0  0.0  27404  8312 ?        Ss   Jan13   1:33 /lib/systemd/systemd-networkd
root     3573159  0.0  0.0  51464 19572 ?        Ss   Jan13   0:19 /lib/systemd/systemd-journald

It seems like it’s only systemd that has open handles to deleted files

root@compute-small-1:~# grep -r '(deleted)$' /proc/[0-9]*/maps
/proc/2089092/maps:7f8f4bcb8000-7f8f4bcc8000 r--p 00000000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/2089092/maps:7f8f4bcc8000-7f8f4bd3d000 r-xp 00010000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/2089092/maps:7f8f4bd3d000-7f8f4bd62000 r--p 00085000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/2089092/maps:7f8f4bd62000-7f8f4bd65000 r--p 000a9000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/2089092/maps:7f8f4bd65000-7f8f4bd66000 rw-p 000ac000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/210/maps:7f9b39fc6000-7f9b39fcc000 r--p 00000000 fc:00 6320                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libnss_systemd.so.2 (deleted)
/proc/210/maps:7f9b39fcc000-7f9b39ff2000 r-xp 00006000 fc:00 6320                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libnss_systemd.so.2 (deleted)
/proc/210/maps:7f9b39ff2000-7f9b39ffe000 r--p 0002c000 fc:00 6320                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libnss_systemd.so.2 (deleted)
/proc/210/maps:7f9b39ffe000-7f9b3a001000 r--p 00037000 fc:00 6320                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libnss_systemd.so.2 (deleted)
/proc/210/maps:7f9b3a001000-7f9b3a002000 rw-p 0003a000 fc:00 6320                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libnss_systemd.so.2 (deleted)
/proc/210/maps:7f9b3a502000-7f9b3a512000 r--p 00000000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/210/maps:7f9b3a512000-7f9b3a587000 r-xp 00010000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/210/maps:7f9b3a587000-7f9b3a5ac000 r--p 00085000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/210/maps:7f9b3a5ac000-7f9b3a5af000 r--p 000a9000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/210/maps:7f9b3a5af000-7f9b3a5b0000 rw-p 000ac000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/213/maps:00400000-00423000 r--p 00000000 fc:00 393071                             /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/bin/python3.8 (deleted)
/proc/213/maps:00423000-006b8000 r-xp 00023000 fc:00 393071                             /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/bin/python3.8 (deleted)
/proc/213/maps:006b8000-008f5000 r--p 002b8000 fc:00 393071                             /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/bin/python3.8 (deleted)
/proc/213/maps:008f5000-008f6000 r--p 004f4000 fc:00 393071                             /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/bin/python3.8 (deleted)
/proc/213/maps:008f6000-0093d000 rw-p 004f5000 fc:00 393071                             /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/bin/python3.8 (deleted)
/proc/213/maps:7f52a5acc000-7f52a5acf000 r--p 00000000 fc:00 4563                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_lzma.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a5acf000-7f52a5ad3000 r-xp 00003000 fc:00 4563                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_lzma.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a5ad3000-7f52a5ad6000 r--p 00007000 fc:00 4563                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_lzma.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a5ad6000-7f52a5ad7000 r--p 00009000 fc:00 4563                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_lzma.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a5ad7000-7f52a5ad8000 rw-p 0000a000 fc:00 4563                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_lzma.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a5aeb000-7f52a5aed000 r--p 00000000 fc:00 4544                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_bz2.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a5aed000-7f52a5aef000 r-xp 00002000 fc:00 4544                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_bz2.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a5aef000-7f52a5af1000 r--p 00004000 fc:00 4544                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_bz2.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a5af1000-7f52a5af2000 r--p 00005000 fc:00 4544                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_bz2.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a5af2000-7f52a5af3000 rw-p 00006000 fc:00 4544                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_bz2.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a5cd0000-7f52a5ce0000 r--p 00000000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/213/maps:7f52a5ce0000-7f52a5d55000 r-xp 00010000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/213/maps:7f52a5d55000-7f52a5d7a000 r--p 00085000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/213/maps:7f52a5d7a000-7f52a5d7d000 r--p 000a9000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/213/maps:7f52a5d7d000-7f52a5d7e000 rw-p 000ac000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/213/maps:7f52a679c000-7f52a679e000 r--p 00000000 fc:00 4561                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_json.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a679e000-7f52a67ac000 r-xp 00002000 fc:00 4561                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_json.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a67ac000-7f52a67ae000 r--p 00010000 fc:00 4561                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_json.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a67ae000-7f52a67af000 r--p 00011000 fc:00 4561                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_json.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/213/maps:7f52a67af000-7f52a67b0000 rw-p 00012000 fc:00 4561                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/python3.8/lib-dynload/_json.cpython-38-x86_64-linux-gnu.so (deleted)
/proc/214/maps:7f2de0a46000-7f2de0a4c000 r--p 00000000 fc:00 6320                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libnss_systemd.so.2 (deleted)
/proc/214/maps:7f2de0a4c000-7f2de0a72000 r-xp 00006000 fc:00 6320                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libnss_systemd.so.2 (deleted)
/proc/214/maps:7f2de0a72000-7f2de0a7e000 r--p 0002c000 fc:00 6320                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libnss_systemd.so.2 (deleted)
/proc/214/maps:7f2de0a7e000-7f2de0a81000 r--p 00037000 fc:00 6320                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libnss_systemd.so.2 (deleted)
/proc/214/maps:7f2de0a81000-7f2de0a82000 rw-p 0003a000 fc:00 6320                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libnss_systemd.so.2 (deleted)
/proc/214/maps:7f2de0e6d000-7f2de0e7d000 r--p 00000000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/214/maps:7f2de0e7d000-7f2de0ef2000 r-xp 00010000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/214/maps:7f2de0ef2000-7f2de0f17000 r--p 00085000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/214/maps:7f2de0f17000-7f2de0f1a000 r--p 000a9000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/214/maps:7f2de0f1a000-7f2de0f1b000 rw-p 000ac000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/219/maps:55fda4f18000-55fda4f23000 r--p 00000000 fc:00 396866                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/systemd-logind (deleted)
/proc/219/maps:55fda4f23000-55fda4f46000 r-xp 0000b000 fc:00 396866                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/systemd-logind (deleted)
/proc/219/maps:55fda4f46000-55fda4f55000 r--p 0002e000 fc:00 396866                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/systemd-logind (deleted)
/proc/219/maps:55fda4f56000-55fda4f5a000 r--p 0003d000 fc:00 396866                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/systemd-logind (deleted)
/proc/219/maps:55fda4f5a000-55fda4f5b000 rw-p 00041000 fc:00 396866                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/systemd-logind (deleted)
/proc/219/maps:7f5167d08000-7f5167d0d000 r--p 00000000 fc:00 6457                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libudev.so.1.6.17 (deleted)
/proc/219/maps:7f5167d0d000-7f5167d29000 r-xp 00005000 fc:00 6457                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libudev.so.1.6.17 (deleted)
/proc/219/maps:7f5167d29000-7f5167d33000 r--p 00021000 fc:00 6457                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libudev.so.1.6.17 (deleted)
/proc/219/maps:7f5167d33000-7f5167d34000 r--p 0002a000 fc:00 6457                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libudev.so.1.6.17 (deleted)
/proc/219/maps:7f5167d34000-7f5167d35000 rw-p 0002b000 fc:00 6457                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libudev.so.1.6.17 (deleted)
/proc/219/maps:7f516837f000-7f51683a7000 r--p 00000000 fc:00 6399                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libseccomp.so.2.4.3 (deleted)
/proc/219/maps:7f51683a7000-7f51683b2000 r-xp 00028000 fc:00 6399                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libseccomp.so.2.4.3 (deleted)
/proc/219/maps:7f51683b2000-7f51683b6000 r--p 00033000 fc:00 6399                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libseccomp.so.2.4.3 (deleted)
/proc/219/maps:7f51683b6000-7f51683d1000 r--p 00036000 fc:00 6399                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libseccomp.so.2.4.3 (deleted)
/proc/219/maps:7f51683d1000-7f51683d2000 rw-p 00051000 fc:00 6399                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libseccomp.so.2.4.3 (deleted)
/proc/219/maps:7f51686ee000-7f5168734000 r--p 00000000 fc:00 396507                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/libsystemd-shared-245.so (deleted)
/proc/219/maps:7f5168734000-7f51688a8000 r-xp 00046000 fc:00 396507                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/libsystemd-shared-245.so (deleted)
/proc/219/maps:7f51688a8000-7f5168935000 r--p 001ba000 fc:00 396507                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/libsystemd-shared-245.so (deleted)
/proc/219/maps:7f5168935000-7f5168936000 ---p 00247000 fc:00 396507                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/libsystemd-shared-245.so (deleted)
/proc/219/maps:7f5168936000-7f5168945000 r--p 00247000 fc:00 396507                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/libsystemd-shared-245.so (deleted)
/proc/219/maps:7f5168945000-7f5168946000 rw-p 00256000 fc:00 396507                     /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/systemd/libsystemd-shared-245.so (deleted)
/proc/236/maps:7fef28a2e000-7fef28a3e000 r--p 00000000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/236/maps:7fef28a3e000-7fef28ab3000 r-xp 00010000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/236/maps:7fef28ab3000-7fef28ad8000 r--p 00085000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/236/maps:7fef28ad8000-7fef28adb000 r--p 000a9000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/236/maps:7fef28adb000-7fef28adc000 rw-p 000ac000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/269/maps:7fa152cf2000-7fa152d02000 r--p 00000000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/269/maps:7fa152d02000-7fa152d77000 r-xp 00010000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/269/maps:7fa152d77000-7fa152d9c000 r--p 00085000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/269/maps:7fa152d9c000-7fa152d9f000 r--p 000a9000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)
/proc/269/maps:7fa152d9f000-7fa152da0000 rw-p 000ac000 fc:00 6429                       /var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs/usr/lib/x86_64-linux-gnu/libsystemd.so.0.28.0 (deleted)

Interestingly enough df -i shows 7% inode usage

root@compute-small-1:~# df -i
Filesystem                                                              Inodes IUsed    IFree IUse% Mounted on
/var/lib/lxd/storage-pools/cephpool/containers/compute-small-1/rootfs   765536 47825   717711    7% /
none                                                                  16480751    28 16480723    1% /dev
udev                                                                  16464820   626 16464194    1% /dev/tty
tmpfs                                                                 16480751     2 16480749    1% /dev/lxd
tmpfs                                                                 16480751    16 16480735    1% /dev/.lxd-mounts
tmpfs                                                                 16480751     1 16480750    1% /dev/shm
tmpfs                                                                 16480751   512 16480239    1% /run
tmpfs                                                                 16480751     3 16480748    1% /run/lock
tmpfs                                                                 16480751    18 16480733    1% /sys/fs/cgroup