Hello Everyone,
after a disk fill up, I have only one container (out of 15) that doesn’t start up. Please, can you help me fixing this situation?
Here the message I have when I try to start it up:
lxc start portainer -v
Error: Failed to run: /snap/lxd/current/bin/lxd forkstart portainer /var/snap/lxd/common/lxd/containers /var/snap/lxd/common/lxd/logs/portainer/lxc.conf:
Try lxc info --show-log portainer for more info
Checking the logs I found this error:
lxc info --show-log portainer
Name: portainer
Location: none
Remote: unix://
Architecture: x86_64
Created: 2021/05/06 07:44 UTC
Status: Stopped
Type: container
Profiles: default
Log:
lxc portainer 20210912204753.284 WARN conf - conf.c:lxc_map_ids:3389 - newuidmap binary is missing
lxc portainer 20210912204753.285 WARN conf - conf.c:lxc_map_ids:3395 - newgidmap binary is missing
lxc portainer 20210912204753.287 WARN conf - conf.c:lxc_map_ids:3389 - newuidmap binary is missing
lxc portainer 20210912204753.287 WARN conf - conf.c:lxc_map_ids:3395 - newgidmap binary is missing
lxc portainer 20210912204753.288 WARN cgfsng - cgroups/cgfsng.c:fchowmodat:1296 - No such file or directory - Failed to fchownat(43, memory.oom.group, 1000000000, 0, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW )
lxc portainer 20210912204753.292 WARN start - start.c:lxc_spawn:1818 - Bad file descriptor - Failed to allocate new network namespace id
lxc portainer 20210912204753.297 ERROR network - network.c:lxc_network_move_created_netdev_priv:3536 - No such process - Failed to move network device “vethc42ef790” with ifindex 55 to network namespace 150408 and rename to phys1mQml6
lxc portainer 20210912204753.297 ERROR start - start.c:lxc_spawn:1826 - Failed to create the network
lxc portainer 20210912204753.304 ERROR lxccontainer - lxccontainer.c:wait_on_daemonized_start:869 - Received container state “ABORTING” instead of “RUNNING”
lxc portainer 20210912204753.304 ERROR start - start.c:__lxc_start:2053 - Failed to spawn container “portainer”
lxc portainer 20210912204753.304 WARN start - start.c:lxc_abort:1051 - No such process - Failed to send SIGKILL via pidfd 44 for process 150408
lxc portainer 20210912204758.317 WARN conf - conf.c:lxc_map_ids:3389 - newuidmap binary is missing
lxc portainer 20210912204758.317 WARN conf - conf.c:lxc_map_ids:3395 - newgidmap binary is missing
lxc 20210912204758.323 ERROR af_unix - af_unix.c:lxc_abstract_unix_recv_fds_iov:220 - Connection reset by peer - Failed to receive response
lxc 20210912204758.323 ERROR commands - commands.c:lxc_cmd_rsp_recv_fds:129 - Failed to receive file descriptors
Here some info about the container config:
cat /var/snap/lxd/common/lxd/logs/portainer/lxc.conf
lxc.log.file = /var/snap/lxd/common/lxd/logs/portainer/lxc.log
lxc.log.level = warn
lxc.console.buffer.size = auto
lxc.console.size = auto
lxc.console.logfile = /var/snap/lxd/common/lxd/logs/portainer/console.log
lxc.mount.auto = proc:rw sys:rw cgroup:mixed
lxc.autodev = 1
lxc.pty.max = 1024
lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file,optional 0 0
lxc.mount.entry = /dev/net/tun dev/net/tun none bind,create=file,optional 0 0
lxc.mount.entry = /proc/sys/fs/binfmt_misc proc/sys/fs/binfmt_misc none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/firmware/efi/efivars sys/firmware/efi/efivars none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/fs/fuse/connections sys/fs/fuse/connections none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/fs/pstore sys/fs/pstore none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/kernel/config sys/kernel/config none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/kernel/debug sys/kernel/debug none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/kernel/security sys/kernel/security none rbind,create=dir,optional 0 0
lxc.mount.entry = /sys/kernel/tracing sys/kernel/tracing none rbind,create=dir,optional 0 0
lxc.mount.entry = /dev/mqueue dev/mqueue none rbind,create=dir,optional 0 0
lxc.include = /snap/lxd/current/lxc/config//common.conf.d/
lxc.mount.entry = proc dev/.lxc/proc proc create=dir,optional 0 0
lxc.mount.entry = sys dev/.lxc/sys sysfs create=dir,optional 0 0
lxc.arch = linux64
lxc.hook.version = 1
lxc.hook.pre-start = /proc/4643/exe callhook /var/snap/lxd/common/lxd “default” “portainer” start
lxc.hook.stop = /snap/lxd/current/bin/lxd callhook /var/snap/lxd/common/lxd “default” “portainer” stopns
lxc.hook.post-stop = /snap/lxd/current/bin/lxd callhook /var/snap/lxd/common/lxd “default” “portainer” stop
lxc.tty.max = 0
lxc.uts.name = portainer
lxc.mount.entry = /var/snap/lxd/common/lxd/devlxd dev/lxd none bind,create=dir 0 0
lxc.apparmor.profile = lxd-portainer_</var/snap/lxd/common/lxd>//&:lxd-portainer_:
lxc.seccomp.profile = /var/snap/lxd/common/lxd/security/seccomp/portainer
lxc.idmap = u 0 1000000 1002
lxc.idmap = u 1002 1002 1
lxc.idmap = u 1003 1001003 999998997
lxc.idmap = g 0 1000000 1001
lxc.idmap = g 1001 1001 1
lxc.idmap = g 1002 1001002 999998998
lxc.cgroup.memory.limit_in_bytes = 0
lxc.cgroup.memory.soft_limit_in_bytes = 0
lxc.mount.auto = shmounts:/var/snap/lxd/common/lxd/shmounts/portainer:/dev/.lxd-mounts
lxc.net.0.type = phys
lxc.net.0.name = eth0
lxc.net.0.flags = up
lxc.net.0.link = vethc42ef790
lxc.rootfs.path = dir:/var/snap/lxd/common/lxd/containers/portainer/rootfs
lxc.mount.entry = /var/snap/lxd/common/lxd/devices/portainer/disk.audiobookspath.opt-share-audiobooks opt/share/audiobooks none bind,create=dir 0 0
lxc.mount.entry = /var/snap/lxd/common/lxd/devices/portainer/disk.downloadpath.opt-share-download opt/share/download none bind,create=dir 0 0
lxc.mount.entry = /var/snap/lxd/common/lxd/devices/portainer/disk.ebookspath.opt-share-ebooks opt/share/ebooks none bind,create=dir 0 0
lxc.mount.entry = /var/snap/lxd/common/lxd/devices/portainer/disk.incompletepath.opt-share-incomplete opt/share/incomplete none bind,create=dir 0 0
lxc.mount.entry = /var/snap/lxd/common/lxd/devices/portainer/disk.moviespath.opt-share-movies opt/share/movies none bind,create=dir 0 0
lxc.mount.entry = /var/snap/lxd/common/lxd/devices/portainer/disk.musicpath.opt-share-music opt/share/music none bind,create=dir 0 0
lxc.mount.entry = /var/snap/lxd/common/lxd/devices/portainer/disk.toruploadspath.opt-share-torrents–upload opt/share/torrents-upload none bind,create=dir 0 0
lxc.mount.entry = /var/snap/lxd/common/lxd/devices/portainer/disk.tvseriespath.opt-share-tvseries opt/share/tvseries none bind,create=dir 0 0
lxc config show portainer -e
architecture: x86_64
config:
boot.autostart: “true”
image.architecture: amd64
image.description: ubuntu 20.04 LTS amd64 (release) (20210429)
image.label: release
image.os: ubuntu
image.release: focal
image.serial: “20210429”
image.type: squashfs
image.version: “20.04”
limits.memory: “0”
raw.idmap: |-
uid 1002 1002
gid 1001 1001
security.nesting: “true”
volatile.base_image: 90e2d54028ddfe65c31605bb9f0cece65752d05ff385989f3eda6b38d92fab5a
volatile.eth0.host_name: vethd948a3ce
volatile.eth0.hwaddr: 00:16:3e:c4:c3:2f
volatile.idmap.base: “0”
volatile.idmap.current: ‘[{“Isuid”:true,“Isgid”:false,“Hostid”:1000000,“Nsid”:0,“Maprange”:1002},{“Isuid”:true,“Isgid”:false,“Hostid”:1002,“Nsid”:1002,“Maprange”:1},{“Isuid”:true,“Isgid”:false,“Hostid”:1001003,“Nsid”:1003,“Maprange”:999998997},{“Isuid”:false,“Isgid”:true,“Hostid”:1000000,“Nsid”:0,“Maprange”:1001},{“Isuid”:false,“Isgid”:true,“Hostid”:1001,“Nsid”:1001,“Maprange”:1},{“Isuid”:false,“Isgid”:true,“Hostid”:1001002,“Nsid”:1002,“Maprange”:999998998}]’
volatile.idmap.next: ‘[{“Isuid”:true,“Isgid”:false,“Hostid”:1000000,“Nsid”:0,“Maprange”:1002},{“Isuid”:true,“Isgid”:false,“Hostid”:1002,“Nsid”:1002,“Maprange”:1},{“Isuid”:true,“Isgid”:false,“Hostid”:1001003,“Nsid”:1003,“Maprange”:999998997},{“Isuid”:false,“Isgid”:true,“Hostid”:1000000,“Nsid”:0,“Maprange”:1001},{“Isuid”:false,“Isgid”:true,“Hostid”:1001,“Nsid”:1001,“Maprange”:1},{“Isuid”:false,“Isgid”:true,“Hostid”:1001002,“Nsid”:1002,“Maprange”:999998998}]’
volatile.last_state.idmap: ‘[{“Isuid”:true,“Isgid”:false,“Hostid”:1000000,“Nsid”:0,“Maprange”:1002},{“Isuid”:true,“Isgid”:false,“Hostid”:1002,“Nsid”:1002,“Maprange”:1},{“Isuid”:true,“Isgid”:false,“Hostid”:1001003,“Nsid”:1003,“Maprange”:999998997},{“Isuid”:false,“Isgid”:true,“Hostid”:1000000,“Nsid”:0,“Maprange”:1001},{“Isuid”:false,“Isgid”:true,“Hostid”:1001,“Nsid”:1001,“Maprange”:1},{“Isuid”:false,“Isgid”:true,“Hostid”:1001002,“Nsid”:1002,“Maprange”:999998998}]’
volatile.last_state.power: STOPPED
volatile.uuid: 3bb11a2b-1c0d-45bb-9ff3-c2f3d32001b1
devices:
audiobookspath:
path: /opt/share/audiobooks
source: /tank/share/audiobooks
type: disk
downloadpath:
path: /opt/share/download
source: /tank/download
type: disk
ebookspath:
path: /opt/share/ebooks
source: /tank/share/ebooks
type: disk
eth0:
name: eth0
nictype: bridged
parent: br0
type: nic
incompletepath:
path: /opt/share/incomplete
source: /tank/share/incomplete
type: disk
moviespath:
path: /opt/share/movies
source: /tank/share/movies
type: disk
musicpath:
path: /opt/share/music
source: /tank/share/music
type: disk
root:
path: /
pool: tankssd
type: disk
toruploadspath:
path: /opt/share/torrents-upload
source: /tank/share/torrents-upload
type: disk
tvseriespath:
path: /opt/share/tvseries
source: /tank/share/tvseries
type: disk
ephemeral: false
profiles:
- default
stateful: false
description: “”
And here some info about the host:
lxc info
config:
core.https_address: 172.30.42.55:8443
core.trust_password: true
api_extensions:
- storage_zfs_remove_snapshots
- container_host_shutdown_timeout
- container_stop_priority
- container_syscall_filtering
- auth_pki
- container_last_used_at
- etag
- patch
- usb_devices
- https_allowed_credentials
- image_compression_algorithm
- directory_manipulation
- container_cpu_time
- storage_zfs_use_refquota
- storage_lvm_mount_options
- network
- profile_usedby
- container_push
- container_exec_recording
- certificate_update
- container_exec_signal_handling
- gpu_devices
- container_image_properties
- migration_progress
- id_map
- network_firewall_filtering
- network_routes
- storage
- file_delete
- file_append
- network_dhcp_expiry
- storage_lvm_vg_rename
- storage_lvm_thinpool_rename
- network_vlan
- image_create_aliases
- container_stateless_copy
- container_only_migration
- storage_zfs_clone_copy
- unix_device_rename
- storage_lvm_use_thinpool
- storage_rsync_bwlimit
- network_vxlan_interface
- storage_btrfs_mount_options
- entity_description
- image_force_refresh
- storage_lvm_lv_resizing
- id_map_base
- file_symlinks
- container_push_target
- network_vlan_physical
- storage_images_delete
- container_edit_metadata
- container_snapshot_stateful_migration
- storage_driver_ceph
- storage_ceph_user_name
- resource_limits
- storage_volatile_initial_source
- storage_ceph_force_osd_reuse
- storage_block_filesystem_btrfs
- resources
- kernel_limits
- storage_api_volume_rename
- macaroon_authentication
- network_sriov
- console
- restrict_devlxd
- migration_pre_copy
- infiniband
- maas_network
- devlxd_events
- proxy
- network_dhcp_gateway
- file_get_symlink
- network_leases
- unix_device_hotplug
- storage_api_local_volume_handling
- operation_description
- clustering
- event_lifecycle
- storage_api_remote_volume_handling
- nvidia_runtime
- container_mount_propagation
- container_backup
- devlxd_images
- container_local_cross_pool_handling
- proxy_unix
- proxy_udp
- clustering_join
- proxy_tcp_udp_multi_port_handling
- network_state
- proxy_unix_dac_properties
- container_protection_delete
- unix_priv_drop
- pprof_http
- proxy_haproxy_protocol
- network_hwaddr
- proxy_nat
- network_nat_order
- container_full
- candid_authentication
- backup_compression
- candid_config
- nvidia_runtime_config
- storage_api_volume_snapshots
- storage_unmapped
- projects
- candid_config_key
- network_vxlan_ttl
- container_incremental_copy
- usb_optional_vendorid
- snapshot_scheduling
- snapshot_schedule_aliases
- container_copy_project
- clustering_server_address
- clustering_image_replication
- container_protection_shift
- snapshot_expiry
- container_backup_override_pool
- snapshot_expiry_creation
- network_leases_location
- resources_cpu_socket
- resources_gpu
- resources_numa
- kernel_features
- id_map_current
- event_location
- storage_api_remote_volume_snapshots
- network_nat_address
- container_nic_routes
- rbac
- cluster_internal_copy
- seccomp_notify
- lxc_features
- container_nic_ipvlan
- network_vlan_sriov
- storage_cephfs
- container_nic_ipfilter
- resources_v2
- container_exec_user_group_cwd
- container_syscall_intercept
- container_disk_shift
- storage_shifted
- resources_infiniband
- daemon_storage
- instances
- image_types
- resources_disk_sata
- clustering_roles
- images_expiry
- resources_network_firmware
- backup_compression_algorithm
- ceph_data_pool_name
- container_syscall_intercept_mount
- compression_squashfs
- container_raw_mount
- container_nic_routed
- container_syscall_intercept_mount_fuse
- container_disk_ceph
- virtual-machines
- image_profiles
- clustering_architecture
- resources_disk_id
- storage_lvm_stripes
- vm_boot_priority
- unix_hotplug_devices
- api_filtering
- instance_nic_network
- clustering_sizing
- firewall_driver
- projects_limits
- container_syscall_intercept_hugetlbfs
- limits_hugepages
- container_nic_routed_gateway
- projects_restrictions
- custom_volume_snapshot_expiry
- volume_snapshot_scheduling
- trust_ca_certificates
- snapshot_disk_usage
- clustering_edit_roles
- container_nic_routed_host_address
- container_nic_ipvlan_gateway
- resources_usb_pci
- resources_cpu_threads_numa
- resources_cpu_core_die
- api_os
- resources_system
- usedby_consistency
- resources_gpu_mdev
- console_vga_type
- projects_limits_disk
- storage_rsync_compression
- gpu_mdev
- resources_pci_iommu
- resources_network_usb
- resources_disk_address
- network_state_vlan
- gpu_sriov
- migration_stateful
- disk_state_quota
- storage_ceph_features
- gpu_mig
- clustering_join_token
- clustering_description
- server_trusted_proxy
- clustering_update_cert
- storage_api_project
- server_instance_driver_operational
- server_supported_storage_drivers
- event_lifecycle_requestor_address
api_status: stable
api_version: “1.0”
auth: trusted
public: false
auth_methods: - tls
environment:
addresses:- 172.30.42.55:8443
architectures: - x86_64
- i686
certificate: |
-----BEGIN CERTIFICATE-----
…
-----END CERTIFICATE-----
certificate_fingerprint: …
driver: lxc | qemu
driver_version: 4.0.10 | 5.2.0
firewall: nftables
kernel: Linux
kernel_architecture: x86_64
kernel_features:
netnsid_getifaddrs: “true”
seccomp_listener: “true”
seccomp_listener_continue: “true”
shiftfs: “false”
uevent_injection: “true”
unpriv_fscaps: “true”
kernel_version: 5.4.0-84-generic
lxc_features:
cgroup2: “true”
devpts_fd: “true”
idmapped_mounts_v2: “true”
mount_injection_file: “true”
network_gateway_device_route: “true”
network_ipvlan: “true”
network_l2proxy: “true”
network_phys_macvlan_mtu: “true”
network_veth_router: “true”
pidfd: “true”
seccomp_allow_deny_syntax: “true”
seccomp_notify: “true”
seccomp_proxy_send_notify_fd: “true”
os_name: Ubuntu
os_version: “20.04”
project: default
server: lxd
server_clustered: false
server_name: arya
server_pid: 4643
server_version: 4.0.7
storage: zfs
storage_version: 0.8.3-1ubuntu12.12
storage_supported_drivers: - name: lvm
version: 2.02.176(2) (2017-11-03) / 1.02.145 (2017-11-03) / 4.41.0
remote: false - name: zfs
version: 0.8.3-1ubuntu12.12
remote: false - name: ceph
version: 12.2.13
remote: true - name: btrfs
version: 4.15.1
remote: false - name: cephfs
version: 12.2.13
remote: true - name: dir
version: “1”
remote: false
- 172.30.42.55:8443
lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 20.04.3 LTS
Release: 20.04
Codename: focal
lxc version
Client version: 4.0.7
Server version: 4.0.7
uname -a
Linux … 5.4.0-84-generic #94-Ubuntu SMP Thu Aug 26 20:27:37 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
