Not able to create container after kernel upgrade

,

Hi,
I am running lxd 3.5 in debian bullseye after upgrading kernel to 5.9.0-2-amd64- not able to create any containers . creating command seems stuck for long time .
also ipv6 address is not showing in lxc list

DBUG[11-24|09:39:56] Connecting to a local LXD over a Unix socket
DBUG[11-24|09:39:56] Sending request to LXD                   method=GET url=http://unix.socket/1.0 etag=
DBUG[11-24|09:39:56] Got response struct from LXD
DBUG[11-24|09:39:56]
        {
                "config": {
                        "core.https_address": "[::]:8448",
                        "core.trust_password": true
                },
                "api_extensions": [
                        "storage_zfs_remove_snapshots",
                        "container_host_shutdown_timeout",
                        "container_stop_priority",
                        "container_syscall_filtering",
                        "auth_pki",
                        "container_last_used_at",
                        "etag",
                        "patch",
                        "usb_devices",
                        "https_allowed_credentials",
                        "image_compression_algorithm",
                        "directory_manipulation",
                        "container_cpu_time",
                        "storage_zfs_use_refquota",
                        "storage_lvm_mount_options",
                        "network",
                        "profile_usedby",
                        "container_push",
                        "container_exec_recording",
                        "certificate_update",
                        "container_exec_signal_handling",
                        "gpu_devices",
                        "container_image_properties",
                        "migration_progress",
                        "id_map",
                        "network_firewall_filtering",
                        "network_routes",
                        "storage",
                        "file_delete",
                        "file_append",
                        "network_dhcp_expiry",
                        "storage_lvm_vg_rename",
                        "storage_lvm_thinpool_rename",
                        "network_vlan",
                        "image_create_aliases",
                        "container_stateless_copy",
                        "container_only_migration",
                        "storage_zfs_clone_copy",
                        "unix_device_rename",
                        "storage_lvm_use_thinpool",
                        "storage_rsync_bwlimit",
                        "network_vxlan_interface",
                        "storage_btrfs_mount_options",
                        "entity_description",
                        "image_force_refresh",
                        "storage_lvm_lv_resizing",
                        "id_map_base",
                        "file_symlinks",
                        "container_push_target",
                        "network_vlan_physical",
                        "storage_images_delete",
                        "container_edit_metadata",
                        "container_snapshot_stateful_migration",
                        "storage_driver_ceph",
                        "storage_ceph_user_name",
                        "resource_limits",
                        "storage_volatile_initial_source",
                        "storage_ceph_force_osd_reuse",
                        "storage_block_filesystem_btrfs",
                        "resources",
                        "kernel_limits",
                        "storage_api_volume_rename",
                        "macaroon_authentication",
                        "network_sriov",
                        "console",
                        "restrict_devlxd",
                        "migration_pre_copy",
                        "infiniband",
                        "maas_network",
                        "devlxd_events",
                        "proxy",
                        "network_dhcp_gateway",
                        "file_get_symlink",
                        "network_leases",
                        "unix_device_hotplug",
                        "storage_api_local_volume_handling",
                        "operation_description",
                        "clustering",
                        "event_lifecycle",
                        "storage_api_remote_volume_handling",
                        "nvidia_runtime",
                        "container_mount_propagation",
                        "container_backup",
                        "devlxd_images",
                        "container_local_cross_pool_handling",
                        "proxy_unix",
                        "proxy_udp",
                        "clustering_join",
                        "proxy_tcp_udp_multi_port_handling",
                        "network_state",
                        "proxy_unix_dac_properties",
                        "container_protection_delete",
                        "unix_priv_drop",
                        "pprof_http",
                        "proxy_haproxy_protocol",
                        "network_hwaddr",
                        "proxy_nat",
                        "network_nat_order",
                        "container_full",
                        "candid_authentication",
                        "backup_compression",
                        "candid_config",
                        "nvidia_runtime_config",
                        "storage_api_volume_snapshots",
                        "storage_unmapped",
                        "projects",
                        "candid_config_key",
                        "network_vxlan_ttl",
                        "container_incremental_copy",
                        "usb_optional_vendorid",
                        "snapshot_scheduling",
                        "container_copy_project",
                        "clustering_server_address",
                        "clustering_image_replication",
                        "container_protection_shift",
                        "snapshot_expiry",
                        "container_backup_override_pool",
                        "snapshot_expiry_creation",
                        "network_leases_location",
                        "resources_cpu_socket",
                        "resources_gpu",
                        "resources_numa",
                        "kernel_features",
                        "id_map_current",
                        "event_location",
                        "storage_api_remote_volume_snapshots",
                        "network_nat_address",
                        "container_nic_routes",
                        "rbac",
                        "cluster_internal_copy",
                        "seccomp_notify",
                        "lxc_features",
                        "container_nic_ipvlan",
                        "network_vlan_sriov",
                        "storage_cephfs",
                        "container_nic_ipfilter",
                        "resources_v2",
                        "container_exec_user_group_cwd",
                        "container_syscall_intercept",
                        "container_disk_shift",
                        "storage_shifted",
                        "resources_infiniband",
                        "daemon_storage",
                        "instances",
                        "image_types",
                        "resources_disk_sata",
                        "clustering_roles",
                        "images_expiry",
                        "resources_network_firmware",
                        "backup_compression_algorithm",
                        "ceph_data_pool_name",
                        "container_syscall_intercept_mount",
                        "compression_squashfs",
                        "container_raw_mount",
                        "container_nic_routed",
                        "container_syscall_intercept_mount_fuse",
                        "container_disk_ceph",
                        "virtual-machines",
                        "image_profiles",
                        "clustering_architecture",
                        "resources_disk_id",
                        "storage_lvm_stripes",
                        "vm_boot_priority",
                        "unix_hotplug_devices",
                        "api_filtering",
                        "instance_nic_network",
                        "clustering_sizing",
                        "firewall_driver",
                        "projects_limits",
                        "container_syscall_intercept_hugetlbfs",
                        "limits_hugepages",
                        "container_nic_routed_gateway",
                        "projects_restrictions",
                        "custom_volume_snapshot_expiry",
                        "volume_snapshot_scheduling",
                        "trust_ca_certificates",
                        "snapshot_disk_usage",
                        "clustering_edit_roles",
                        "container_nic_routed_host_address",
                        "container_nic_ipvlan_gateway",
                        "resources_usb_pci",
                        "resources_cpu_threads_numa",
                        "resources_cpu_core_die",
                        "api_os",
                        "container_nic_routed_host_table",
                        "container_nic_ipvlan_host_table",
                        "container_nic_ipvlan_mode",
                        "resources_system",
                        "images_push_relay",
                        "network_dns_search",
                        "container_nic_routed_limits",
                        "instance_nic_bridged_vlan",
                        "network_state_bond_bridge",
                        "usedby_consistency",
                        "custom_block_volumes",
                        "clustering_failure_domains",
                        "resources_gpu_mdev",
                        "console_vga_type",
                        "projects_limits_disk",
                        "network_type_macvlan",
                        "network_type_sriov",
                        "container_syscall_intercept_bpf_devices",
                        "network_type_ovn",
                        "projects_networks",
                        "projects_networks_restricted_uplinks",
                        "custom_volume_backup",
                        "backup_override_name",
                        "storage_rsync_compression",
                        "network_type_physical",
                        "network_ovn_external_subnets",
                        "network_ovn_nat",
                        "network_ovn_external_routes_remove",
                        "tpm_device_type",
                        "storage_zfs_clone_copy_rebase"
                ],
                "api_status": "stable",
                "api_version": "1.0",
                "auth": "trusted",
                "public": false,
                "auth_methods": [
                        "tls"
                ],
                "environment": {
                        "addresses": [
                                "149.56.107.165:8448",
                                "[2607:5300:60:a5a5::]:8448",
                                "10.0.3.1:8448",
                                "10.78.204.1:8448",
                                "[fd42:ca1b:d8d3:acb0::1]:8448"
                        ],
                        "architectures": [
                                "x86_64",
                                "i686"
                        ],
                        "certificate": "-----BEGIN CERTIFICATE-----\nMIICCjCCAZCgAwIBAgIRAKkGm4yD9urh45y1pu5Fv58wCgYIKoZIzj0EAwMwNjEc\nMBoGA1UEChMTbGludXhjb250YWluZXJzLm9yZzEWMBQGA1UEAwwNcm9vdEBuczUz\nMDA0OTAeFw0yMDEwMjgxMjI2NTNaFw0zMDEwMjYxMjI2NTNaMDYxHDAaBgNVBAoT\nE2xpbnV4Y29udGFpbmVycy5vcmcxFjAUBgNVBAMMDXJvb3RAbnM1MzAwNDkwdjAQ\nBgcqhkjOPQIBBgUrgQQAIgNiAASDYZb6BUX/3x//IlKy/VjTzts71iN9lfENdsf6\noGzwjNRIlqlPhgZaFevr4FUYv//XguAd44oxnCqWxH376me7LcYbvVzSWRJzbUi1\nI3H5b3ZwxwSTapJrpYNF8+N5FdijYjBgMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUE\nDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMCsGA1UdEQQkMCKCCG5zNTMwMDQ5\nhwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMAoGCCqGSM49BAMDA2gAMGUCMFo3XzhD\nJMxevscuRWc5GJ16/zhtky/ies01CZnIHczWJsgwRWwq49AO4tcHp5MsmQIxALMi\nyIEzBKowPf5T0pTj5v1rgMS31MEo/39vZe+YlQf7iGuMBwN+UHCXwZBwIn/AsA==\n-----END CERTIFICATE-----\n",
                        "certificate_fingerprint": "21a4f8c30273c49dbae2907de56b8b21f4f2f53d17d08658d90da7b99eafa115",
                        "driver": "lxc",
                        "driver_version": "4.0.4",
                        "firewall": "xtables",
                        "kernel": "Linux",
                        "kernel_architecture": "x86_64",
                        "kernel_features": {
                                "netnsid_getifaddrs": "true",
                                "seccomp_listener": "true",
                                "seccomp_listener_continue": "true",
                                "shiftfs": "false",
                                "uevent_injection": "true",
                                "unpriv_fscaps": "true"
                        },
                        "kernel_version": "5.9.0-1-amd64",
                        "lxc_features": {
                                "cgroup2": "true",
                                "devpts_fd": "false",
                                "mount_injection_file": "true",
                                "network_gateway_device_route": "true",
                                "network_ipvlan": "true",
                                "network_l2proxy": "true",
                                "network_phys_macvlan_mtu": "true",
                                "network_veth_router": "true",
                                "pidfd": "true",
                                "seccomp_allow_deny_syntax": "true",
                                "seccomp_notify": "true",
                                "seccomp_proxy_send_notify_fd": "false"
                        },
                        "os_name": "Debian GNU/Linux",
                        "os_version": "",
                        "project": "default",
                        "server": "lxd",
                        "server_clustered": false,
                        "server_name": "cpu-6228",
                        "server_pid": 1352,
                        "server_version": "4.8",
                        "storage": "zfs",
                        "storage_version": "0.8.5-2"
                }
        }
Creating test
DBUG[11-24|09:39:56] Connecting to a remote simplestreams server
DBUG[11-24|09:39:56] Connected to the websocket: ws://unix.socket/1.0/events
DBUG[11-24|09:39:56] Sending request to LXD                   method=POST url=http://unix.socket/1.0/instances etag=
DBUG[11-24|09:39:56]
        {
                "architecture": "",
                "config": {},
                "devices": {},
                "ephemeral": false,
                "profiles": null,
                "stateful": false,
                "description": "",
                "name": "test",
                "source": {
                        "type": "image",
                        "certificate": "",
                        "alias": "debian/buster",
                        "server": "https://images.linuxcontainers.org",
                        "protocol": "simplestreams",
                        "mode": "pull"
                },
                "instance_type": "",
                "type": "container"
        }
DBUG[11-24|09:50:03] Got operation from LXD
DBUG[11-24|09:50:03]
        {
                "id": "8766af50-1d59-4fa4-a606-41dabeb61fa0",
                "class": "task",
                "description": "Creating container",
                "created_at": "2020-11-24T09:50:03.148393655Z",
                "updated_at": "2020-11-24T09:50:03.148393655Z",
                "status": "Running",
                "status_code": 103,
                "resources": {
                        "containers": [
                                "/1.0/containers/test"
                        ],
                        "instances": [
                                "/1.0/instances/test"
                        ]
                },
                "metadata": null,
                "may_cancel": false,
                "err": "",
                "location": "none"
        }
DBUG[11-24|09:50:03] Sending request to LXD                   method=GET url=http://unix.socket/1.0/operations/8766af50-1d59-4fa4-a606-41dabeb61fa0 etag=
DBUG[11-24|09:50:03] Got response struct from LXD
DBUG[11-24|09:50:03]
        {
                "id": "8766af50-1d59-4fa4-a606-41dabeb61fa0",
                "class": "task",
                "description": "Creating container",
                "created_at": "2020-11-24T09:50:03.148393655Z",
                "updated_at": "2020-11-24T09:50:03.148393655Z",
                "status": "Running",
                "status_code": 103,
                "resources": {
                        "containers": [
                                "/1.0/containers/test"
                        ],
                        "instances": [
                                "/1.0/instances/test"
                        ]
                },
                "metadata": null,
                "may_cancel": false,
                "err": "",
                "location": "none"
        }



+----------+---------+------------------------+------+-----------+-----------+
|   NAME   |  STATE  |          IPV4          | IPV6 |   TYPE    | SNAPSHOTS |
+----------+---------+------------------------+------+-----------+-----------+
| vm618027 | RUNNING | 1.1.1.1(eth0) |      | CONTAINER | 0         |
+----------+---------+------------------------+------+-----------+-----------+

please have a look on this .

any help ?

Hi!

Is the new Linux kernel a stock kernel for Debian?

The problem is not clear from what you show above. The lxc list shows that there is a VM (not container), that gets an IP address of 1.1.1.1. Is that the real IP address that you configured for the VM or did you mask the real IP address? Because 1.1.1.1 is not a private IP address (in fact it is a public DNS server).

yes its a new kernel . and vmXXX is just a name and type is container . ip address was a public Ip address and I can’t share its that’s why I just replaced Ip address to 1.1.1.1

root@cpu-6228:~# lxc info vm618027
Name: vm618027
Location: none
Remote: unix://
Architecture: x86_64
Created: 2020/10/28 16:10 UTC
Status: Running
Type: container
Profiles: default
Pid: 30745
Ips:
  eth0: inet    x.x.x.x veth43a3c644
  eth0: inet6   fe30::216:3eff:fe7a:8117        veth43a3c644
  lo:   inet    127.0.0.1
  lo:   inet6   ::1
Resources:
  Processes: 7
  Disk usage:
    root: 849.18MB
    vm618027: 918.26MB
  CPU usage:
    CPU usage (in seconds): 0
  Memory usage:
    Memory (current): 23.94MB
    Memory (peak): 26.69MB
  Network usage:
    eth0:
      Bytes received: 679.02kB
      Bytes sent: 78.11kB
      Packets received: 15773
      Packets sent: 1525
    lo:
      Bytes received: 516B
      Bytes sent: 516B
      Packets received: 6
      Packets sent: 6

before kernel upgrade ipv6 was there in lxc list now it’s not there . I suspect this is the reason why I can’t create container.

while I start lxd in debug mode I got below warning

WARN[11-24|10:34:35] Failed to update instance types: Get "https://us.images.linuxcontainers.org/meta/instance-types/.yaml": read tcp [2607:5300:60:a5a5::]:34094->[2001:67c:1562::41]:443: read: connection reset by peer

@brauner ideas?

any help ?

It’s pretty unclear what is happening here and not a lot to go on. Is the container actually created when the creation is hanging?

hi @brauner
container vm618027 was created before the kernel upgrade . after the kernel upgrade ipv6 networking is disabled. and also not able to create new container . the creation command hangs.

any idea ?

Ok, so the slowness is most likely due to connectivity issue with our US based image server. This may have fixed by now (the server was moved across location over the last weekend and the Canonical IS team has been dealing with network issues since…), if not, you could force the use of the UK based server instead (uk.images.linuxcontainers.org) which should not suffer from such issues.

That doesn’t really explain the IPv6 issue though, but from your output above, your container does have a link local address so it’s not completely lacking IPv6 support.
How does the container usually get its global IPv6 address? Is this a managed LXD bridge?

okay now I am able to create containers , I think it’s because of the connectivity issue .
but still Ipv6 is not showing

, While starting LXD i got below error something related with apparmor

EROR[11-30|13:37:32] Failed to bring up network               err="Failed to run: apparmor_parser -rWL /var/lib/lxd/security/apparmor/cache /var/lib/lxd/security/apparmor/profiles/lxd_dnsmasq-lxdbr0: Found reference to variable PROC, but is never declared" name=lxdbr0

Can you show the content of /var/lib/lxd/security/apparmor/profiles/lxd_dnsmasq-lxdbr0?


root@cpu-5219:/etc/apparmor/init/network-interface-security# cat /var/lib/lxd/security/apparmor/profiles/lxd_dnsmasq-lxdbr0
#include <tunables/global>
profile "lxd_dnsmasq-lxdbr0_</var/lib/lxd>" flags=(attach_disconnected,mediate_deleted) {
  #include <abstractions/base>
  #include <abstractions/dbus>
  #include <abstractions/nameservice>

  # Capabilities
  capability chown,
  capability net_bind_service,
  capability setgid,
  capability setuid,
  capability dac_override,
  capability dac_read_search,
  capability net_admin,         # for DHCP server
  capability net_raw,           # for DHCP server ping checks

  # Network access
  network inet raw,
  network inet6 raw,

  # Network-specific paths
  /var/lib/lxd/networks/lxdbr0/dnsmasq.hosts/{,*} r,
  /var/lib/lxd/networks/lxdbr0/dnsmasq.leases rw,
  /var/lib/lxd/networks/lxdbr0/dnsmasq.raw r,

  # Additional system files
  @{PROC}/sys/net/ipv6/conf/*/mtu r,
  @{PROC}/@{pid}/fd/ r,

  # System configuration access
  /etc/gai.conf           r,
  /etc/group              r,
  /etc/host.conf          r,
  /etc/hosts              r,
  /etc/nsswitch.conf      r,
  /etc/passwd             r,
  /etc/protocols          r,

  /etc/resolv.conf        r,
  /etc/resolvconf/run/resolv.conf r,

  /run/{resolvconf,NetworkManager,systemd/resolve,connman,netconfig}/resolv.conf r,
  /run/systemd/resolve/stub-resolv.conf r,
}

above is the content

seem it’s fixed by adding

@{PROC}=/proc/

in /etc/apparmor.d/tunables/proc

Is that with a LXD snap or manually built?

I would have expected the snap to have the right apparmor bits out of the box.

On Ubuntu, /etc/apparmor.d/tunables/proc has that variable defined with the file originating from the apparmor package itself.

@stgraber
hi ,
its manually built one . when I checked in other servers which are working fine , /etc/apparmor.d/tunables/proc is existing and the declaration of PROC is existing .
but here it was missing , I just touched a file named proc and added @{PROC}=/proc/