So I have ran some updates and rebooted my machine . After rebooting , I tried to start up the nodes again and I noticed they’re stuck in a stopped state…
Here are my logs that this spits out when in debug mode.
[root@dev ~]# lxd --debug
INFO[10-12|21:36:52] LXD 2.15 is starting in normal mode path=/var/lib/lxd
WARN[10-12|21:36:52] AppArmor support has been disabled because of lack of kernel support
INFO[10-12|21:36:52] Kernel uid/gid map:
INFO[10-12|21:36:52] - u 0 0 4294967295
INFO[10-12|21:36:52] - g 0 0 4294967295
INFO[10-12|21:36:52] Configured LXD uid/gid map:
INFO[10-12|21:36:52] - u 0 1000000 1000000000
INFO[10-12|21:36:52] - g 0 1000000 1000000000
DBUG[10-12|21:36:52] Initializing and checking storage pool "default".
DBUG[10-12|21:36:52] Initializing a DIR driver.
DBUG[10-12|21:36:52] Checking DIR storage pool "default".
INFO[10-12|21:36:52] Connecting to a remote simplestreams server
INFO[10-12|21:36:52] Connecting to a remote simplestreams server
INFO[10-12|21:36:52] Expiring log files
INFO[10-12|21:36:52] Done expiring log files
INFO[10-12|21:36:52] Starting /dev/lxd handler
DBUG[10-12|21:36:52] Looking for existing certificates cert=/var/lib/lxd/server.crt key=/var/lib/lxd/server.key
INFO[10-12|21:36:52] LXD isn't socket activated
INFO[10-12|21:36:52] REST API daemon:
INFO[10-12|21:36:52] - binding Unix socket socket=/var/lib/lxd/unix.socket
INFO[10-12|21:36:52] Pruning expired images
INFO[10-12|21:36:52] Done pruning expired images
INFO[10-12|21:36:52] Updating images
DBUG[10-12|21:36:52] Processing image alias=16.04 fp=03c2fa6716b5f41684457ca5e1b7316df520715b7fea0378f9306d16fdc646ee protocol=simplestreams server=https://cloud-images.ubuntu.com/releases
INFO[10-12|21:36:52] Connecting to a remote simplestreams server
DBUG[10-12|21:36:52] Initializing a DIR driver.
INFO[10-12|21:36:52] Starting container action=start created=2017-10-02T14:56:47+0000 ephemeral=false name=centy stateful=false used=2017-10-09T19:31:55+0000
DBUG[10-12|21:36:52] handling ip=@ method=GET url=/1.0
DBUG[10-12|21:36:52] Initializing a DIR driver.
DBUG[10-12|21:36:52]
{
"type": "sync",
"status": "Success",
"status_code": 200,
"operation": "",
"error_code": 0,
"error": "",
"metadata": {
"config": {},
"api_extensions": [
"storage_zfs_remove_snapshots",
"container_host_shutdown_timeout",
"container_syscall_filtering",
"auth_pki",
"container_last_used_at",
"etag",
"patch",
"usb_devices",
"https_allowed_credentials",
"image_compression_algorithm",
"directory_manipulation",
"container_cpu_time",
"storage_zfs_use_refquota",
"storage_lvm_mount_options",
"network",
"profile_usedby",
"container_push",
"container_exec_recording",
"certificate_update",
"container_exec_signal_handling",
"gpu_devices",
"container_image_properties",
"migration_progress",
"id_map",
"network_firewall_filtering",
"network_routes",
"storage",
"file_delete",
"file_append",
"network_dhcp_expiry",
"storage_lvm_vg_rename",
"storage_lvm_thinpool_rename",
"network_vlan",
"image_create_aliases",
"container_stateless_copy",
"container_only_migration",
"storage_zfs_clone_copy",
"unix_device_rename",
"storage_lvm_use_thinpool",
"storage_rsync_bwlimit",
"network_vxlan_interface",
"storage_btrfs_mount_options",
"entity_description",
"image_force_refresh",
"storage_lvm_lv_resizing",
"id_map_base",
"file_symlinks",
"container_push_target"
],
"api_status": "stable",
"api_version": "1.0",
"auth": "trusted",
"public": false,
"environment": {
"addresses": [],
"architectures": [
"x86_64",
"i686"
],
"certificate": "-----BEGIN CERTIFICATE-----\nMIIFdDCCA1ygAwIBAgIRAMqsodky16LPR4vMe8sZGkAwDQYJKoZIhvcNAQELBQAw\nQzEcMBoGA1UEChMTbGludXhjb250YWluZXJzLm9yZzEjMCEGA1UEAwwacm9vdEBs\nb2NhbGhvc3QubG9jYWxkb21haW4wHhcNMTcwNzAzMTgzNzAxWhcNMjcwNzAxMTgz\nNzAxWjBDMRwwGgYDVQQKExNsaW51eGNvbnRhaW5lcnMub3JnMSMwIQYDVQQDDBpy\nb290QGxvY2FsaG9zdC5sb2NhbGRvbWFpbjCCAiIwDQYJKoZIhvcNAQEBBQADggIP\nADCCAgoCggIBAMQsJduyhlQt0GBLd65DCwY4mf5esPjkF6Eqg8KP+lQNP3FAl++i\n4sttzPxEQS2VHUnIuNufNGpfyc0FO6GP0+LON74JsixvMLeTGosk4SEFstqukXK/\nPBPyizgTzNzwk+/62MYr7EcyjtpgwoF8o/ctYOA5OAjCyF9y4c8I8PW78gxHM4bp\np9jbZ3Z8MqedG7ZQcuW9OA5JSSrzLOXiqCDO8jNMO+L1x4arveT4H/Uq8CwQnL3f\nRlctk0QIMgDf+sH5UD8viQpPVmwNKXhVV98tqSyYbMHKGewP+OMJUMmrQ7lsoCff\n/rtr3LQKArAQCqUvW/UPtMjQvLTuGWgmEJwRkGzKKOlvnSGh4a0BLSSDMXrO7un/\nXP0a14zMMV4sYtOuM+/T6zbk7t95MP9atXUp+HHn6TsbaGJ16ynoAaXk5BWKtKwM\nd8ZMvfKEEMB+l8QMeMZvXpm5JPvNs1HdxzWK0hqT3y7U7s6itdhhWki7lyA7yKQn\n7Pp8cR8lbaF5VhI9KVFCVXZD++M46PIUZHGmagCWgAfTb8DHScPdLBCVFFTcsTj9\npCW1tVEAu/GvbtVCQ+ANXgm3/1ksfdIzXrZb8paGmDw61s+RzeZSfOPv+hOz93pV\nLK/2Ok2Y6RFH/vPmMWObKtDdivjIKuQQoWSOt5tSBlTafWcgCu46KxrjAgMBAAGj\nYzBhMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMB\nAf8EAjAAMCwGA1UdEQQlMCOCFWxvY2FsaG9zdC5sb2NhbGRvbWFpbocECmQABYcE\nwKh6ATANBgkqhkiG9w0BAQsFAAOCAgEAcz2eQtTWu9yMrx89xU28y6kQbtw7FP4o\n7fxD+zchPBaEfalViaFdJ5bxljpNUt/+j/qNAumrRuHyx6T/g7RLfDWsnJw/QI7k\nz7aW/AqGiUa+PgWLjeg18df5loANfEYWi4ZBTe5VAh6V9BsT+2bWM7or6Ctz++fi\noFH4gNqCChrxsBP4vTvYqZI6XWmY0cfR9Csd7yg16XFy8AopHX4JIDLyb8dLmLlE\nUB+2vc8r4xqV92hVqIkrSIWqsyjuTH+NvnrhQnZK4+5dpTua4uJg2hCD6kj9GUE7\nZymKfpKOksE+IIcagWQYjw38Fs6pnaYuH+v36anzRuNSo2ajCCtFhAqB3r4r1PJ3\nI+oYIKeWbw5p8HtHCa8qMTq7I8F0vg+e92W4Sw32U9XplI22RssDJl9Crewv0zhE\nVzbiXFgdE0dedVeh9lBW7ZMQicbxJu4EiuAgKXte/SxK+a5ht7TRRyymsSimR8EC\nS7OFgpl4eppNmyUy8p4kRcJaRDQyj/wjemynGNuTe0jt6KM6ZSEHIJg2HlDO6lkI\nYR4s67KjsV7oTOxdOBkGXPVsfTsSAon00qvkgxO71Id+//+b5cHnFBKgVXpHpTJc\nwkhbmqR0/Hb5luWvJxbhbSA33x3zm0Y+E72dFxIpY/d0wC7oKDK3LzaJQtqec9gB\npnjnLp14TYU=\n-----END CERTIFICATE-----\n",
"certificate_fingerprint": "d07b69b84e69f017c325f7df6ecd46363c8c61da30b661875ad0ad0b4604c0f9",
"driver": "lxc",
"driver_version": "2.0.7",
"kernel": "Linux",
"kernel_architecture": "x86_64",
"kernel_version": "3.10.0-693.2.2.el7.x86_64",
"server": "lxd",
"server_pid": 29411,
"server_version": "2.15",
"storage": "dir",
"storage_version": "1"
}
}
}
DBUG[10-12|21:36:53] handling ip=@ method=GET url=/internal/containers/14/onstart
DBUG[10-12|21:36:53] Initializing a DIR driver.
DBUG[10-12|21:36:53] Scheduler: container centy started: re-balancing
DBUG[10-12|21:36:53]
{
"type": "sync",
"status": "Success",
"status_code": 200,
"operation": "",
"error_code": 0,
"error": "",
"metadata": {}
}
DBUG[10-12|21:36:53] handling ip=@ method=GET url=/1.0
DBUG[10-12|21:36:53] Initializing a DIR driver.
DBUG[10-12|21:36:53]
{
"type": "sync",
"status": "Success",
"status_code": 200,
"operation": "",
"error_code": 0,
"error": "",
"metadata": {
"config": {},
"api_extensions": [
"storage_zfs_remove_snapshots",
"container_host_shutdown_timeout",
"container_syscall_filtering",
"auth_pki",
"container_last_used_at",
"etag",
"patch",
"usb_devices",
"https_allowed_credentials",
"image_compression_algorithm",
"directory_manipulation",
"container_cpu_time",
"storage_zfs_use_refquota",
"storage_lvm_mount_options",
"network",
"profile_usedby",
"container_push",
"container_exec_recording",
"certificate_update",
"container_exec_signal_handling",
"gpu_devices",
"container_image_properties",
"migration_progress",
"id_map",
"network_firewall_filtering",
"network_routes",
"storage",
"file_delete",
"file_append",
"network_dhcp_expiry",
"storage_lvm_vg_rename",
"storage_lvm_thinpool_rename",
"network_vlan",
"image_create_aliases",
"container_stateless_copy",
"container_only_migration",
"storage_zfs_clone_copy",
"unix_device_rename",
"storage_lvm_use_thinpool",
"storage_rsync_bwlimit",
"network_vxlan_interface",
"storage_btrfs_mount_options",
"entity_description",
"image_force_refresh",
"storage_lvm_lv_resizing",
"id_map_base",
"file_symlinks",
"container_push_target"
],
"api_status": "stable",
"api_version": "1.0",
"auth": "trusted",
"public": false,
"environment": {
"addresses": [],
"architectures": [
"x86_64",
"i686"
],
"certificate": "-----BEGIN CERTIFICATE-----\nMIIFdDCCA1ygAwIBAgIRAMqsodky16LPR4vMe8sZGkAwDQYJKoZIhvcNAQELBQAw\nQzEcMBoGA1UEChMTbGludXhjb250YWluZXJzLm9yZzEjMCEGA1UEAwwacm9vdEBs\nb2NhbGhvc3QubG9jYWxkb21haW4wHhcNMTcwNzAzMTgzNzAxWhcNMjcwNzAxMTgz\nNzAxWjBDMRwwGgYDVQQKExNsaW51eGNvbnRhaW5lcnMub3JnMSMwIQYDVQQDDBpy\nb290QGxvY2FsaG9zdC5sb2NhbGRvbWFpbjCCAiIwDQYJKoZIhvcNAQEBBQADggIP\nADCCAgoCggIBAMQsJduyhlQt0GBLd65DCwY4mf5esPjkF6Eqg8KP+lQNP3FAl++i\n4sttzPxEQS2VHUnIuNufNGpfyc0FO6GP0+LON74JsixvMLeTGosk4SEFstqukXK/\nPBPyizgTzNzwk+/62MYr7EcyjtpgwoF8o/ctYOA5OAjCyF9y4c8I8PW78gxHM4bp\np9jbZ3Z8MqedG7ZQcuW9OA5JSSrzLOXiqCDO8jNMO+L1x4arveT4H/Uq8CwQnL3f\nRlctk0QIMgDf+sH5UD8viQpPVmwNKXhVV98tqSyYbMHKGewP+OMJUMmrQ7lsoCff\n/rtr3LQKArAQCqUvW/UPtMjQvLTuGWgmEJwRkGzKKOlvnSGh4a0BLSSDMXrO7un/\nXP0a14zMMV4sYtOuM+/T6zbk7t95MP9atXUp+HHn6TsbaGJ16ynoAaXk5BWKtKwM\nd8ZMvfKEEMB+l8QMeMZvXpm5JPvNs1HdxzWK0hqT3y7U7s6itdhhWki7lyA7yKQn\n7Pp8cR8lbaF5VhI9KVFCVXZD++M46PIUZHGmagCWgAfTb8DHScPdLBCVFFTcsTj9\npCW1tVEAu/GvbtVCQ+ANXgm3/1ksfdIzXrZb8paGmDw61s+RzeZSfOPv+hOz93pV\nLK/2Ok2Y6RFH/vPmMWObKtDdivjIKuQQoWSOt5tSBlTafWcgCu46KxrjAgMBAAGj\nYzBhMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMB\nAf8EAjAAMCwGA1UdEQQlMCOCFWxvY2FsaG9zdC5sb2NhbGRvbWFpbocECmQABYcE\nwKh6ATANBgkqhkiG9w0BAQsFAAOCAgEAcz2eQtTWu9yMrx89xU28y6kQbtw7FP4o\n7fxD+zchPBaEfalViaFdJ5bxljpNUt/+j/qNAumrRuHyx6T/g7RLfDWsnJw/QI7k\nz7aW/AqGiUa+PgWLjeg18df5loANfEYWi4ZBTe5VAh6V9BsT+2bWM7or6Ctz++fi\noFH4gNqCChrxsBP4vTvYqZI6XWmY0cfR9Csd7yg16XFy8AopHX4JIDLyb8dLmLlE\nUB+2vc8r4xqV92hVqIkrSIWqsyjuTH+NvnrhQnZK4+5dpTua4uJg2hCD6kj9GUE7\nZymKfpKOksE+IIcagWQYjw38Fs6pnaYuH+v36anzRuNSo2ajCCtFhAqB3r4r1PJ3\nI+oYIKeWbw5p8HtHCa8qMTq7I8F0vg+e92W4Sw32U9XplI22RssDJl9Crewv0zhE\nVzbiXFgdE0dedVeh9lBW7ZMQicbxJu4EiuAgKXte/SxK+a5ht7TRRyymsSimR8EC\nS7OFgpl4eppNmyUy8p4kRcJaRDQyj/wjemynGNuTe0jt6KM6ZSEHIJg2HlDO6lkI\nYR4s67KjsV7oTOxdOBkGXPVsfTsSAon00qvkgxO71Id+//+b5cHnFBKgVXpHpTJc\nwkhbmqR0/Hb5luWvJxbhbSA33x3zm0Y+E72dFxIpY/d0wC7oKDK3LzaJQtqec9gB\npnjnLp14TYU=\n-----END CERTIFICATE-----\n",
"certificate_fingerprint": "d07b69b84e69f017c325f7df6ecd46363c8c61da30b661875ad0ad0b4604c0f9",
"driver": "lxc",
"driver_version": "2.0.7",
"kernel": "Linux",
"kernel_architecture": "x86_64",
"kernel_version": "3.10.0-693.2.2.el7.x86_64",
"server": "lxd",
"server_pid": 29411,
"server_version": "2.15",
"storage": "dir",
"storage_version": "1"
}
}
}
DBUG[10-12|21:36:53] handling ip=@ method=GET url="/internal/containers/14/onstop?target=stop"
EROR[10-12|21:36:53] stop hook failed container=centy err="Container is already running a start operation"
DBUG[10-12|21:36:53]
{
"error": "Container is already running a start operation",
"error_code": 500,
"type": "error"
}
DBUG[10-12|21:36:53] Scheduler: network: vethTVH9GF has been added: updating network priorities
DBUG[10-12|21:36:54] Image already exists in the db image=03c2fa6716b5f41684457ca5e1b7316df520715b7fea0378f9306d16fdc646ee
DBUG[10-12|21:36:54] Already up to date fp=03c2fa6716b5f41684457ca5e1b7316df520715b7fea0378f9306d16fdc646ee
DBUG[10-12|21:36:54] Processing image alias=centos/7 fp=cc972a0b0b8026427b7aa1f8c6de232d2cca588eb5a751ffb2886079043d130f protocol=simplestreams server=https://images.linuxcontainers.org
INFO[10-12|21:36:54] Connecting to a remote simplestreams server
INFO[10-12|21:36:56] Downloading image alias=centos/7 server=https://images.linuxcontainers.org
EROR[10-12|21:36:58] Failed starting container action=start created=2017-10-02T14:56:47+0000 ephemeral=false name=centy stateful=false used=2017-10-09T19:31:55+0000
INFO[10-12|21:37:13] Image downloaded alias=centos/7 server=https://images.linuxcontainers.org
INFO[10-12|21:37:13] Done updating images
This Error sticks out near the bottom.
EROR[10-12|21:36:53] stop hook failed container=centy err="Container is already running a start operation"
Any ideas for how I can fix this?
I am running this on CENTOS 7 . I see an app armor warning at the top. So I am not sure if that is anything to worry about either.
Let me know if you want more information.
lxd --version
2.15
thanks!
