snap refresh lxd --channel=latest/stable
lxd 5.5-37534be from Canonical✓ refreshed
root@h:~# lvs
LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert
LXDThinPool ALL twi-aotz-- <7.25t 0.58 10.66
containers_asafe ALL Vwi-aotz-k 10.00g LXDThinPool 92.54
containers_bir ALL Vwi-aotz-k 10.00g LXDThinPool 94.18
containers_faucex ALL Vwi-aotz-k 10.00g LXDThinPool 34.41
containers_holdip ALL Vwi-aotz-k 10.00g LXDThinPool 33.87
containers_pny ALL Vwi-aotz-k 10.00g LXDThinPool 63.83
containers_sapp ALL Vwi-aotz-k 10.00g LXDThinPool 99.77
images_e299296138c256b79dda4e61ac7454cf4ac134b43f5521f1ac894f49a9421d00 ALL Vwi---tz-k 10.00g LXDThinPool
top on the host (for only the processes inside my containers) RES keeps going up and up and up
top - 15:25:32 up 1 day, 23:22, 5 users, load average: 5.52, 4.68, 3.08
Tasks: 6 total, 0 running, 6 sleeping, 0 stopped, 0 zombie
%Cpu(s): 1.2 us, 6.4 sy, 7.4 ni, 80.0 id, 3.8 wa, 0.0 hi, 1.3 si, 0.0 st
MiB Mem : 64214.8 total, 51722.8 free, 5398.5 used, 7093.5 buff/cache
MiB Swap: 10752.0 total, 10749.0 free, 3.0 used. 58010.1 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
901138 root 20 0 890872 61824 27148 S 0.3 0.1 0:05.29 relay
901041 root 20 0 919144 91024 28740 S 0.0 0.1 0:11.33 faucex
901586 root 34 14 2850744 510764 152780 S 66.3 0.8 5:56.35 SAPP.daemon
901814 root 39 19 2791840 443136 83708 S 18.3 0.7 1:12.45 PNY.daemon
901694 root 39 19 3403632 991604 137604 S 12.3 1.5 1:02.80 BIR.daemon
901470 root 34 14 4543480 2.2g 137512 S 36.0 3.4 6:14.83 ASAFE.daemon
notice 2.2g for one process (that will start next with others) even though I’m using both nice and cpulimit on each process inside the container and I have set
limits.cpu.allowance: 7ms/10ms
limits.cpu.priority: "10"
limits.memory: 10%
limits.memory.enforce: hard
limits.memory.swap.priority: "10"
on each container
One of the processes for example hasn’t done anything for 8 minutes
tail /home/SAPP/data/debug.log
2022-08-31 15:22:22 UpdateTip: new best=bc39f923cd206f6fb0682f97b8c05b493255393a3504b64cc3f933b7cd183999 height=884574 version=7 log2_work=68.6262293348501657 tx=8237975 date=2021-01-02 05:41:07 progress=0.525336 cache=9.1MiB(59501txo)
2022-08-31 15:22:22 ProcessNewBlock : ACCEPTED Block 884574 in 312571 milliseconds with size=147180
2022-08-31 15:22:22 receive version message: /Sapphire Core:1.4.2.3/: version 70929, blocks=1741676, us=167.114.211.121:46382, peer=13
2022-08-31 15:22:22 timeOffset (-312 seconds) too large. Disconnecting node 209.145.49.8:45328
2022-08-31 15:22:22 CheckBlock: Masternode/Budget payment checks skipped on sync
strace on the process shows
sapp t
strace: Process 394 attached
restart_syscall(<... resuming interrupted read ...>) = -1 ETIMEDOUT (Connection timed out)
futex(0x7ffd548ca480, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffd548ca4d0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=170975, tv_nsec=143569148}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
futex(0x7ffd548ca480, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffd548ca4d0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=170975, tv_nsec=350256423}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
futex(0x7ffd548ca480, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffd548ca4d0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=170975, tv_nsec=550503722}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
futex(0x7ffd548ca480, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffd548ca4d0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=170975, tv_nsec=750820321}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
futex(0x7ffd548ca480, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffd548ca4d0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=170975, tv_nsec=951103035}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
futex(0x7ffd548ca480, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffd548ca4d0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=170976, tv_nsec=151353984}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
futex(0x7ffd548ca480, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffd548ca4d0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=170976, tv_nsec=351636217}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
futex(0x7ffd548ca480, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffd548ca4d0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=170976, tv_nsec=551851948}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
futex(0x7ffd548ca480, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffd548ca4d0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=170976, tv_nsec=752255501}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
futex(0x7ffd548ca480, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7ffd548ca4d0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=170976, tv_nsec=952530679}, FUTEX_BITSET_MATCH_ANY^Cstrace: Process 394 detached
<detached ...>
see swap on the host has 10 GB
root@h:~# swapon -s
Filename Type Size Used Priority
/dev/sda3 partition 524284 3084 -2
/rttswapfile file 10485756 0 -3
see swap in the container
root@sapp:~# swapon -s
root@sapp:~#
I see each container is not using swap
Tasks: 26 total, 1 running, 25 sleeping, 0 stopped, 0 zombie
%Cpu(s): 4.3 us, 0.0 sy, 0.0 ni, 95.7 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
MiB Mem : 6271.0 total, 5135.0 free, 502.3 used, 633.7 buff/cache
MiB Swap: 0.0 total, 0.0 free, 0.0 used. 5768.7 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
394 root 34 14 2849576 513200 151940 S 65.6 8.0 12:04.77 SAPP.daemon
In my old setup before learning LXD I used solus VM and had a vm for each daemon and I had 14 different daemons each with node.js app running side-by-side in each container.
I moved to LXD because I wanted to do more now I can’t even run a few!
These containers actually appear to slow each other down and step on each other.
When I had raspberry pis I could run 4 of these same daemons and 4 node.js apps per Pi.
The daemons are on config settings that are the lowest just as if they were on a Pi.