Hey there ! 
No clue if this is the right place to post, but I would like to share a weird behavior that I am seeing using the provided Grafana Dashboard !
So the Incus server have 4 cores. When looking at the graphs, I am seeing some peaks at 10000% ???
How is that even possible ? Is there something in the multiplicator formula that I can tweak ?
Cheers !
To add a bit of details, the project consist of a single VM running inside qemu (of course) 
I just want to make sure that the metrics I am looking at are the right ones before investigating on the peaks
Can you show the incus config show --expanded of that single VM?
Also the output of incus query /1.0/metrics may be useful.
Sure thing !
Here you go ! (ok there is also a container instance but it is kind of futile in terms of resources utilization)
root@ipster:/home/derek# incus config show fw --expanded
architecture: x86_64
config:
boot.autostart: "true"
boot.autostart.delay: "90"
boot.autostart.priority: "10"
limits.cpu: "2"
limits.memory: 2GB
security.csm: "true"
security.secureboot: "false"
volatile.cloud-init.instance-id: 1d01da75-a4a8-4df6-9ba4-9a149e52678e
volatile.last_state.power: RUNNING
volatile.uuid: 333502b2-d94a-4336-a19c-7422a4ba2a78
volatile.uuid.generation: 333502b2-d94a-4336-a19c-7422a4ba2a78
volatile.vsock_id: "1637289788"
volatile.vtnet0.host_name: tap29b6dd15
volatile.vtnet0.hwaddr: 00:16:3e:f7:7a:ad
volatile.vtnet1.host_name: tap881d1441
volatile.vtnet1.hwaddr: 00:16:3e:8c:37:95
volatile.vtnet2.host_name: tap0797eb2f
volatile.vtnet2.hwaddr: 00:16:3e:7b:95:09
devices:
root:
path: /
pool: incus-sp
size: 40GiB
type: disk
vtnet0:
name: vtnet0
nictype: bridged
parent: br0
type: nic
vlan.tagged: 3,4,666
vtnet1:
name: vtnet1
nictype: bridged
parent: br-vlan2
type: nic
vtnet2:
name: vtnet2
nictype: bridged
parent: br-vlan53
type: nic
ephemeral: false
profiles:
- default
stateful: false
description: ipster - pfSense
root@ipster:/home/derek# incus query /1.0/metrics
# HELP incus_cpu_seconds_total The total number of CPU time used in seconds.
# TYPE incus_cpu_seconds_total counter
incus_cpu_seconds_total{cpu="0",mode="system",name="dns",project="default",type="container"} 138.429496
incus_cpu_seconds_total{cpu="0",mode="user",name="dns",project="default",type="container"} 90.227777
incus_cpu_seconds_total{cpu="0",mode="iowait",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="0",mode="irq",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="0",mode="idle",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="0",mode="nice",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="0",mode="softirq",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="0",mode="steal",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="0",mode="system",name="fw",project="default",type="virtual-machine"} 3393.31
incus_cpu_seconds_total{cpu="0",mode="user",name="fw",project="default",type="virtual-machine"} 1013.66
incus_cpu_seconds_total{cpu="1",mode="iowait",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="1",mode="irq",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="1",mode="idle",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="1",mode="nice",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="1",mode="softirq",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="1",mode="steal",name="fw",project="default",type="virtual-machine"} 0
incus_cpu_seconds_total{cpu="1",mode="system",name="fw",project="default",type="virtual-machine"} 2987.88
incus_cpu_seconds_total{cpu="1",mode="user",name="fw",project="default",type="virtual-machine"} 758.9
# HELP incus_cpu_effective_total The total number of effective CPUs.
# TYPE incus_cpu_effective_total gauge
incus_cpu_effective_total{name="dns",project="default",type="container"} 1
incus_cpu_effective_total{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_disk_read_bytes_total The total number of bytes read.
# TYPE incus_disk_read_bytes_total counter
incus_disk_read_bytes_total{device="dm-0",name="dns",project="default",type="container"} 36864
incus_disk_read_bytes_total{device="sda",name="dns",project="default",type="container"} 3.984128e+07
incus_disk_read_bytes_total{device="/machine/system.flash0",name="fw",project="default",type="virtual-machine"} 0
incus_disk_read_bytes_total{device="/machine/system.flash1",name="fw",project="default",type="virtual-machine"} 0
incus_disk_read_bytes_total{device="dev-incus_root",name="fw",project="default",type="virtual-machine"} 1.47302912e+08
# HELP incus_disk_reads_completed_total The total number of completed reads.
# TYPE incus_disk_reads_completed_total counter
incus_disk_reads_completed_total{device="dm-0",name="dns",project="default",type="container"} 4
incus_disk_reads_completed_total{device="sda",name="dns",project="default",type="container"} 1292
incus_disk_reads_completed_total{device="/machine/system.flash0",name="fw",project="default",type="virtual-machine"} 0
incus_disk_reads_completed_total{device="/machine/system.flash1",name="fw",project="default",type="virtual-machine"} 0
incus_disk_reads_completed_total{device="dev-incus_root",name="fw",project="default",type="virtual-machine"} 7878
# HELP incus_disk_written_bytes_total The total number of bytes written.
# TYPE incus_disk_written_bytes_total counter
incus_disk_written_bytes_total{device="dm-0",name="dns",project="default",type="container"} 0
incus_disk_written_bytes_total{device="sda",name="dns",project="default",type="container"} 0
incus_disk_written_bytes_total{device="/machine/system.flash0",name="fw",project="default",type="virtual-machine"} 0
incus_disk_written_bytes_total{device="/machine/system.flash1",name="fw",project="default",type="virtual-machine"} 0
incus_disk_written_bytes_total{device="dev-incus_root",name="fw",project="default",type="virtual-machine"} 7.6240289792e+10
# HELP incus_disk_writes_completed_total The total number of completed writes.
# TYPE incus_disk_writes_completed_total counter
incus_disk_writes_completed_total{device="dm-0",name="dns",project="default",type="container"} 0
incus_disk_writes_completed_total{device="sda",name="dns",project="default",type="container"} 0
incus_disk_writes_completed_total{device="/machine/system.flash0",name="fw",project="default",type="virtual-machine"} 0
incus_disk_writes_completed_total{device="/machine/system.flash1",name="fw",project="default",type="virtual-machine"} 0
incus_disk_writes_completed_total{device="dev-incus_root",name="fw",project="default",type="virtual-machine"} 7.937077e+06
# HELP incus_filesystem_avail_bytes The number of available space in bytes.
# TYPE incus_filesystem_avail_bytes gauge
incus_filesystem_avail_bytes{device="incus-sp/containers/dns",fstype="zfs",mountpoint="/",name="dns",project="default",type="container"} 2.1460025344e+10
# HELP incus_filesystem_free_bytes The number of free space in bytes.
# TYPE incus_filesystem_free_bytes gauge
incus_filesystem_free_bytes{device="incus-sp/containers/dns",fstype="zfs",mountpoint="/",name="dns",project="default",type="container"} 2.1460025344e+10
# HELP incus_filesystem_size_bytes The size of the filesystem in bytes.
# TYPE incus_filesystem_size_bytes gauge
incus_filesystem_size_bytes{device="incus-sp/containers/dns",fstype="zfs",mountpoint="/",name="dns",project="default",type="container"} 2.1636317184e+10
# HELP incus_memory_Active_anon_bytes The amount of anonymous memory on active LRU list.
# TYPE incus_memory_Active_anon_bytes gauge
incus_memory_Active_anon_bytes{name="dns",project="default",type="container"} 1.2763136e+07
incus_memory_Active_anon_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Active_file_bytes The amount of file-backed memory on active LRU list.
# TYPE incus_memory_Active_file_bytes gauge
incus_memory_Active_file_bytes{name="dns",project="default",type="container"} 3.5684352e+07
incus_memory_Active_file_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Active_bytes The amount of memory on active LRU list.
# TYPE incus_memory_Active_bytes gauge
incus_memory_Active_bytes{name="dns",project="default",type="container"} 4.8447488e+07
incus_memory_Active_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Cached_bytes The amount of cached memory.
# TYPE incus_memory_Cached_bytes gauge
incus_memory_Cached_bytes{name="dns",project="default",type="container"} 3.899392e+07
incus_memory_Cached_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Dirty_bytes The amount of memory waiting to get written back to the disk.
# TYPE incus_memory_Dirty_bytes gauge
incus_memory_Dirty_bytes{name="dns",project="default",type="container"} 0
incus_memory_Dirty_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_HugepagesFree_bytes The amount of free memory for hugetlb.
# TYPE incus_memory_HugepagesFree_bytes gauge
incus_memory_HugepagesFree_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_HugepagesTotal_bytes The amount of used memory for hugetlb.
# TYPE incus_memory_HugepagesTotal_bytes gauge
incus_memory_HugepagesTotal_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Inactive_anon_bytes The amount of anonymous memory on inactive LRU list.
# TYPE incus_memory_Inactive_anon_bytes gauge
incus_memory_Inactive_anon_bytes{name="dns",project="default",type="container"} 0
incus_memory_Inactive_anon_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Inactive_file_bytes The amount of file-backed memory on inactive LRU list.
# TYPE incus_memory_Inactive_file_bytes gauge
incus_memory_Inactive_file_bytes{name="dns",project="default",type="container"} 8192
incus_memory_Inactive_file_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Inactive_bytes The amount of memory on inactive LRU list.
# TYPE incus_memory_Inactive_bytes gauge
incus_memory_Inactive_bytes{name="dns",project="default",type="container"} 8192
incus_memory_Inactive_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Mapped_bytes The amount of mapped memory.
# TYPE incus_memory_Mapped_bytes gauge
incus_memory_Mapped_bytes{name="dns",project="default",type="container"} 1.476608e+07
incus_memory_Mapped_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_MemAvailable_bytes The amount of available memory.
# TYPE incus_memory_MemAvailable_bytes gauge
incus_memory_MemAvailable_bytes{name="dns",project="default",type="container"} 9.85628672e+08
incus_memory_MemAvailable_bytes{name="fw",project="default",type="virtual-machine"} 2.38788e+08
# HELP incus_memory_MemFree_bytes The amount of free memory.
# TYPE incus_memory_MemFree_bytes gauge
incus_memory_MemFree_bytes{name="dns",project="default",type="container"} 9.46634752e+08
incus_memory_MemFree_bytes{name="fw",project="default",type="virtual-machine"} 2.38788e+08
# HELP incus_memory_MemTotal_bytes The amount of used memory.
# TYPE incus_memory_MemTotal_bytes gauge
incus_memory_MemTotal_bytes{name="dns",project="default",type="container"} 9.9999744e+08
incus_memory_MemTotal_bytes{name="fw",project="default",type="virtual-machine"} 2e+09
# HELP incus_memory_RSS_bytes The amount of anonymous and swap cache memory.
# TYPE incus_memory_RSS_bytes gauge
incus_memory_RSS_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Shmem_bytes The amount of cached filesystem data that is swap-backed.
# TYPE incus_memory_Shmem_bytes gauge
incus_memory_Shmem_bytes{name="dns",project="default",type="container"} 3.301376e+06
incus_memory_Shmem_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Swap_bytes The amount of used swap memory.
# TYPE incus_memory_Swap_bytes gauge
incus_memory_Swap_bytes{name="dns",project="default",type="container"} 0
incus_memory_Swap_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Unevictable_bytes The amount of unevictable memory.
# TYPE incus_memory_Unevictable_bytes gauge
incus_memory_Unevictable_bytes{name="dns",project="default",type="container"} 0
incus_memory_Unevictable_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_Writeback_bytes The amount of memory queued for syncing to disk.
# TYPE incus_memory_Writeback_bytes gauge
incus_memory_Writeback_bytes{name="dns",project="default",type="container"} 0
incus_memory_Writeback_bytes{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_memory_OOM_kills_total The number of out of memory kills.
# TYPE incus_memory_OOM_kills_total counter
incus_memory_OOM_kills_total{name="dns",project="default",type="container"} 0
incus_memory_OOM_kills_total{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_network_receive_bytes_total The amount of received bytes on a given interface.
# TYPE incus_network_receive_bytes_total counter
incus_network_receive_bytes_total{device="lo",name="dns",project="default",type="container"} 733
incus_network_receive_bytes_total{device="eth0",name="dns",project="default",type="container"} 9.005841e+06
incus_network_receive_bytes_total{device="vtnet1",name="fw",project="default",type="virtual-machine"} 8.663982594e+09
incus_network_receive_bytes_total{device="vtnet2",name="fw",project="default",type="virtual-machine"} 4.308988e+06
incus_network_receive_bytes_total{device="vtnet0",name="fw",project="default",type="virtual-machine"} 3.5273815441e+10
# HELP incus_network_receive_drop_total The amount of received dropped bytes on a given interface.
# TYPE incus_network_receive_drop_total counter
incus_network_receive_drop_total{device="lo",name="dns",project="default",type="container"} 0
incus_network_receive_drop_total{device="eth0",name="dns",project="default",type="container"} 0
incus_network_receive_drop_total{device="vtnet1",name="fw",project="default",type="virtual-machine"} 0
incus_network_receive_drop_total{device="vtnet2",name="fw",project="default",type="virtual-machine"} 0
incus_network_receive_drop_total{device="vtnet0",name="fw",project="default",type="virtual-machine"} 0
# HELP incus_network_receive_errs_total The amount of received errors on a given interface.
# TYPE incus_network_receive_errs_total counter
incus_network_receive_errs_total{device="lo",name="dns",project="default",type="container"} 0
incus_network_receive_errs_total{device="eth0",name="dns",project="default",type="container"} 0
incus_network_receive_errs_total{device="vtnet1",name="fw",project="default",type="virtual-machine"} 0
incus_network_receive_errs_total{device="vtnet2",name="fw",project="default",type="virtual-machine"} 0
incus_network_receive_errs_total{device="vtnet0",name="fw",project="default",type="virtual-machine"} 0
# HELP incus_network_receive_packets_total The amount of received packets on a given interface.
# TYPE incus_network_receive_packets_total counter
incus_network_receive_packets_total{device="lo",name="dns",project="default",type="container"} 14
incus_network_receive_packets_total{device="eth0",name="dns",project="default",type="container"} 50226
incus_network_receive_packets_total{device="vtnet1",name="fw",project="default",type="virtual-machine"} 1.3316177e+07
incus_network_receive_packets_total{device="vtnet2",name="fw",project="default",type="virtual-machine"} 47516
incus_network_receive_packets_total{device="vtnet0",name="fw",project="default",type="virtual-machine"} 4.2268828e+07
# HELP incus_network_transmit_bytes_total The amount of transmitted bytes on a given interface.
# TYPE incus_network_transmit_bytes_total counter
incus_network_transmit_bytes_total{device="lo",name="dns",project="default",type="container"} 733
incus_network_transmit_bytes_total{device="eth0",name="dns",project="default",type="container"} 4.420085e+06
incus_network_transmit_bytes_total{device="vtnet1",name="fw",project="default",type="virtual-machine"} 1.8112523894e+10
incus_network_transmit_bytes_total{device="vtnet2",name="fw",project="default",type="virtual-machine"} 5.972462e+06
incus_network_transmit_bytes_total{device="vtnet0",name="fw",project="default",type="virtual-machine"} 2.5626399986e+10
# HELP incus_network_transmit_drop_total The amount of transmitted dropped bytes on a given interface.
# TYPE incus_network_transmit_drop_total counter
incus_network_transmit_drop_total{device="lo",name="dns",project="default",type="container"} 0
incus_network_transmit_drop_total{device="eth0",name="dns",project="default",type="container"} 0
incus_network_transmit_drop_total{device="vtnet1",name="fw",project="default",type="virtual-machine"} 0
incus_network_transmit_drop_total{device="vtnet2",name="fw",project="default",type="virtual-machine"} 0
incus_network_transmit_drop_total{device="vtnet0",name="fw",project="default",type="virtual-machine"} 0
# HELP incus_network_transmit_errs_total The amount of transmitted errors on a given interface.
# TYPE incus_network_transmit_errs_total counter
incus_network_transmit_errs_total{device="lo",name="dns",project="default",type="container"} 0
incus_network_transmit_errs_total{device="eth0",name="dns",project="default",type="container"} 0
incus_network_transmit_errs_total{device="vtnet1",name="fw",project="default",type="virtual-machine"} 0
incus_network_transmit_errs_total{device="vtnet2",name="fw",project="default",type="virtual-machine"} 0
incus_network_transmit_errs_total{device="vtnet0",name="fw",project="default",type="virtual-machine"} 0
# HELP incus_network_transmit_packets_total The amount of transmitted packets on a given interface.
# TYPE incus_network_transmit_packets_total counter
incus_network_transmit_packets_total{device="lo",name="dns",project="default",type="container"} 14
incus_network_transmit_packets_total{device="eth0",name="dns",project="default",type="container"} 48880
incus_network_transmit_packets_total{device="vtnet1",name="fw",project="default",type="virtual-machine"} 1.835876e+07
incus_network_transmit_packets_total{device="vtnet2",name="fw",project="default",type="virtual-machine"} 47905
incus_network_transmit_packets_total{device="vtnet0",name="fw",project="default",type="virtual-machine"} 3.6516162e+07
# HELP incus_procs_total The number of running processes.
# TYPE incus_procs_total gauge
incus_procs_total{name="dns",project="default",type="container"} 41
incus_procs_total{name="fw",project="default",type="virtual-machine"} 0
# HELP incus_operations_total The number of running operations
# TYPE incus_operations_total counter
incus_operations_total 0
# HELP incus_warnings_total The number of active warnings.
# TYPE incus_warnings_total counter
incus_warnings_total 0
# HELP incus_uptime_seconds The daemon uptime in seconds.
# TYPE incus_uptime_seconds counter
incus_uptime_seconds 77660.532869604
# HELP incus_go_goroutines Number of goroutines that currently exist.
# TYPE incus_go_goroutines gauge
incus_go_goroutines 32
# HELP incus_go_alloc_bytes Number of bytes allocated and still in use.
# TYPE incus_go_alloc_bytes gauge
incus_go_alloc_bytes 1.9422336e+07
# HELP incus_go_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE incus_go_alloc_bytes_total counter
incus_go_alloc_bytes_total 1.0603338888e+10
# HELP incus_go_sys_bytes Number of bytes obtained from system.
# TYPE incus_go_sys_bytes gauge
incus_go_sys_bytes 3.7154093e+07
# HELP incus_go_lookups_total Total number of pointer lookups.
# TYPE incus_go_lookups_total counter
incus_go_lookups_total 0
# HELP incus_go_mallocs_total Total number of mallocs.
# TYPE incus_go_mallocs_total counter
incus_go_mallocs_total 6.6316192e+07
# HELP incus_go_frees_total Total number of frees.
# TYPE incus_go_frees_total counter
incus_go_frees_total 6.6206029e+07
# HELP incus_go_heap_alloc_bytes Number of heap bytes allocated and still in use.
# TYPE incus_go_heap_alloc_bytes gauge
incus_go_heap_alloc_bytes 1.9422336e+07
# HELP incus_go_heap_sys_bytes Number of heap bytes obtained from system.
# TYPE incus_go_heap_sys_bytes gauge
incus_go_heap_sys_bytes 2.8311552e+07
# HELP incus_go_heap_idle_bytes Number of heap bytes waiting to be used.
# TYPE incus_go_heap_idle_bytes gauge
incus_go_heap_idle_bytes 6.332416e+06
# HELP incus_go_heap_inuse_bytes Number of heap bytes that are in use.
# TYPE incus_go_heap_inuse_bytes gauge
incus_go_heap_inuse_bytes 2.1979136e+07
# HELP incus_go_heap_released_bytes Number of heap bytes released to OS.
# TYPE incus_go_heap_released_bytes gauge
incus_go_heap_released_bytes 3.596288e+06
# HELP incus_go_heap_objects Number of allocated objects.
# TYPE incus_go_heap_objects gauge
incus_go_heap_objects 110163
# HELP incus_go_stack_inuse_bytes Number of bytes in use by the stack allocator.
# TYPE incus_go_stack_inuse_bytes gauge
incus_go_stack_inuse_bytes 1.048576e+06
# HELP incus_go_stack_sys_bytes Number of bytes obtained from system for stack allocator.
# TYPE incus_go_stack_sys_bytes gauge
incus_go_stack_sys_bytes 1.048576e+06
# HELP incus_go_mspan_inuse_bytes Number of bytes in use by mspan structures.
# TYPE incus_go_mspan_inuse_bytes gauge
incus_go_mspan_inuse_bytes 245784
# HELP incus_go_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
# TYPE incus_go_mspan_sys_bytes gauge
incus_go_mspan_sys_bytes 293328
# HELP incus_go_mcache_inuse_bytes Number of bytes in use by mcache structures.
# TYPE incus_go_mcache_inuse_bytes gauge
incus_go_mcache_inuse_bytes 4800
# HELP incus_go_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
# TYPE incus_go_mcache_sys_bytes gauge
incus_go_mcache_sys_bytes 15600
# HELP incus_go_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
# TYPE incus_go_buck_hash_sys_bytes gauge
incus_go_buck_hash_sys_bytes 1.951887e+06
# HELP incus_go_gc_sys_bytes Number of bytes used for garbage collection system metadata.
# TYPE incus_go_gc_sys_bytes gauge
incus_go_gc_sys_bytes 4.85772e+06
# HELP incus_go_other_sys_bytes Number of bytes used for other system allocations.
# TYPE incus_go_other_sys_bytes gauge
incus_go_other_sys_bytes 675430
# HELP incus_go_next_gc_bytes Number of heap bytes when next garbage collection will take place.
# TYPE incus_go_next_gc_bytes gauge
incus_go_next_gc_bytes 2.2153608e+07
# EOF
One of the “weird” thing that I am noticing is
incus_cpu_effective_total{name="fw",project="default",type="virtual-machine"} 0
After giving another shot at the documentation, it seems like VMs may require an agent to properly inform Incus metrics.
From what I can read, the agent is linux compatible only. Does that means that since my VM is FreeBSD, the agent won’t work ?
Yeah, when no VM agent, it will pull some limited data from QEMU itself on the host which may not be ideal.