geoffk
February 8, 2025, 12:38pm
1
So I have an incus cluster w/ 2 Nodes that I’m using for lab purposes.
ubu-node-0 - 192.168.178.200
Unfortunately my home router only supports static routes with a single IP - so i figure the answer is to use VRRP via keepalived to provide OVN a single IP that floats between 2 hosts in the primary node falls over.
node 0
cat /etc/keepalived/keepalived.conf
vrrp_instance OVN_GATEWAY {
state MASTER
interface bond0
virtual_router_id 51
priority 101 # MASTER: higher priority
advert_int 1
authentication {
auth_type PASS
auth_pass mysecret
}
virtual_ipaddress {
192.168.178.250/24
}
}
node 1
cat /etc/keepalived/keepalived.conf
vrrp_instance OVN_GATEWAY {
state BACKUP
interface bond0
virtual_router_id 51
priority 100 # BACKUP: lower priority
advert_int 1
authentication {
auth_type PASS
auth_pass mysecret
}
virtual_ipaddress {
192.168.178.250/24
}
}
Whats not clear is how to give this to Incus such that it can manage creating new OVN networks via the northbound router config
so my bridge uplink on each node looks like this;
My cluster settings
root@ubu-node-1:/home# incus config show
config:
cluster.https_address: 192.168.178.201:8443
core.https_address: 192.168.178.201:8443
core.metrics_address: :8444
loki.api.url: http://10.116.146.199:3100
loki.instance: incus
loki.types: lifecycle,logging,network-acl
network.ovn.northbound_connection: tcp:192.168.178.200:6641,tcp:192.168.178.201:6641
bridge config
incusbr1
project: default
name: incusbr1
description: ''
type: bridge
config:
ipv4.address: 10.116.146.1/24
ipv4.dhcp: 'true'
ipv4.dhcp.ranges: 10.116.146.21-10.116.146.80
ipv4.nat: 'true'
ipv4.ovn.ranges: 10.116.146.2-10.116.146.20
ipv6.address: fd42:7824:c152:4341::1/64
ipv6.nat: 'true'
br-int
root@ubu-node-0:~# incus network info br-int
Name: br-int
MAC address: 52:fd:5d:4a:ea:52
MTU: 1442
State: down
Type: broadcast
Network usage:
Bytes received: 0B
Bytes sent: 0B
Packets received: 0
Packets sent: 0
root@ubu-node-0:~# incus network show br-int
config: {}
description: ""
name: br-int
type: bridge
used_by: []
managed: false
status: ""
locations: []
project: default
geoffk
February 9, 2025, 9:45am
2
hmmm. I see the problem now.
Incus just created a managed bridge on the cluster because I didn’t provide it one. Currently rebuilding to clean it up.
geoffk
February 9, 2025, 6:23pm
3
Ok so this appears to have worked, although it seems the desired effect of having a floating IP for a consistent ovn gateway didn’t work.
I guess I’ll grab the ovn uplink IP in the parent physical network and add that to the route.
doesn’t appear to be any need for keepalived.
For those interested, here are all my steps.
/bin/bash!
# Configure bonded NIC's in a bridge
/etc/netplan/01-netplan-br.yaml
node1
network:
version: 2
renderer: networkd
ethernets:
eno1:
dhcp4: false
enx1657c193e204:
dhcp4: false
bonds:
bond0:
interfaces:
- eno1
- enx1657c193e204
parameters:
mode: 802.3ad
lacp-rate: fast
transmit-hash-policy: layer3+4
mii-monitor-interval: 5s
bridges:
br-ovn:
interfaces: [bond0]
addresses: [192.168.178.200/24]
routes:
- to: default
via: 192.168.178.1
nameservers:
addresses: [192.168.178.1]
wifis: {}
node2
network:
version: 2
renderer: networkd
ethernets:
eno1:
dhcp4: false
enx00249b6ac4ca:
dhcp4: false
bonds:
bond0:
interfaces:
- eno1
- enx00249b6ac4ca
parameters:
mode: 802.3ad
lacp-rate: fast
transmit-hash-policy: layer3+4
mii-monitor-interval: 5s
bridges:
br-ovn:
interfaces: [bond0]
addresses: [192.168.178.201/24]
routes:
- to: default
via: 192.168.178.1
nameservers:
addresses: [192.168.178.1]
wifis: {}
# fix my usb eth randomly changing device names
node1
enx1657c193e204
ID_USB_SERIAL_SHORT=0000249B6AC4C9
vi /etc/systemd/network/10-enx1657c193e204.link
Property=ID_USB_SERIAL_SHORT=0000249B6AC4C9
[Link] Name=enx1657c193e204 MACAddress=00:24:9b:6a:c4:c9
node2
enx00249b6ac4ca
ID_USB_SERIAL_SHORT=0000249B6AC4CA
vi /etc/systemd/network/10-enx00249b6ac4ca.link
Property=ID_USB_SERIAL_SHORT=0000249B6AC4CA
[Link] Name=enx00249b6ac4ca MACAddress=00:24:9b:6a:c4:ca
# Keepalived - not required
# /etc/default/keepalived/keepalived.conf
# vrrp_instance OVN_Uplink {
# state MASTER
# interface br-ovn
# virtual_router_id 51
# priority 150 # Highest priority so Node1 becomes MASTER
# advert_int 1
# authentication {
# auth_type PASS
# auth_pass toosecret4you
# }
# virtual_ipaddress {
# 192.168.178.210/24
# }
# }
# vrrp_instance OVN_Uplink {
# state BACKUP
# interface br-ovn
# virtual_router_id 51
# priority 100 # Lower than Node1
# advert_int 1
# authentication {
# auth_type PASS
# auth_pass toosecret4you
# }
# virtual_ipaddress {
# 192.168.178.210/24
# }
# }
#configure ovn-central
/etc/default/ovn-central
node 1
OVN_CTL_OPTS=" \
--db-nb-addr=192.168.178.200 \
--db-nb-create-insecure-remote=yes \
--db-sb-addr=192.168.178.200 \
--db-sb-create-insecure-remote=yes \
--db-nb-cluster-local-addr=192.168.178.200 \
--db-sb-cluster-local-addr=192.168.178.200 \
--ovn-northd-nb-db=tcp:192.168.178.200:6641,tcp:192.168.178.201:6641 \
--ovn-northd-sb-db=tcp:192.168.178.200:6642,tcp:192.168.178.201:6642 \
--ovn-controller-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket' \
--ovn-northd-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket' \
--ovn-nb-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket' \
--ovn-sb-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket'"
node 2
OVN_CTL_OPTS=" \
--db-nb-addr=192.168.178.201 \
--db-nb-cluster-remote-addr=192.168.178.200 \
--db-nb-create-insecure-remote=yes \
--db-sb-addr=192.168.178.201 \
--db-sb-cluster-remote-addr=192.168.178.200 \
--db-sb-create-insecure-remote=yes \
--db-nb-cluster-local-addr=192.168.178.201 \
--db-sb-cluster-local-addr=192.168.178.201 \
--ovn-northd-nb-db=tcp:192.168.178.200:6641,tcp:192.168.178.201:6641 \
--ovn-northd-sb-db=tcp:192.168.178.200:6642,tcp:192.168.178.201:6642 \
--ovn-controller-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket' \
--ovn-northd-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket' \
--ovn-nb-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket' \
--ovn-sb-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket'"
#configure OVSwitches
node1
ovs-vsctl set open_vswitch . \
external_ids:ovn-remote=tcp:192.168.178.200:6642,tcp:192.168.178.201:6642 \
external_ids:ovn-encap-type=geneve \
external_ids:ovn-encap-ip=192.168.178.200
node2
ovs-vsctl set open_vswitch . \
external_ids:ovn-remote=tcp:192.168.178.200:6642,tcp:192.168.178.201:6642 \
external_ids:ovn-encap-type=geneve \
external_ids:ovn-encap-ip=192.168.178.201
# install incus
apt-get install incus incus-ui-canonical qemu-system nfs-common
# add hosts file entries
vi /etc/hosts
192.168.178.200 ubu-node-0
192.168.178.201 ubu-node-1
# Create your uplink network - note the ovn.ranges ip's should be reserved outside of your network's DHCP
incus network create UPLINK --type=physical parent=br-ovn --target=ubu-node-0
incus network create UPLINK --type=physical parent=br-ovn --target=ubu-node-1
incus network create UPLINK --type=physical \
ipv4.ovn.ranges=192.168.178.211-192.168.178.219 \
ipv4.gateway=192.168.178.1/24 \
dns.nameservers=192.168.178.1
# Configure incus' northbound controller connections
incus config set network.ovn.northbound_connection tcp:192.168.178.200:6641,tcp:192.168.178.201:6641
# Configure logging to loki
incus config set core.syslog_socket=true
incus config set loki.types=logging,lifecycle,network-acl
#iscsi w/ sanlock
apt-get install sanlock lvm2-lockd
#configure lvm
/etc/lvm/lvm.conf
global {
locking_type = 1
use_lvmlockd = 1
lvmlockd_lock_retries = 3
system_id_source = “lvmlocal”
}
/etc/lvm/lvmlocal.conf
node1
local {
system_id = “ubu-node-0”
host_id = 3
}
node 2
local {
system_id = “ubu-node-1”
host_id = 8
}
#edit your iscsi initiator names - configure upstream on iscsi target
/etc/iscsi/initiatorname.iscsi
node1
iqn.2004-10.ubu-node-0.com.ubuntu:01:1657c193e204
node2
iqn.2004-10.ubu-node-1.com.ubuntu:01:d2c7a0edf575
# discover your iscsi portal
iscsiadm -m discovery -t sendtargets -p 192.168.178.190:3260
# connect to it
iscsiadm -m node --login --target iqn.2004-04.com.qnap:tbs-464:iscsi.target-0.8a9ff4
# create the shared volume
node1
vgcreate --shared vgiscsi --locktype sanlock /dev/sdd
# enable locking for the nodes
node1
vgchange --lock-start vgiscsi
node2
vgscan
vgchange --lock-start vgiscsi
# configure the cluster storage
incus storage create iscsi_pool lvmcluster source=vgiscsi lvm.vg_name=vgiscsi --target=ubu-node-0
incus storage create iscsi_pool lvmcluster source=vgiscsi lvm.vg_name=vgiscsi --target=ubu-node-1
incus storage create iscsi_pool lvmcluster
#example re-create routed network - add a static route for 10.128.62.0/24 via 192.168.178.212 (volatile is there because incus made it and i re-made it)
incus network create ovn10-128-62 network=UPLINK --type ovn ipv4.address=10.128.62.1/24 ipv4.nat=true volatile.network.ipv4.address=192.168.178.212