Projekt KVM Cluster

IDG Cluster

The IDG cluster is a 2 node heartbeat/drbd cluster running on sles It is an active/active configuration for 2 service stacks with a db2 database on top of each.

Testcluster

There is a 3 node test cluster showing what heartbeat and drbd can do. It was originally setup as suse 9.3 vmware images. I currently transfer this to kvm since vmware on linux is a pain (kernel updates)

vmware files

suse-9.3-c1:
insgesamt 3,1G
-rw------- 1 joachim users 8,5K 13. Dez 2005  nvram
-rw------- 1 joachim users 8,5K 27. Nov 2005  nvram.sav
-rw------- 1 joachim users 1,4G 27. Nov 2005  SUSE-Linux-9.3-s001.vmdk
-rw------- 1 joachim users 1,4G 27. Nov 2005  SUSE-Linux-9.3-s002.vmdk
-rw------- 1 joachim users  37M 27. Nov 2005  SUSE-Linux-9.3-s003.vmdk
-rw------- 1 joachim users 277M 27. Nov 2005  SUSE-Linux-9.3-s004.vmdk
-rw------- 1 joachim users 9,2M 27. Nov 2005  SUSE-Linux-9.3-s005.vmdk
-rw------- 1 joachim users  64K 27. Nov 2005  SUSE-Linux-9.3-s006.vmdk
-rw------- 1 joachim users  586 27. Nov 2005  SUSE-Linux-9.3.vmdk
-rwxr-xr-x 1 joachim users 2,7K 26. Feb 07:57 SUSE-Linux-9.3.vmx
-rw-r--r-- 1 joachim users  269 26. Feb 07:57 SUSE-Linux-9.3.vmxf
-rw------- 1 joachim users 2,6K 13. Dez 2005  SUSE-Linux-9.3.vmx.sav
-rw-r--r-- 1 joachim users  30K 13. Dez 2005  vmware.log

vmware image config

$ cat suse-9.3-c1/SUSE-Linux-9.3.vmx
#!/usr/bin/vmware
.encoding = "UTF-8"
config.version = "7"
virtualHW.version = "3"
scsi0.present = "TRUE"
memsize = "384"
scsi0:0.present = "TRUE"
scsi0:0.fileName = "SUSE-Linux-9.3.vmdk"
ide1:0.present = "TRUE"
ide1:0.fileName = "/dev/dvd"
ide1:0.deviceType = "atapi-cdrom"
floppy0.startConnected = "FALSE"
floppy0.fileName = "/dev/fd0"
Ethernet0.present = "TRUE"
Ethernet0.connectionType = "custom"
displayName = "SUSE Linux 9.3 Cluster Node 1"
guestOS = "suse"
priority.grabbed = "normal"
priority.ungrabbed = "normal"
powerType.powerOff = "soft"
powerType.powerOn = "soft"
powerType.suspend = "soft"
powerType.reset = "soft"

Ethernet0.vnet = "/dev/vmnet1"
Ethernet1.present = "TRUE"
Ethernet1.connectionType = "custom"
Ethernet1.vnet = "/dev/vmnet2"
Ethernet2.present = "TRUE"
Ethernet2.connectionType = "custom"
Ethernet2.vnet = "/dev/vmnet3"
Ethernet3.present = "TRUE"
Ethernet3.connectionType = "custom"
Ethernet3.vnet = "/dev/vmnet4"
usb.present = "FALSE"

#serial0.present = "TRUE"
#serial0.fileType = "pipe"
#serial0.fileName = "/tmp/vmware-suse-9.3-cluster-serial-1"
#serial1.present = "TRUE"
#serial1.fileType = "pipe"
#serial1.fileName = "/tmp/vmware-suse-9.3-cluster-serial-3"
#serial1.pipe.endPoint = "client"

undopoint.action = "prompt"

annotation = "Minimal Suse 9.3 pro for Cluster Nodes.|0AYou date: 2005-10-19|0AHOWTO Clone:|0ACopy everything to new directory|0Acustomize SUSE-Linux-9.3.vmx:|0A  change Display Name|0A  change Annotation|0A  change ethernetX.address to unique values|0Astart vm|0A  change /etc/HOSTNAME|0A  change /etc/hosts if host ips change|0A  change /etc/sysconfig/network/ifcfg-eth-X|0A"

uuid.location = "56 4d e7 5d 52 90 b5 59-b1 00 58 b7 15 2f 14 1a"
uuid.bios = "56 4d e7 5d 52 90 b5 59-b1 00 58 b7 15 2f 14 1a"

#                    vmwareid cl nd if
ethernet0.address = "00:50:56:01:01:00"
ethernet1.address = "00:50:56:01:01:01"
ethernet2.address = "00:50:56:01:01:02"
ethernet3.address = "00:50:56:01:01:03"

tools.remindInstall = "FALSE"

undopoints.seqNum = "0"
scsi0:0.mode = "undoable"
undopoint.restoreFromCheckpoint = "FALSE"
undopoint.checkpointedOnline = "FALSE"

scsi0:0.redo = "./SUSE-Linux-9.3.vmdk.REDO_a2m6ht"
ide1:0.startConnected = "FALSE"
tools.syncTime = "TRUE"

undopoint.protected = "FALSE"
gui.restricted = "FALSE"

Ethernet0.addressType = "static"
Ethernet1.addressType = "static"
Ethernet2.addressType = "static"
Ethernet3.addressType = "static"

isolation.tools.hgfs.disable = "TRUE"

extendedConfigFile = "SUSE-Linux-9.3.vmxf"
scsi0:0.deviceType = "scsi-hardDisk"
virtualHW.productCompatibility = "hosted"
sound.present = "FALSE"

Convert the image from vmdk to raw format

Easy as that

qemu-img convert SUSE-Linux-9.3.vmdk SUSE-Linux-9.3.raw

Setup host networking

• Enable kvm host via yast module "Install Hypervisor and tools"
  • Enable bridging with the public interface of the host (here eth0 -> br0)
• for each vmware EthernetX configuration create a virtual network with virt-manager/connection details
  • vmnet1 -> virbr1 (same subnet and type, here 192.168.100.0/24 NAT)
  • vmnet2 -> virbr2 (same subnet and type, here 192.168.101.0/24 isolated)
  • vmnet3 -> virbr3 (same subnet and type, here 192.168.102.0/24 isolated)
  • vmnet4 -> virbr4 (same subnet and type, here 192.168.103.0/24 isolated)
• enable access to bridges for qemu in /etc/qemu/bridge.conf (e.g. ad line allow all)

Make kvm startup script

#!/bin/bash

# start cluster node 1

pass=somepw
port=someport

if ! ps auxwww | grep -q "[5]64de75d-5290-b559-b100-58b7152f141a"; then
       qemu-kvm \
       -drive file=SUSE-Linux-9.3.raw,if=ide \
       -uuid 564de75d-5290-b559-b100-58b7152f141a \
       -m 384 \
       -rtc base=localtime \
       -soundhw ac97 \
       -usb -usbdevice tablet \
       -net nic,vlan=0,model=e1000,macaddr=00:50:56:01:01:00 \
       -net bridge,vlan=0,br=virbr1 \
       -net nic,vlan=1,model=e1000,macaddr=00:50:56:01:01:01 \
       -net bridge,vlan=1,br=virbr2 \
       -net nic,vlan=2,model=e1000,macaddr=00:50:56:01:01:02 \
       -net bridge,vlan=2,br=virbr3 \
       -net nic,vlan=3,model=e1000,macaddr=00:50:56:01:01:03 \
       -net bridge,vlan=3,br=virbr4 \
       -vga qxl \
       -enable-kvm \
       -device virtio-serial \
       -chardev spicevmc,id=vdagent,name=vdagent \
       -device virtserialport,chardev=vdagent,name=com.redhat.spice.0 \
       -spice port=$port,addr=localhost,password=$pass \
       -name suse-9.3-c1 &
fi
exec spicec -h localhost -p $port -t 'Suse-9.3-C1' -w $pass

• The uuid (grep and -uuid) comes from the vmx file
• the macaddr also comes from the vmx file, this way the guest network will just work.

The drive interface is ide, because the scsi drivers of the guests are not compatible with kvm scsi controller. That means, references to the disk sda need to be changed to hda (on grub screen when guest boots and later in /boot/grub/menu.lst and /etc/fstab)

Image config

• edit /etc/grub/menu.lst
  • change sda to hda everywhere
  • change vga=0 if another number is used and spice looks awful
• edit /etc/fstab
  • change sda to hda everywhere
• edit /etc/resolv.conf
  • use the hosts nameserver
• edit /etc/ntp.conf
  • use a valid ntp server if necessary (e.g. fritz.box)
• collect syslog warnings on host
  • edit /etc/syslog-ng/syslog-ng.conf.in: add udp(...);

destination warn { file("/var/log/warn" fsync(yes)); udp("192.168.1.1" port(514)); };

• • use the new config

SuSEconfig 
rcsyslog restart

• • on the target host, using rsyslog
    • enable udb in /etc/rsyslog.d/remote.conf
    • add vm bridges virbr1-4 to internal network in /etc/sysconfig/SuSEfirewall2
    • restart firewall and syslog

Update heartbeat =

Source tar ball

wget http://pkgs.fedoraproject.org/repo/pkgs/heartbeat/heartbeat-2.1.3.tar.gz/bca53530a3802f7677772323047405cd/heartbeat-2.1.3.tar.gz

Package source

http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/

Packages needed for compile

wget ftp://bo.mirror.garr.it/pub/1/suse/discontinued/i386/9.3/suse/i586/python-devel-2.4-14.i586.rpm
wget ftp://bo.mirror.garr.it/pub/1/suse/discontinued/i386/9.3/suse/i586/python-tk-2.4-14.i586.rpm
wget ftp://bo.mirror.garr.it/pub/1/suse/discontinued/i386/9.3/suse/i586/blt-2.4z-205.i586.rpm
wget ftp://bo.mirror.garr.it/pub/1/suse/ftp.suse.com/projects/tcl/8.4.12/9.2-i386/tk-8.4.12-2.1.i586.rpm
wget ftp://bo.mirror.garr.it/pub/1/suse/ftp.suse.com/projects/tcl/8.4.12/9.2-i386/tcl-8.4.12-5.1.i586.rpm
wget ftp://bo.mirror.garr.it/pub/1/suse/discontinued/i386/9.3/suse/i586/gdbm-devel-1.8.3-230.i586.rpm
wget http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/gnutls-1.2.0-3.i586.rpm
wget http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/gnutls-devel-1.2.0-3.i586.rpm
wget http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/lzo-1.08-107.i586.rpm
wget http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/libopencdk-0.5.5-3.i586.rpm
wget http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/libgcrypt-1.2.1-3.i586.rpm
wget http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/libgcrypt-devel-1.2.1-3.i586.rpm
wget http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/libgpg-error-1.0-3.i586.rpm
wget http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/libgpg-error-devel-1.0-3.i586.rpm
wget http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/e2fsprogs-devel-1.36-5.i586.rpm
wget http://ftp.hosteurope.de/mirror/ftp.suse.com/pub/suse/discontinued/i386/9.3/suse/i586/pam-devel-0.78-8.i586.rpm
wget ftp://bo.mirror.garr.it/pub/1/suse/discontinued/i386/9.3/suse/i586/readline-devel-5.0-7.2.i586.rpm
wget ftp://bo.mirror.garr.it/pub/1/suse/discontinued/i386/9.3/suse/i586/readline-5.0-7.2.i586.rpm

Make rpms

tar xzf heartbeat-2.1.3.tar.gz
cd heartbeat-2.1.3
./ConfigureMe package

Todo

• fix broken resource scripts (db2)