Switch from raw images to qcow2 format

This allows to use minimal copy-on-write clones of the base image as
guest images, which in turn saves a lot of disk space.

diff --git a/testing/config/kvm/alice.xml b/testing/config/kvm/alice.xml
index 275aeceb41d7040108a0e845f690fa41ab4626b8..16a97478644ab2b6c58fba2b22630d034bf8f343 100644 (file)
--- a/testing/config/kvm/alice.xml
+++ b/testing/config/kvm/alice.xml
@@ -7,7 +7,7 @@
   <os>
     <type arch='x86_64' machine='pc-1.1'>hvm</type>
        <kernel>/var/run/kvm-swan-kernel</kernel>
-    <cmdline>root=/dev/vda loglevel=1</cmdline>
+    <cmdline>root=/dev/vda1 loglevel=1</cmdline>
     <boot dev='hd'/>
   </os>
   <features>
@@ -22,8 +22,8 @@
   <devices>
     <emulator>/usr/bin/kvm</emulator>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='raw' cache='writethrough'/>
-      <source file='/var/lib/libvirt/images/alice.img'/>
+      <driver name='qemu' type='qcow2' cache='writethrough'/>
+      <source file='/var/lib/libvirt/images/alice.qcow2'/>
       <target dev='vda' bus='virtio'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
     </disk>

diff --git a/testing/config/kvm/bob.xml b/testing/config/kvm/bob.xml
index 4ebeca1a88fd445e62908932e23802a3725cf77c..723825bbbb239491b0705f900a22c75ca1e0acc0 100644 (file)
--- a/testing/config/kvm/bob.xml
+++ b/testing/config/kvm/bob.xml
@@ -7,7 +7,7 @@
   <os>
     <type arch='x86_64' machine='pc-1.1'>hvm</type>
        <kernel>/var/run/kvm-swan-kernel</kernel>
-    <cmdline>root=/dev/vda loglevel=1</cmdline>
+    <cmdline>root=/dev/vda1 loglevel=1</cmdline>
     <boot dev='hd'/>
   </os>
   <features>
@@ -22,8 +22,8 @@
   <devices>
     <emulator>/usr/bin/kvm</emulator>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='raw' cache='writethrough'/>
-      <source file='/var/lib/libvirt/images/bob.img'/>
+      <driver name='qemu' type='qcow2' cache='writethrough'/>
+      <source file='/var/lib/libvirt/images/bob.qcow2'/>
       <target dev='vda' bus='virtio'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
     </disk>

diff --git a/testing/config/kvm/carol.xml b/testing/config/kvm/carol.xml
index f255b2211140a3d5016900a364bea2fc7120f13a..ec018611d744f3dcb5c2e3c012e27fafd357caf9 100644 (file)
--- a/testing/config/kvm/carol.xml
+++ b/testing/config/kvm/carol.xml
@@ -7,7 +7,7 @@
   <os>
     <type arch='x86_64' machine='pc-1.1'>hvm</type>
        <kernel>/var/run/kvm-swan-kernel</kernel>
-    <cmdline>root=/dev/vda loglevel=1</cmdline>
+    <cmdline>root=/dev/vda1 loglevel=1</cmdline>
     <boot dev='hd'/>
   </os>
   <features>
@@ -22,8 +22,8 @@
   <devices>
     <emulator>/usr/bin/kvm</emulator>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='raw' cache='writethrough'/>
-      <source file='/var/lib/libvirt/images/carol.img'/>
+      <driver name='qemu' type='qcow2' cache='writethrough'/>
+      <source file='/var/lib/libvirt/images/carol.qcow2'/>
       <target dev='vda' bus='virtio'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
     </disk>

diff --git a/testing/config/kvm/dave.xml b/testing/config/kvm/dave.xml
index 9f1774c9d703c0bfefe248cc37491bb30b1340d7..019191842f5d4f37fc1ad8cf02ccafc359d182f1 100644 (file)
--- a/testing/config/kvm/dave.xml
+++ b/testing/config/kvm/dave.xml
@@ -7,7 +7,7 @@
   <os>
     <type arch='x86_64' machine='pc-1.1'>hvm</type>
        <kernel>/var/run/kvm-swan-kernel</kernel>
-    <cmdline>root=/dev/vda loglevel=1</cmdline>
+    <cmdline>root=/dev/vda1 loglevel=1</cmdline>
     <boot dev='hd'/>
   </os>
   <features>
@@ -22,8 +22,8 @@
   <devices>
     <emulator>/usr/bin/kvm</emulator>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='raw' cache='writethrough'/>
-      <source file='/var/lib/libvirt/images/dave.img'/>
+      <driver name='qemu' type='qcow2' cache='writethrough'/>
+      <source file='/var/lib/libvirt/images/dave.qcow2'/>
       <target dev='vda' bus='virtio'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
     </disk>

diff --git a/testing/config/kvm/moon.xml b/testing/config/kvm/moon.xml
index 2c50b30b5660a09620f40711a24133c682aa57f6..8e5769de2bc9aaeaa8c57965202a68e6f55f733b 100644 (file)
--- a/testing/config/kvm/moon.xml
+++ b/testing/config/kvm/moon.xml
@@ -7,7 +7,7 @@
   <os>
     <type arch='x86_64' machine='pc-1.1'>hvm</type>
        <kernel>/var/run/kvm-swan-kernel</kernel>
-    <cmdline>root=/dev/vda loglevel=1</cmdline>
+    <cmdline>root=/dev/vda1 loglevel=1</cmdline>
     <boot dev='hd'/>
   </os>
   <features>
@@ -22,8 +22,8 @@
   <devices>
     <emulator>/usr/bin/kvm</emulator>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='raw' cache='writethrough'/>
-      <source file='/var/lib/libvirt/images/moon.img'/>
+      <driver name='qemu' type='qcow2' cache='writethrough'/>
+      <source file='/var/lib/libvirt/images/moon.qcow2'/>
       <target dev='vda' bus='virtio'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
     </disk>

diff --git a/testing/config/kvm/sun.xml b/testing/config/kvm/sun.xml
index 70efc770739276e7c20161167c5e4f65b5758384..01fed44c680cc9c1fdfd47e9aa86b3f11cb67956 100644 (file)
--- a/testing/config/kvm/sun.xml
+++ b/testing/config/kvm/sun.xml
@@ -7,7 +7,7 @@
   <os>
     <type arch='x86_64' machine='pc-1.1'>hvm</type>
        <kernel>/var/run/kvm-swan-kernel</kernel>
-    <cmdline>root=/dev/vda loglevel=1</cmdline>
+    <cmdline>root=/dev/vda1 loglevel=1</cmdline>
     <boot dev='hd'/>
   </os>
   <features>
@@ -22,8 +22,8 @@
   <devices>
     <emulator>/usr/bin/kvm</emulator>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='raw' cache='writethrough'/>
-      <source file='/var/lib/libvirt/images/sun.img'/>
+      <driver name='qemu' type='qcow2' cache='writethrough'/>
+      <source file='/var/lib/libvirt/images/sun.qcow2'/>
       <target dev='vda' bus='virtio'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
     </disk>

diff --git a/testing/config/kvm/venus.xml b/testing/config/kvm/venus.xml
index 8771fdf9ff18f7c32b8a342a0ae16e1132d66895..3002e23c8690acc2eab9c981df6ef6ca1775c37a 100644 (file)
--- a/testing/config/kvm/venus.xml
+++ b/testing/config/kvm/venus.xml
@@ -7,7 +7,7 @@
   <os>
     <type arch='x86_64' machine='pc-1.1'>hvm</type>
        <kernel>/var/run/kvm-swan-kernel</kernel>
-    <cmdline>root=/dev/vda loglevel=1</cmdline>
+    <cmdline>root=/dev/vda1 loglevel=1</cmdline>
     <boot dev='hd'/>
   </os>
   <features>
@@ -22,8 +22,8 @@
   <devices>
     <emulator>/usr/bin/kvm</emulator>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='raw' cache='writethrough'/>
-      <source file='/var/lib/libvirt/images/venus.img'/>
+      <driver name='qemu' type='qcow2' cache='writethrough'/>
+      <source file='/var/lib/libvirt/images/venus.qcow2'/>
       <target dev='vda' bus='virtio'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
     </disk>

diff --git a/testing/config/kvm/winnetou.xml b/testing/config/kvm/winnetou.xml
index 15e23ade8be1762dd462d62fbad475f89ca5c410..3f018f1a8c727996c0a17d173ce43b2ce9546a87 100644 (file)
--- a/testing/config/kvm/winnetou.xml
+++ b/testing/config/kvm/winnetou.xml
@@ -7,7 +7,7 @@
   <os>
     <type arch='x86_64' machine='pc-1.1'>hvm</type>
        <kernel>/var/run/kvm-swan-kernel</kernel>
-    <cmdline>root=/dev/vda loglevel=1</cmdline>
+    <cmdline>root=/dev/vda1 loglevel=1</cmdline>
     <boot dev='hd'/>
   </os>
   <features>
@@ -22,8 +22,8 @@
   <devices>
     <emulator>/usr/bin/kvm</emulator>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='raw' cache='writethrough'/>
-      <source file='/var/lib/libvirt/images/winnetou.img'/>
+      <driver name='qemu' type='qcow2' cache='writethrough'/>
+      <source file='/var/lib/libvirt/images/winnetou.qcow2'/>
       <target dev='vda' bus='virtio'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
     </disk>

diff --git a/testing/scripts/build-baseimage b/testing/scripts/build-baseimage
index b8f7092163949962b4b3eb25864b02f39c705baf..c73c454300e42562c4ac65db913b89744c7e80ee 100755 (executable)
--- a/testing/scripts/build-baseimage
+++ b/testing/scripts/build-baseimage
@@ -10,6 +10,7 @@ echo "Building base image"
 EXTRAS=build-essential,gperf,libgmp-dev,libldap2-dev,libcurl4-openssl-dev,libxml2-dev,libtspi-dev,libsqlite3-dev,openssh-server,tcpdump,psmisc,openssl,vim,sqlite3,conntrack,gdb,cmake,libxerces-c2-dev,libltdl-dev,liblog4cxx10-dev,libboost-thread-dev,libboost-system-dev,git-core
 SERVICES="isc-dhcp-server apache2 slapd"
 PACKAGES=$EXTRAS,${SERVICES// /,}
+
 CACHEDIR=$BUILDDIR/cache
 APTCACHE=$LOOPDIR/var/cache/apt/archives
 
@@ -19,28 +20,45 @@ mkdir -p $CACHEDIR
 rm -f $ROOTFS
 
 echo "`date`, building $ROOTFS" >>$LOGFILE
+
+load_qemu_nbd
+
 log_action "Creating image $ROOTFS"
-execute "dd if=/dev/null of=$ROOTFS bs=1M seek=$ROOTFSSIZE count=1"
+execute "qemu-img create -f qcow2 $ROOTFS ${ROOTFSSIZE}M"
+
+log_action "Connecting image to NBD device $NBDEV"
+execute "qemu-nbd -c $NBDEV $ROOTFS"
+sync
+
+log_action "Partitioning disk"
+sfdisk /dev/nbd0 -D -uM >>$LOGFILE 2>&1 << EOF
+;
+EOF
+if [ $? != 0 ]
+then
+       log_status 1
+       exit 1
+else
+       log_status 0
+fi
 
 log_action "Creating ext3 filesystem"
-execute "mkfs.ext3 -F $ROOTFS"
+execute "mkfs.ext3 $NBDPARTITION"
 
-log_action "Mounting image to $LOOPDIR"
-execute "mount -o loop $ROOTFS $LOOPDIR"
+log_action "Mounting $NBDPARTITION to $LOOPDIR"
+execute "mount $NBDPARTITION $LOOPDIR"
 
-log_action "Mounting cache to $CACHEDIR"
+log_action "Using $CACHEDIR as archive for apt"
 mkdir -p $APTCACHE
 execute "mount -o bind $CACHEDIR $APTCACHE"
 
-do_on_exit umount -l $APTCACHE
-do_on_exit umount -l $LOOPDIR
+do_on_exit graceful_umount $APTCACHE
+do_on_exit graceful_umount $LOOPDIR
+do_on_exit qemu-nbd -d $NBDEV
 
 log_action "Running debootstrap ($ROOTFSSUITE, $ROOTFSARCH)"
 execute "debootstrap --arch=$ROOTFSARCH --include=$PACKAGES $ROOTFSSUITE $LOOPDIR $ROOTFSMIRROR"
 
-log_action "Disabling root password"
-execute_chroot "passwd -d root"
-
 for service in $SERVICES
 do
        log_action "Stopping service $service"
@@ -48,3 +66,6 @@ do
        log_action "Disabling service $service"
        execute_chroot "update-rc.d -f $service remove"
 done
+
+log_action "Disabling root password"
+execute_chroot "passwd -d root"

diff --git a/testing/scripts/build-umlhostfs b/testing/scripts/build-umlhostfs
index 9c419ac9b05eb9d009aa55efa933d080380affb2..1028ef26657eb82311bd8f6e50aeceac2a6888d5 100755 (executable)
--- a/testing/scripts/build-umlhostfs
+++ b/testing/scripts/build-umlhostfs
@@ -1,5 +1,5 @@
 #!/bin/bash
-# create UML host file systems
+# create specific guest images
 #
 # Copyright (C) 2004  Eric Marchionni, Patrik Rayo
 # Zuercher Hochschule Winterthur
@@ -20,12 +20,14 @@ echo "Creating guest images"
 
 [ `id -u` -eq 0 ] || die "You must be root to run $0"
 
-BASE=$BUILDDIR/base.img
+BASE=$BUILDDIR/base.qcow2
 HOSTSDIR=$PWD/hosts
 
 [ -f $BASE ] || die "Base image $BASE not found"
 [ -f $HOSTDIR ] || die "Hosts directory $HOSTSDIR not found"
 
+load_qemu_nbd
+
 mkdir -p $BUILDDIR
 mkdir -p $LOOPDIR
 
@@ -41,8 +43,10 @@ do_on_exit umount $LOOPDIR
 for host in $HOSTS
 do
        log_action "Creating guest image for $host"
-       execute "cp $BASE $ROOTFSDIR/$host.img" 0
-       execute "mount -o loop $ROOTFSDIR/$host.img $LOOPDIR" 0
+       execute "qemu-img create -b $BASE -f qcow2 $ROOTFSDIR/$host.qcow2" 0
+       execute "qemu-nbd -c $NBDEV $ROOTFSDIR/$host.qcow2" 0
+       sync
+       execute "mount $NBDPARTITION $LOOPDIR" 0
        execute "cp -rf $HOSTSDIR/${host}/etc $LOOPDIR" 0
        execute "cp -rf $HOSTSDIR/default/etc $LOOPDIR" 0
        if [ "$host" = "winnetou" ]
@@ -58,6 +62,7 @@ do
                execute_chroot "chown -R openldap:openldap /var/lib/ldap" 0
        fi
        sync
+       execute "umount $LOOPDIR" 0
+       execute "qemu-nbd -d $NBDEV" 0
        log_status 0
-       umount $LOOPDIR
 done

diff --git a/testing/scripts/build-umlrootfs b/testing/scripts/build-umlrootfs
index cf11844ce9898695926e80996af7353dd19bee9c..c67e4d557710e918ebc2cde5c3e1f95279493049 100755 (executable)
--- a/testing/scripts/build-umlrootfs
+++ b/testing/scripts/build-umlrootfs
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Create UML root filesystem
+# Create guest root image
 #
 # Copyright (C) 2004  Eric Marchionni, Patrik Rayo
 # Zuercher Hochschule Winterthur
@@ -22,17 +22,24 @@ echo "Building root image"
 
 [ -f "$ROOTFS" ] || die "Root image $ROOTFS not found"
 
+load_qemu_nbd
+
 mkdir -p $ROOTFSDIR
 mkdir -p $LOOPDIR
 mkdir -p $ROOTFSCOMPILEDIR
 cd $ROOTFSDIR
 
-BASE=$BUILDDIR/base.img
+BASE=$BUILDDIR/base.qcow2
 log_action "Creating $BASE"
 execute "cp $ROOTFS $BASE"
 
-log_action "Mounting base image $BASE"
-execute "mount -o loop $BASE $LOOPDIR"
+log_action "Connecting base image to NBD device $NBDEV"
+execute "qemu-nbd -c $NBDEV $BASE"
+sync
+
+log_action "Mounting $NBDPARTITION to $LOOPDIR"
+execute "mount $NBDPARTITION $LOOPDIR"
+
 log_action "Mounting proc filesystem to $LOOPDIR/proc"
 execute "mount -t proc none $LOOPDIR/proc"
 
@@ -43,6 +50,7 @@ execute "mount -o bind $ROOTFSCOMPILEDIR $LOOPDIR/root/compile"
 do_on_exit umount $LOOPDIR/root/compile
 do_on_exit umount $LOOPDIR/proc
 do_on_exit umount $LOOPDIR
+do_on_exit qemu-nbd -d $NBDEV
 
 echo "Installing software from source"
 RECPDIR=$TESTDIR/testing/scripts/recipes

diff --git a/testing/scripts/function.sh b/testing/scripts/function.sh
index 0d4ecb48455da5d225b71b74e36282068699f29c..d72708246f60ae63fe78023ac07d96423e3fd566 100755 (executable)
--- a/testing/scripts/function.sh
+++ b/testing/scripts/function.sh
@@ -117,6 +117,37 @@ do_on_exit()
        fi
 }
 
+# wait for a mount to disappear
+# $1 - device/image to wait for
+# $2 - maximum time to wait in seconds, default is 5 seconds
+graceful_umount()
+{
+       secs=$2
+       [ ! $secs ] && secs=5
+
+       let steps=$secs*100
+       for i in `seq 1 $steps`
+       do
+               umount $1 >>$LOGFILE 2>&1
+               mount | grep $1 >/dev/null 2>&1
+               [ $? -eq 0 ] || return 0
+               sleep 0.01
+       done
+
+       return 1
+}
+
+# load qemu NBD kernel module, if not already loaded
+load_qemu_nbd()
+{
+       lsmod | grep ^nbd[[:space:]]* >/dev/null 2>&1
+       if [ $? != 0 ]
+       then
+               log_action "Loading NBD kernel module"
+               execute "modprobe nbd max_part=16"
+       fi
+}
+
 #############################################
 # search and replace strings throughout a
 # whole directory

diff --git a/testing/start-testing b/testing/start-testing
index e6784322b77d7444581702539df284c0a97ae836..e4083bb4d4a0923c334ec936515e155e98a71ce6 100755 (executable)
--- a/testing/start-testing
+++ b/testing/start-testing
@@ -22,7 +22,7 @@ done
 
 for host in $STRONGSWANHOSTS
 do
-       ln -fs $ROOTFSDIR/$host.img $VIRTIMGSTORE/$host.img
+       ln -fs $ROOTFSDIR/$host.qcow2 $VIRTIMGSTORE/$host.qcow2
        log_action "Guest $host"
        execute "virsh create $CONFDIR/$host.xml"
 done

diff --git a/testing/stop-testing b/testing/stop-testing
index 3020531280034d624ad2c91918eeb058f5cac03b..0f38bc4c4c1fc0f223af8945e158cdb8737f6ed0 100755 (executable)
--- a/testing/stop-testing
+++ b/testing/stop-testing
@@ -19,7 +19,7 @@ for host in $STRONGSWANHOSTS
 do
        log_action "Guest $host"
        execute "virsh destroy $host"
-       rm -f $VIRTIMGSTORE/$host.img
+       rm -f $VIRTIMGSTORE/$host.qcow2
 done
 
 log_action "Removing kernel $KERNEL"

diff --git a/testing/testing.conf b/testing/testing.conf
index 6599334e027d573ade49ddf222c361262c795d8d..edb7102b46c5fb0e04e2dc8d49fa3cfe2ba2f890 100644 (file)
--- a/testing/testing.conf
+++ b/testing/testing.conf
@@ -40,7 +40,7 @@ LOOPDIR=$BUILDDIR/loop
 ROOTFSSIZE=1024
 ROOTFSSUITE=wheezy
 ROOTFSARCH=amd64
-ROOTFS=$BUILDDIR/debian-$ROOTFSSUITE-$ROOTFSARCH.img
+ROOTFS=$BUILDDIR/debian-$ROOTFSSUITE-$ROOTFSARCH.qcow2
 ROOTFSMIRROR=http://cdn.debian.net/debian
 ROOTFSCOMPILEDIR=$BUILDDIR/compile
 
@@ -48,6 +48,8 @@ ROOTFSCOMPILEDIR=$BUILDDIR/compile
 ROOTFSDIR=$BUILDDIR/root-fs
 
 # libvirt config
+NBDEV=/dev/nbd0
+NBDPARTITION=${NBDEV}p1
 VIRTIMGSTORE=/var/lib/libvirt/images
 
 # Directory where test results will be stored