From: Serge Hallyn <serge.hallyn@ubuntu.com>
Date: Fri, 17 May 2013 21:23:17 +0000 (+0200)
Subject: Move container creation fully into the api
X-Git-Tag: lxc-1.0.0.alpha1~1^2~182
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1897e3bcd36af9f3fe6d3649910a9adb93e5e988;p=thirdparty%2Flxc.git

Move container creation fully into the api

1. implement bdev->create:

python and lua: send NULL for bdevtype and bdevspecs.
They'll want to be updated to pass those in in a way that makes
sense, but I can't think about that right now.

2. templates: pass --rootfs

If the container is backed by a device which must be mounted (i.e.
lvm) then pass the actual rootfs mount destination to the
templates.

Note that the lxc.rootfs can be a mounted block device.  The template
should actually be installing the rootfs under the path where the
lxc.rootfs is *mounted*.

Still, some people like to run templates by hand and assume purely
directory backed containers, so continue to support that use case
(i.e. if no --rootfs is listed).

Make sure the templates don't re-write lxc.rootfs if it is
already in the config.  (Most were already checking for that)

3. Replace lxc-create script with lxc_create.c program.

Changelog:
May 24: when creating a container, create $lxcpath/$name/partial,
and flock it.  When done, close that file and unlink it.  In
lxc_container_new() and lxcapi_start(), check for this file.  If
it is locked, create is ongoing.  If it exists but is not locked,
create() was killed - remove the container.

May 24: dont disk-lock during lxcapi_create.  The partial lock
is sufficient.

Signed-off-by: Serge Hallyn <serge.hallyn@ubuntu.com>
---

diff --git a/configure.ac b/configure.ac
index 372fc75cb..d802406cf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -381,7 +381,6 @@ AC_CONFIG_FILES([
 	src/lxc/lxc-netstat
 	src/lxc/lxc-checkconfig
 	src/lxc/lxc-version
-	src/lxc/lxc-create
 	src/lxc/lxc-start-ephemeral
 	src/lxc/legacy/lxc-ls
 	src/lxc/lxc.functions
diff --git a/src/lua-lxc/core.c b/src/lua-lxc/core.c
index 364178685..778ef9926 100644
--- a/src/lua-lxc/core.c
+++ b/src/lua-lxc/core.c
@@ -111,7 +111,7 @@ static int container_create(lua_State *L)
 	argv[i] = strdupa(luaL_checkstring(L, i+3));
     argv[i] = NULL;
 
-    lua_pushboolean(L, !!c->create(c, template_name, argv));
+    lua_pushboolean(L, !!c->create(c, template_name, NULL, NULL, argv));
     return 1;
 }
 
diff --git a/src/lxc/Makefile.am b/src/lxc/Makefile.am
index 6974b272e..70fcd257c 100644
--- a/src/lxc/Makefile.am
+++ b/src/lxc/Makefile.am
@@ -121,8 +121,7 @@ bin_SCRIPTS = \
 	lxc-ps \
 	lxc-netstat \
 	lxc-checkconfig \
-	lxc-version \
-	lxc-create
+	lxc-version
 
 EXTRA_DIST = \
 	lxc-device \
@@ -160,7 +159,8 @@ bin_PROGRAMS = \
 	lxc-restart \
 	lxc-kill \
 	lxc-config \
-	lxc-destroy
+	lxc-destroy \
+    lxc-create
 
 pkglibexec_PROGRAMS = \
 	lxc-init
@@ -194,6 +194,7 @@ lxc_unfreeze_SOURCES = lxc_unfreeze.c
 lxc_unshare_SOURCES = lxc_unshare.c
 lxc_wait_SOURCES = lxc_wait.c
 lxc_kill_SOURCES = lxc_kill.c
+lxc_create_SOURCES = lxc_create.c
 
 install-exec-local: install-soPROGRAMS
 	mkdir -p $(DESTDIR)$(datadir)/lxc
diff --git a/src/lxc/arguments.h b/src/lxc/arguments.h
index 37de4b562..b01bd0826 100644
--- a/src/lxc/arguments.h
+++ b/src/lxc/arguments.h
@@ -75,6 +75,13 @@ struct lxc_arguments {
 	/* close fds from parent? */
 	int close_all_fds;
 
+	/* lxc-create */
+	char *bdevtype, *configfile, *template;
+	char *fstype;
+	unsigned long fssize;
+	char *lvname, *vgname;
+	char *zfsroot, *lowerdir, *dir;
+
 	/* remaining arguments */
 	char *const *argv;
 	int argc;
diff --git a/src/lxc/bdev.c b/src/lxc/bdev.c
index 6d61d2de4..5b89ba4a6 100644
--- a/src/lxc/bdev.c
+++ b/src/lxc/bdev.c
@@ -410,12 +410,35 @@ static int dir_destroy(struct bdev *orig)
 	return 0;
 }
 
+static int dir_create(struct bdev *bdev, const char *dest, const char *n,
+			struct bdev_specs *specs)
+{
+	bdev->src = strdup(dest);
+	bdev->dest = strdup(dest);
+	if (!bdev->src || !bdev->dest) {
+		ERROR("Out of memory");
+		return -1;
+	}
+
+	if (mkdir_p(bdev->src, 0755) < 0) {
+		ERROR("Error creating %s\n", bdev->src);
+		return -1;
+	}
+	if (mkdir_p(bdev->dest, 0755) < 0) {
+		ERROR("Error creating %s\n", bdev->dest);
+		return -1;
+	}
+
+	return 0;
+}
+
 struct bdev_ops dir_ops = {
 	.detect = &dir_detect,
 	.mount = &dir_mount,
 	.umount = &dir_umount,
 	.clone_paths = &dir_clonepaths,
 	.destroy = &dir_destroy,
+	.create = &dir_create,
 };
 
 
@@ -620,12 +643,51 @@ static int zfs_destroy(struct bdev *orig)
 	exit(1);
 }
 
+static int zfs_create(struct bdev *bdev, const char *dest, const char *n,
+			struct bdev_specs *specs)
+{
+	const char *zfsroot;
+	char option[MAXPATHLEN];
+	int ret;
+	pid_t pid;
+
+	if (!specs || !specs->u.zfs.zfsroot)
+		zfsroot = default_zfs_root();
+	else
+		zfsroot = specs->u.zfs.zfsroot;
+
+	if (!(bdev->dest = strdup(dest))) {
+		ERROR("No mount target specified or out of memory");
+		return -1;
+	}
+	if (!(bdev->src = strdup(bdev->dest))) {
+		ERROR("out of memory");
+		return -1;
+	}
+
+	ret = snprintf(option, MAXPATHLEN, "-omountpoint=%s", bdev->dest);
+	if (ret < 0  || ret >= MAXPATHLEN)
+		return -1;
+	if ((pid = fork()) < 0)
+		return -1;
+	if (pid)
+		return wait_for_pid(pid);
+
+	char dev[MAXPATHLEN];
+	ret = snprintf(dev, MAXPATHLEN, "%s/%s", zfsroot, n);
+	if (ret < 0  || ret >= MAXPATHLEN)
+		exit(1);
+	execlp("zfs", "zfs", "create", option, dev, NULL);
+	exit(1);
+}
+
 struct bdev_ops zfs_ops = {
 	.detect = &zfs_detect,
 	.mount = &zfs_mount,
 	.umount = &zfs_umount,
 	.clone_paths = &zfs_clonepaths,
 	.destroy = &zfs_destroy,
+	.create = &zfs_create,
 };
 
 //
@@ -693,7 +755,7 @@ static int lvm_umount(struct bdev *bdev)
  * not yet exist.  This function will attempt to create /dev/$vg/$lv of
  * size $size.
  */
-static int lvm_create(const char *path, unsigned long size)
+static int do_lvm_create(const char *path, unsigned long size)
 {
 	int ret, pid;
 	char sz[24], *pathdup, *vg, *lv;
@@ -839,7 +901,7 @@ static int lvm_clonepaths(struct bdev *orig, struct bdev *new, const char *oldna
 			return -1;
 		}
 	} else {
-		if (lvm_create(new->src, size) < 0) {
+		if (do_lvm_create(new->src, size) < 0) {
 			ERROR("Error creating new lvm blockdev");
 			return -1;
 		}
@@ -866,12 +928,71 @@ static int lvm_destroy(struct bdev *orig)
 	return wait_for_pid(pid);
 }
 
+#define DEFAULT_LVM_SZ 1024000000
+#define DEFAULT_LVM_FSTYPE "ext3"
+static int lvm_create(struct bdev *bdev, const char *dest, const char *n,
+			struct bdev_specs *specs)
+{
+	const char *vg, *fstype, *lv = n;
+	unsigned long sz;
+	int ret, len;
+
+	if (!specs)
+		return -1;
+
+	vg = specs->u.lvm.vg;
+	if (!vg)
+		vg = default_lvm_vg();
+
+	/* /dev/$vg/$lv */
+	if (specs->u.lvm.lv)
+		lv = specs->u.lvm.lv;
+	len = strlen(vg) + strlen(lv) + 7;
+	bdev->src = malloc(len);
+	if (!bdev->src)
+		return -1;
+
+	ret = snprintf(bdev->src, len, "/dev/%s/%s", vg, lv);
+	if (ret < 0 || ret >= len)
+		return -1;
+
+	// lvm.fssize is in bytes.
+	sz = specs->u.lvm.fssize;
+	if (!sz)
+		sz = DEFAULT_LVM_SZ;
+
+	INFO("Error creating new lvm blockdev %s size %lu", bdev->src, sz);
+	if (do_lvm_create(bdev->src, sz) < 0) {
+		ERROR("Error creating new lvm blockdev %s size %lu", bdev->src, sz);
+		return -1;
+	}
+
+	fstype = specs->u.lvm.fstype;
+	if (!fstype)
+		fstype = DEFAULT_LVM_FSTYPE;
+	if (do_mkfs(bdev->src, fstype) < 0) {
+		ERROR("Error creating filesystem type %s on %s", fstype,
+			bdev->src);
+		return -1;
+	}
+	if (!(bdev->dest = strdup(dest)))
+		return -1;
+
+	if (mkdir_p(bdev->dest, 0755) < 0) {
+		ERROR("Error creating %s\n", bdev->dest);
+		return -1;
+	}
+
+	return 0;
+}
+
 struct bdev_ops lvm_ops = {
 	.detect = &lvm_detect,
 	.mount = &lvm_mount,
 	.umount = &lvm_umount,
 	.clone_paths = &lvm_clonepaths,
 	.destroy = &lvm_destroy,
+	.create = &lvm_create,
 };
 
 //
@@ -895,21 +1016,31 @@ struct btrfs_ioctl_space_args {
 #define BTRFS_IOC_SPACE_INFO _IOWR(BTRFS_IOCTL_MAGIC, 20, \
                                     struct btrfs_ioctl_space_args)
 
-static int btrfs_detect(const char *path)
+static bool is_btrfs_fs(const char *path)
 {
-	struct stat st;
 	int fd, ret;
 	struct btrfs_ioctl_space_args sargs;
 
 	// make sure this is a btrfs filesystem
 	fd = open(path, O_RDONLY);
 	if (fd < 0)
-		return 0;
+		return false;
 	sargs.space_slots = 0;
 	sargs.total_spaces = 0;
 	ret = ioctl(fd, BTRFS_IOC_SPACE_INFO, &sargs);
 	close(fd);
 	if (ret < 0)
+		return false;
+
+	return true;
+}
+
+static int btrfs_detect(const char *path)
+{
+	struct stat st;
+	int ret;
+
+	if (!is_btrfs_fs(path))
 		return 0;
 
 	// and make sure it's a subvolume.
@@ -1138,12 +1269,23 @@ static int btrfs_destroy(struct bdev *orig)
 	return ret;
 }
 
+static int btrfs_create(struct bdev *bdev, const char *dest, const char *n,
+			struct bdev_specs *specs)
+{
+	bdev->src = strdup(dest);
+	bdev->dest = strdup(dest);
+	if (!bdev->src || !bdev->dest)
+		return -1;
+	return btrfs_subvolume_create(bdev->dest);
+}
+
 struct bdev_ops btrfs_ops = {
 	.detect = &btrfs_detect,
 	.mount = &btrfs_mount,
 	.umount = &btrfs_umount,
 	.clone_paths = &btrfs_clonepaths,
 	.destroy = &btrfs_destroy,
+	.create = &btrfs_create,
 };
 
 //
@@ -1321,12 +1463,60 @@ int overlayfs_destroy(struct bdev *orig)
 	return lxc_rmdir_onedev(upper);
 }
 
+/*
+ * to say 'lxc-create -t ubuntu -n o1 -B overlayfs' means you want
+ * $lxcpath/$lxcname/rootfs to have the created container, while all
+ * changes after starting the container are written to
+ * $lxcpath/$lxcname/delta0
+ */
+static int overlayfs_create(struct bdev *bdev, const char *dest, const char *n,
+			struct bdev_specs *specs)
+{
+	char *delta;
+	int ret, len = strlen(dest), newlen;
+
+	if (len < 8 || strcmp(dest+len-7, "/rootfs") != 0)
+		return -1;
+
+	if (!(bdev->dest = strdup(dest))) {
+		ERROR("Out of memory");
+		return -1;
+	}
+
+	delta = strdupa(dest);
+	strcpy(delta+len-6, "delta0");
+
+	if (mkdir_p(delta, 0755) < 0) {
+		ERROR("Error creating %s\n", delta);
+		return -1;
+	}
+
+	/* overlayfs:lower:upper */
+	newlen = (2 * len) + strlen("overlayfs:") + 2;
+	bdev->src = malloc(newlen);
+	if (!bdev->src) {
+		ERROR("Out of memory");
+		return -1;
+	}
+	ret = snprintf(bdev->src, newlen, "overlayfs:%s:%s", dest, delta);
+	if (ret < 0 || ret >= newlen)
+		return -1;
+
+	if (mkdir_p(bdev->dest, 0755) < 0) {
+		ERROR("Error creating %s\n", bdev->dest);
+		return -1;
+	}
+
+	return 0;
+}
+
 struct bdev_ops overlayfs_ops = {
 	.detect = &overlayfs_detect,
 	.mount = &overlayfs_mount,
 	.umount = &overlayfs_umount,
 	.clone_paths = &overlayfs_clonepaths,
 	.destroy = &overlayfs_destroy,
+	.create = &overlayfs_create,
 };
 
 struct bdev_type bdevs[] = {
@@ -1497,3 +1687,60 @@ struct bdev *bdev_copy(const char *src, const char *oldname, const char *cname,
 
 	exit(0);
 }
+
+/*
+ * bdev_create:
+ * Create a backing store for a container.
+ * If successfull, return a struct bdev *, with the bdev mounted and ready
+ * for use.  Before completing, the caller will need to call the
+ * umount operation and bdev_put().
+ * @dest: the mountpoint (i.e. /var/lib/lxc/$name/rootfs)
+ * @type: the bdevtype (dir, btrfs, zfs, etc)
+ * @cname: the container name
+ * @specs: details about the backing store to create, like fstype
+ */
+struct bdev *bdev_create(const char *dest, const char *type,
+			const char *cname, struct bdev_specs *specs)
+{
+	struct bdev *bdev;
+
+	if (!type) {
+		char *p, *p1;
+
+		type = "dir";
+
+		/*
+		 * $lxcpath/$lxcname/rootfs doesn't yet exist.  Check
+		 * whether $lxcpath/$lxcname is btrfs.  If so, specify
+		 * btrfs backing store for the container.
+		 */
+		p = strdupa(dest);
+		p1 = rindex(p, '/');
+		if (p1) {
+			*p1 = '\0';
+			if (is_btrfs_fs(p))
+				type = "btrfs";
+		}
+	}
+
+	bdev = bdev_get(type);
+	if (!bdev) {
+		ERROR("Unknown fs type: %s\n", type);
+		return NULL;
+	}
+
+	if (bdev->ops->create(bdev, dest, cname, specs) < 0) {
+		 bdev_put(bdev);
+		 return NULL;
+	}
+
+	return bdev;
+}
+
+char *overlayfs_getlower(char *p)
+{
+	char *p1 = index(p, ':');
+	if (p1)
+		*p1 = '\0';
+	return p;
+}
diff --git a/src/lxc/bdev.h b/src/lxc/bdev.h
index e4f1e6080..7b60d953e 100644
--- a/src/lxc/bdev.h
+++ b/src/lxc/bdev.h
@@ -10,6 +10,23 @@
 
 struct bdev;
 
+/*
+ * specifications for how to create a new backing store
+ */
+struct bdev_specs {
+	union {
+		struct {
+			char *zfsroot;
+		} zfs;
+		struct {
+			char *vg;
+			char *lv;
+			char *fstype;
+			unsigned long fssize;  // fs size in bytes
+		} lvm;
+	} u;
+};
+
 struct bdev_ops {
 	/* detect whether path is of this bdev type */
 	int (*detect)(const char *path);
@@ -17,6 +34,8 @@ struct bdev_ops {
 	int (*mount)(struct bdev *bdev);
 	int (*umount)(struct bdev *bdev);
 	int (*destroy)(struct bdev *bdev);
+	int (*create)(struct bdev *bdev, const char *dest, const char *n,
+			struct bdev_specs *specs);
 	/* given original mount, rename the paths for cloned container */
 	int (*clone_paths)(struct bdev *orig, struct bdev *new, const char *oldname,
 			const char *cname, const char *oldpath, const char *lxcpath,
@@ -38,6 +57,8 @@ struct bdev {
 	char *data;
 };
 
+char *overlayfs_getlower(char *p);
+
 /*
  * Instantiate a bdev object.  The src is used to determine which blockdev
  * type this should be.  The dst and data are optional, and will be used
@@ -54,6 +75,8 @@ struct bdev *bdev_init(const char *src, const char *dst, const char *data);
 struct bdev *bdev_copy(const char *src, const char *oldname, const char *cname,
 			const char *oldpath, const char *lxcpath, const char *bdevtype,
 			int snap, const char *bdevdata, unsigned long newsize);
+struct bdev *bdev_create(const char *dest, const char *type,
+			const char *cname, struct bdev_specs *specs);
 void bdev_put(struct bdev *bdev);
 
 /* define constants if the kernel/glibc headers don't define them */
diff --git a/src/lxc/lxc_create.c b/src/lxc/lxc_create.c
new file mode 100644
index 000000000..261360430
--- /dev/null
+++ b/src/lxc/lxc_create.c
@@ -0,0 +1,205 @@
+/*
+ *
+ * Copyright © 2013 Serge Hallyn <serge.hallyn@ubuntu.com>.
+ * Copyright © 2013 Canonical Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include "../lxc/lxccontainer.h"
+
+#include <stdio.h>
+#include <libgen.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <sys/types.h>
+
+#include <lxc/lxc.h>
+#include <lxc/log.h>
+#include <lxc/bdev.h>
+
+#include "arguments.h"
+#include "utils.h"
+
+lxc_log_define(lxc_create, lxc);
+
+/* we pass fssize in bytes */
+static unsigned long get_fssize(char *s)
+{
+	unsigned long ret;
+	char *end;
+
+	ret = strtoul(s, &end, 0);
+	if (end == s)
+		return 0;
+	while (isblank(*end))
+		end++;
+	if (!(*end))
+		return ret;
+	if (*end == 'g' || *end == 'G')
+		ret *= 1000000000;
+	else if (*end == 'm' || *end == 'M')
+		ret *= 1000000;
+	else if (*end == 'k' || *end == 'K')
+		ret *= 1000;
+	return ret;
+}
+
+static int my_parser(struct lxc_arguments* args, int c, char* arg)
+{
+	switch (c) {
+	case 'B': args->bdevtype = arg; break;
+	case 'f': args->configfile = arg; break;
+	case 't': args->template = arg; break;
+	case '0': args->lvname = arg; break;
+	case '1': args->vgname = arg; break;
+	case '2': args->fstype = arg; break;
+	case '3': args->fssize = get_fssize(arg); break;
+	case '4': args->zfsroot = arg; break;
+	case '5': args->dir = arg; break;
+	}
+	return 0;
+}
+
+static const struct option my_longopts[] = {
+	{"bdev", required_argument, 0, 'B'},
+	{"config", required_argument, 0, 'f'},
+	{"template", required_argument, 0, 't'},
+	{"lvname", required_argument, 0, '0'},
+	{"vgname", required_argument, 0, '1'},
+	{"fstype", required_argument, 0, '2'},
+	{"fssize", required_argument, 0, '3'},
+	{"zfsroot", required_argument, 0, '4'},
+	{"dir", required_argument, 0, '5'},
+	LXC_COMMON_OPTIONS
+};
+
+static struct lxc_arguments my_args = {
+	.progname = "lxc-create",
+	.help     = "\
+--name=NAME [-w] [-r] [-t timeout] [-P lxcpath]\n\
+\n\
+lxc-creae creates a container\n\
+\n\
+Options :\n\
+  -n, --name=NAME   NAME for name of the container\n\
+  -f, --config=file initial configuration file\n\
+  -t, --template=t  template to use to setup container\n\
+  -B, --bdev=BDEV   backing store type to use\n\
+  --lxcpath=PATH    place container under PATH\n\
+  --lvname=LVNAME   Use LVM lv name LVNAME\n\
+                    (Default: container name)\n\
+  --vgname=VG       Use LVM vg called VG\n\
+                    (Default: lxc))\n\
+  --fstype=TYPE     Create fstype TYPE\n\
+                    (Default: ext3))\n\
+  --fssize=SIZE     Create filesystem of size SIZE\n\
+                    (Default: 1G))\n\
+  --dir=DIR         Place rootfs directory under DIR\n\
+  --zfsroot=PATH    Create zfs under given zfsroot\n\
+                    (Default: tank/lxc))\n",
+	.options  = my_longopts,
+	.parser   = my_parser,
+	.checker  = NULL,
+};
+
+bool validate_bdev_args(struct lxc_arguments *a)
+{
+	if (strcmp(a->bdevtype, "lvm") != 0) {
+		if (a->fstype || a->fssize) {
+			fprintf(stderr, "filesystem type and size are only valid with block devices\n");
+			return false;
+		}
+		if (a->lvname || a->vgname) {
+			fprintf(stderr, "--lvname and --vgname are only valid with -B lvm\n");
+			return false;
+		}
+	}
+	if (strcmp(a->bdevtype, "zfs") != 0) {
+		if (a->zfsroot) {
+			fprintf(stderr, "zfsroot is only valid with -B zfs\n");
+			return false;
+		}
+	}
+	return true;
+}
+
+/* grab this through autoconf from @config-path@ ? */
+#define DEFAULT_CONFIG "/etc/lxc/default.conf"
+int main(int argc, char *argv[])
+{
+	struct lxc_container *c;
+	struct bdev_specs spec;
+
+	/* this is a short term test.  We'll probably want to check for
+	 * write access to lxcpath instead */
+	if (geteuid()) {
+		fprintf(stderr, "%s must be run as root\n", argv[0]);
+		exit(1);
+	}
+
+	if (lxc_arguments_parse(&my_args, argc, argv))
+		exit(1);
+
+	if (lxc_log_init(my_args.name, my_args.log_file, my_args.log_priority,
+			 my_args.progname, my_args.quiet, my_args.lxcpath[0]))
+		exit(1);
+
+	memset(&spec, 0, sizeof(spec));
+	if (!my_args.bdevtype)
+		my_args.bdevtype = "_unset";
+	if (!validate_bdev_args(&my_args))
+		exit(1);
+
+	c = lxc_container_new(my_args.name, my_args.lxcpath[0]);
+	if (!c) {
+		fprintf(stderr, "System error loading container\n");
+		exit(1);
+	}
+	if (c->is_defined(c)) {
+		fprintf(stderr, "Container already exists\n");
+		exit(1);
+	}
+	if (my_args.configfile)
+		c->load_config(c, my_args.configfile);
+	else
+		c->load_config(c, DEFAULT_CONFIG);
+
+	if (strcmp(my_args.bdevtype, "zfs") == 0) {
+		if (my_args.zfsroot)
+			spec.u.zfs.zfsroot = my_args.zfsroot;
+	} else if (strcmp(my_args.bdevtype, "lvm") == 0) {
+		if (my_args.lvname)
+			spec.u.lvm.lv = my_args.lvname;
+		if (my_args.vgname)
+			spec.u.lvm.vg = my_args.vgname;
+		if (my_args.fstype)
+			spec.u.lvm.fstype = my_args.fstype;
+		if (my_args.fssize)
+			spec.u.lvm.fssize = my_args.fssize;
+	} else if (my_args.dir) {
+		ERROR("--dir is not yet supported");
+		exit(1);
+	}
+
+	if (strcmp(my_args.bdevtype, "_unset") == 0)
+		my_args.bdevtype = NULL;
+	if (!c->create(c, my_args.template, my_args.bdevtype, &spec, &argv[optind])) {
+		ERROR("Error creating container %s", c->name);
+		lxc_container_put(c);
+		exit(1);
+	}
+	INFO("container %s created", c->name);
+	exit(0);
+}
diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
index 1d1174259..2ea9556b3 100644
--- a/src/lxc/lxccontainer.c
+++ b/src/lxc/lxccontainer.c
@@ -106,7 +106,7 @@ int create_partial(struct lxc_container *c)
 		ERROR("Error writing partial pathname");
 		return -1;
 	}
-	if (process_lock() < 0)
+	if (process_lock())
 		return -1;
 	if ((fd=open(path, O_CREAT | O_EXCL, 0755)) < 0) {
 		SYSERROR("Erorr creating partial file");
@@ -645,12 +645,54 @@ static bool create_container_dir(struct lxc_container *c)
 	return ret == 0;
 }
 
+static const char *lxcapi_get_config_path(struct lxc_container *c);
+static bool lxcapi_set_config_item(struct lxc_container *c, const char *key, const char *v);
+
 /*
- * backing stores not (yet) supported
- * for ->create, argv contains the arguments to pass to the template,
- * terminated by NULL.  If no arguments, you can just pass NULL.
+ * do_bdev_create: thin wrapper around bdev_create().  Like bdev_create(),
+ * it returns a mounted bdev on success, NULL on error.
  */
-static bool lxcapi_create(struct lxc_container *c, const char *t, char *const argv[])
+static struct bdev *do_bdev_create(struct lxc_container *c, const char *type,
+			 struct bdev_specs *specs)
+{
+	char *dest;
+	const char *lxcpath = lxcapi_get_config_path(c);
+	size_t len;
+	struct bdev *bdev;
+	int ret;
+
+	/* lxcpath/lxcname/rootfs */
+	len = strlen(c->name) + strlen(lxcpath) + 9;
+	dest = alloca(len);
+	ret = snprintf(dest, len, "%s/%s/rootfs", lxcpath, c->name);
+	if (ret < 0 || ret >= len)
+		return NULL;
+
+	bdev = bdev_create(dest, type, c->name, specs);
+	if (!bdev)
+		return NULL;
+	lxcapi_set_config_item(c, "lxc.rootfs", bdev->src);
+	return bdev;
+}
+
+static bool lxcapi_destroy(struct lxc_container *c);
+/*
+ * lxcapi_create:
+ * create a container with the given parameters.
+ * @c: container to be created.  It has the lxcpath, name, and a starting
+ *     configuration already set
+ * @t: the template to execute to instantiate the root filesystem and
+ *     adjust the configuration.
+ * @bdevtype: backing store type to use.  If NULL, dir will be used.
+ * @specs: additional parameters for the backing store, i.e. LVM vg to
+ *         use.
+ *
+ * @argv: the arguments to pass to the template, terminated by NULL.  If no
+ * arguments, you can just pass NULL.
+ */
+static bool lxcapi_create(struct lxc_container *c, const char *t,
+		const char *bdevtype, struct bdev_specs *specs,
+		char *const argv[])
 {
 	bool bret = false;
 	pid_t pid;
@@ -685,12 +727,53 @@ static bool lxcapi_create(struct lxc_container *c, const char *t, char *const ar
 	if ((partial_fd = create_partial(c)) < 0)
 		goto out;
 
-	/* we're going to fork.  but since we'll wait for our child, we
-	 * don't need to lxc_container_get */
+	/* no need to get disk lock bc we have the partial locked */
 
-	if (container_disk_lock(c))
+	/*
+	 * Create the backing store
+	 * Note we can't do this in the same task as we use to execute the
+	 * template because of the way zfs works.
+	 * After you 'zfs create', zfs mounts the fs only in the initial
+	 * namespace.
+	 */
+	pid = fork();
+	if (pid < 0) {
+		SYSERROR("failed to fork task for container creation template\n");
+		goto out_unlock;
+	}
+
+	if (pid == 0) { // child
+		struct bdev *bdev = NULL;
+
+		if (!(bdev = do_bdev_create(c, bdevtype, specs))) {
+			ERROR("Error creating backing store type %s for %s",
+				bdevtype ? bdevtype : "(none)", c->name);
+			exit(1);
+		}
+
+		/* save config file again to store the new rootfs location */
+		if (!c->save_config(c, NULL)) {
+			ERROR("failed to save starting configuration for %s\n", c->name);
+			// parent task won't see bdev in config so we delete it
+			bdev->ops->umount(bdev);
+			bdev->ops->destroy(bdev);
+			exit(1);
+		}
+		exit(0);
+	}
+	if (wait_for_pid(pid) != 0)
+		goto out;
+
+	/* reload config to get the rootfs */
+	if (c->lxc_conf)
+		lxc_conf_free(c->lxc_conf);
+	c->lxc_conf = NULL;
+	if (!load_config_locked(c, c->configfile))
 		goto out;
 
+	/*
+	 * now execute the template
+	 */
 	pid = fork();
 	if (pid < 0) {
 		SYSERROR("failed to fork task for container creation template\n");
@@ -698,7 +781,8 @@ static bool lxcapi_create(struct lxc_container *c, const char *t, char *const ar
 	}
 
 	if (pid == 0) { // child
-		char *patharg, *namearg;
+		char *patharg, *namearg, *rootfsarg, *src;
+		struct bdev *bdev = NULL;
 		int i;
 
 		close(0);
@@ -708,13 +792,38 @@ static bool lxcapi_create(struct lxc_container *c, const char *t, char *const ar
 		open("/dev/null", O_RDWR);
 		open("/dev/null", O_RDWR);
 
+		if (unshare(CLONE_NEWNS) < 0) {
+			ERROR("error unsharing mounts");
+			exit(1);
+		}
+
+		src = c->lxc_conf->rootfs.path;
+		/*
+		 * for an overlayfs create, what the user wants is the template to fill
+		 * in what will become the readonly lower layer.  So don't mount for
+		 * the template
+		 */
+		if (strncmp(src, "overlayfs:", 10) == 0) {
+			src = overlayfs_getlower(src+10);
+		}
+		bdev = bdev_init(src, c->lxc_conf->rootfs.mount, NULL);
+		if (!bdev) {
+			ERROR("Error opening rootfs");
+			exit(1);
+		}
+
+		if (bdev->ops->mount(bdev) < 0) {
+			ERROR("Error mounting rootfs");
+			exit(1);
+		}
+
 		/*
 		 * create our new array, pre-pend the template name and
 		 * base args
 		 */
 		if (argv)
-			for (; argv[nargs]; nargs++) ;
-		nargs += 3;  // template, path and name args
+			for (nargs = 0; argv[nargs]; nargs++) ;
+		nargs += 4;  // template, path, rootfs and name args
 		newargv = malloc(nargs * sizeof(*newargv));
 		if (!newargv)
 			exit(1);
@@ -737,10 +846,19 @@ static bool lxcapi_create(struct lxc_container *c, const char *t, char *const ar
 			exit(1);
 		newargv[2] = namearg;
 
+		len = strlen("--rootfs=") + 1 + strlen(bdev->dest);
+		rootfsarg = malloc(len);
+		if (!rootfsarg)
+			exit(1);
+		ret = snprintf(rootfsarg, len, "--rootfs=%s", bdev->dest);
+		if (ret < 0 || ret >= len)
+			exit(1);
+		newargv[3] = rootfsarg;
+
 		/* add passed-in args */
 		if (argv)
-			for (i = 3; i < nargs; i++)
-				newargv[i] = argv[i-3];
+			for (i = 4; i < nargs; i++)
+				newargv[i] = argv[i-4];
 
 		/* add trailing NULL */
 		nargs++;
@@ -774,6 +892,8 @@ out_unlock:
 out:
 	if (tpath)
 		free(tpath);
+	if (!bret && c)
+		lxcapi_destroy(c);
 	return bret;
 }
 
@@ -818,7 +938,8 @@ static bool lxcapi_shutdown(struct lxc_container *c, int timeout)
 	return retv;
 }
 
-static bool lxcapi_createl(struct lxc_container *c, const char *t, ...)
+static bool lxcapi_createl(struct lxc_container *c, const char *t,
+		const char *bdevtype, struct bdev_specs *specs, ...)
 {
 	bool bret = false;
 	char **args = NULL, **temp;
@@ -832,7 +953,7 @@ static bool lxcapi_createl(struct lxc_container *c, const char *t, ...)
 	 * since we're going to wait for create to finish, I don't think we
 	 * need to get a copy of the arguments.
 	 */
-	va_start(ap, t);
+	va_start(ap, specs);
 	while (1) {
 		char *arg;
 		arg = va_arg(ap, char *);
@@ -851,7 +972,7 @@ static bool lxcapi_createl(struct lxc_container *c, const char *t, ...)
 	if (args)
 		args[nargs] = NULL;
 
-	bret = c->create(c, t, args);
+	bret = c->create(c, t, bdevtype, specs, args);
 
 out:
 	if (args)
@@ -1048,15 +1169,13 @@ static bool lxcapi_save_config(struct lxc_container *c, const char *alt_file)
 	return true;
 }
 
-static const char *lxcapi_get_config_path(struct lxc_container *c);
 // do we want the api to support --force, or leave that to the caller?
 static bool lxcapi_destroy(struct lxc_container *c)
 {
-	struct bdev *r;
+	struct bdev *r = NULL;
 	bool ret = false;
 
-	/* container is already destroyed if we don't have a config and rootfs.path is not accessible */
-	if (!c || !lxcapi_is_defined(c) || !c->lxc_conf || !c->lxc_conf->rootfs.path)
+	if (!c || !lxcapi_is_defined(c))
 		return false;
 
 	if (lxclock(c->privlock, 0))
@@ -1072,7 +1191,8 @@ static bool lxcapi_destroy(struct lxc_container *c)
 		goto out;
 	}
 
-	r = bdev_init(c->lxc_conf->rootfs.path, c->lxc_conf->rootfs.mount, NULL);
+	if (c->lxc_conf->rootfs.path && c->lxc_conf->rootfs.mount)
+		r = bdev_init(c->lxc_conf->rootfs.path, c->lxc_conf->rootfs.mount, NULL);
 	if (r) {
 		if (r->ops->destroy(r) < 0) {
 			ERROR("Error destroying rootfs for %s", c->name);
@@ -1848,8 +1968,9 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
 
 	if (ongoing_create(c) == 2) {
 		ERROR("Error: %s creation was not completed", c->name);
-		c->destroy(c);
-		goto err;
+		lxcapi_destroy(c);
+		lxc_conf_free(c->lxc_conf);
+		c->lxc_conf = NULL;
 	}
 
 	// assign the member functions
diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
index 3a80d0f9f..bd3d22a54 100644
--- a/src/lxc/lxccontainer.h
+++ b/src/lxc/lxccontainer.h
@@ -12,6 +12,8 @@
 #define LXC_CLONE_SNAPSHOT        (1 << 3)
 #define LXC_CLONE_MAXFLAGS        (1 << 4)
 
+struct bdev_specs;
+
 struct lxc_container {
 	// private fields
 	char *name;
@@ -48,8 +50,10 @@ struct lxc_container {
 	bool (*set_config_item)(struct lxc_container *c, const char *key, const char *value);
 	bool (*destroy)(struct lxc_container *c);
 	bool (*save_config)(struct lxc_container *c, const char *alt_file);
-	bool (*create)(struct lxc_container *c, const char *t, char *const argv[]);
-	bool (*createl)(struct lxc_container *c, const char *t, ...);
+	bool (*create)(struct lxc_container *c, const char *t, const char *bdevtype,
+			struct bdev_specs *specs, char *const argv[]);
+	bool (*createl)(struct lxc_container *c, const char *t, const char *bdevtype,
+			struct bdev_specs *specs, ...);
 	/* send SIGINT to ask container to reboot */
 	bool (*reboot)(struct lxc_container *c);
 	/* send SIGPWR.  if timeout is not 0 or -1, do a hard stop after timeout seconds */
diff --git a/src/python-lxc/lxc.c b/src/python-lxc/lxc.c
index cdc1e8b83..0b31a372e 100644
--- a/src/python-lxc/lxc.c
+++ b/src/python-lxc/lxc.c
@@ -249,7 +249,7 @@ Container_create(Container *self, PyObject *args, PyObject *kwds)
         }
     }
 
-    if (self->container->create(self->container, template_name, create_args))
+    if (self->container->create(self->container, template_name, NULL, NULL, create_args))
         retval = Py_True;
     else
         retval = Py_False;
diff --git a/src/tests/cgpath.c b/src/tests/cgpath.c
index 5fb1d319d..55c6664bc 100644
--- a/src/tests/cgpath.c
+++ b/src/tests/cgpath.c
@@ -279,7 +279,7 @@ static int test_container(const char *lxcpath,
 		c = lxc_container_new(name, lxcpath);
 	}
 	c->set_config_item(c, "lxc.network.type", "empty");
-	if (!c->createl(c, template, NULL)) {
+	if (!c->createl(c, template, NULL, NULL, NULL)) {
 		TSTERR("creating container %s", name);
 		goto out2;
 	}
diff --git a/src/tests/clonetest.c b/src/tests/clonetest.c
index fbeacdf71..f15f400d9 100644
--- a/src/tests/clonetest.c
+++ b/src/tests/clonetest.c
@@ -53,7 +53,7 @@ int main(int argc, char *argv[])
 		goto out;
 	}
 	c->save_config(c, NULL);
-	if (!c->createl(c, "ubuntu", NULL)) {
+	if (!c->createl(c, "ubuntu", NULL, NULL, NULL)) {
 		fprintf(stderr, "%d: failed to create a container\n", __LINE__);
 		goto out;
 	}
diff --git a/src/tests/createtest.c b/src/tests/createtest.c
index c2abee233..f33f59bfc 100644
--- a/src/tests/createtest.c
+++ b/src/tests/createtest.c
@@ -50,7 +50,7 @@ int main(int argc, char *argv[])
 	}
 	c->set_config_item(c, "lxc.network.link", "lxcbr0");
 	c->set_config_item(c, "lxc.network.flags", "up");
-	if (!c->createl(c, "ubuntu", "-r", "lucid", NULL)) {
+	if (!c->createl(c, "ubuntu", NULL, NULL, "-r", "lucid", NULL)) {
 		fprintf(stderr, "%d: failed to create a lucid container\n", __LINE__);
 		goto out;
 	}
diff --git a/src/tests/get_item.c b/src/tests/get_item.c
index d3e6d29ad..95118995c 100644
--- a/src/tests/get_item.c
+++ b/src/tests/get_item.c
@@ -170,7 +170,7 @@ int main(int argc, char *argv[])
 		ret = 1;
 		goto out;
 	}
-	if (!c->createl(c, "ubuntu", "-r", "lucid", NULL)) {
+	if (!c->createl(c, "ubuntu", NULL, NULL, "-r", "lucid", NULL)) {
 		fprintf(stderr, "%d: failed to create a lucid container\n", __LINE__);
 		ret = 1;
 		goto out;
diff --git a/src/tests/shutdowntest.c b/src/tests/shutdowntest.c
index a1a84e813..6e50cb097 100644
--- a/src/tests/shutdowntest.c
+++ b/src/tests/shutdowntest.c
@@ -51,7 +51,7 @@ int main(int argc, char *argv[])
 	}
 	c->set_config_item(c, "lxc.network.link", "lxcbr0");
 	c->set_config_item(c, "lxc.network.flags", "up");
-	if (!c->createl(c, "ubuntu", "-r", "lucid", NULL)) {
+	if (!c->createl(c, "ubuntu", NULL, NULL, "-r", "lucid", NULL)) {
 		fprintf(stderr, "%d: failed to create a lucid container\n", __LINE__);
 		goto out;
 	}
diff --git a/templates/lxc-alpine.in b/templates/lxc-alpine.in
index 98347ed67..6180afd0c 100644
--- a/templates/lxc-alpine.in
+++ b/templates/lxc-alpine.in
@@ -150,7 +150,8 @@ die() {
 usage() {
     cat >&2 <<EOF
 Usage: $(basename $0) [-h|--help] [-r|--repository <url>] [-a|--arch <arch>]
-                   -p|--path <path> -n|--name <name> [PKG...]
+                   [--rootfs <rootfs>] -p|--path <path> -n|--name <name>
+                   [PKG...]
 EOF
 }
 
@@ -180,6 +181,11 @@ while [ $# -gt 0 ]; do
         name=$1
         shift
         ;;
+    --rootfs)
+        optarg_check $opt "$1"
+        rootfs=$1
+        shift
+        ;;
     -p|--path)
         optarg_check $opt "$1"
         path=$1
@@ -218,9 +224,11 @@ if [ -z "${path}" ]; then
     path="${default_path}/${name}"
 fi
 
-rootfs=`awk -F= '$1 ~ /^lxc.rootfs/ { print $2 }' "$path/config" 2>/dev/null`
 if [ -z "$rootfs" ]; then
-    rootfs="${path}/rootfs"
+    rootfs=`awk -F= '$1 ~ /^lxc.rootfs/ { print $2 }' "$path/config" 2>/dev/null`
+    if [ -z "$rootfs" ]; then
+        rootfs="${path}/rootfs"
+    fi
 fi
 
 lxc_arch=$arch
diff --git a/templates/lxc-altlinux.in b/templates/lxc-altlinux.in
index cce214c0f..798f88249 100644
--- a/templates/lxc-altlinux.in
+++ b/templates/lxc-altlinux.in
@@ -337,7 +337,7 @@ usage:
         [-p|--path=<path>] [-c|--clean] [-R|--release=<ALTLinux_release>]
         [-4|--ipv4=<ipv4 address>] [-6|--ipv6=<ipv6 address>]
         [-g|--gw=<gw address>] [-d|--dns=<dns address>]
-        [-P|--profile=<name of the profile>]
+        [-P|--profile=<name of the profile>] [--rootfs=<path>]
         [-A|--arch=<arch of the container>]
         [-h|--help]
 Mandatory args:
@@ -353,12 +353,13 @@ Optional args:
   -d,--dns          specify the DNS server, eg. 192.168.1.2
   -P,--profile      Profile name is the file name in /etc/lxc/profiles contained packages name for install to cache.
   -A,--arch         NOT USED YET. Define what arch the container will be [i686,x86_64]
+  ---rootfs         rootfs path
   -h,--help         print this help
 EOF
     return 0
 }
 
-options=$(getopt -o hp:n:P:cR:4:6:g:d: -l help,path:,name:,profile:,clean,release:ipv4:ipv6:gw:dns: -- "$@")
+options=$(getopt -o hp:n:P:cR:4:6:g:d: -l help,rootfs:,path:,name:,profile:,clean,release:ipv4:ipv6:gw:dns: -- "$@")
 if [ $? -ne 0 ]; then
     usage $(basename $0)
     exit 1
@@ -370,6 +371,7 @@ do
     case "$1" in
         -h|--help)      usage $0 && exit 0;;
         -p|--path)      path=$2; shift 2;;
+        --rootfs)       rootfs_path=$2; shift 2;;
         -n|--name)      name=$2; shift 2;;
         -P|--profile)   profile=$2; shift 2;;
         -c|--clean)     clean=$2; shift 2;;
@@ -422,7 +424,15 @@ if [ "$(id -u)" != "0" ]; then
     exit 1
 fi
 
-rootfs_path=$path/$name/rootfs
+# check for 'lxc.rootfs' passed in through default config by lxc-create
+if [ -z "$rootfs_path" ]; then
+    if grep -q '^lxc.rootfs' $path/config 2>/dev/null ; then
+        rootfs_path=`grep 'lxc.rootfs =' $path/config | awk -F= '{ print $2 }'`
+    else
+        rootfs_path=$path/$name/rootfs
+    fi
+fi
+
 config_path=$default_path/$name
 cache=$cache_base/$release/$profile
 
@@ -431,11 +441,6 @@ if [ -f $config_path/config ]; then
     exit 1
 fi
 
-# check for 'lxc.rootfs' passed in through default config by lxc-create
-if grep -q '^lxc.rootfs' $path/config 2>/dev/null ; then
-    rootfs_path=`grep 'lxc.rootfs =' $path/config | awk -F= '{ print $2 }'`
-fi
-
 install_altlinux
 if [ $? -ne 0 ]; then
     echo "failed to install altlinux"
diff --git a/templates/lxc-archlinux.in b/templates/lxc-archlinux.in
index 98d54242a..64f16e8a8 100644
--- a/templates/lxc-archlinux.in
+++ b/templates/lxc-archlinux.in
@@ -125,7 +125,6 @@ lxc.utsname=${name}
 lxc.autodev=1
 lxc.tty=1
 lxc.pts=1024
-lxc.rootfs=${rootfs_path}
 lxc.mount=${config_path}/fstab
 lxc.cap.drop=mknod sys_module mac_admin mac_override sys_time
 lxc.kmsg=0
@@ -153,6 +152,8 @@ lxc.cgroup.devices.allow = c 5:2 rwm
 lxc.cgroup.devices.allow = c 136:* rwm
 EOF
 
+    grep -q "^lxc.rootfs" ${config_path}/config 2>/dev/null || echo "lxc.rootfs = ${rootfs_path}" >> ${config_path}/config
+
     cat > "${config_path}/fstab" << EOF
 sysfs sys sysfs ro,defaults 0 0
 proc proc proc nodev,noexec,nosuid 0 0
@@ -191,7 +192,7 @@ EOF
     return 0
 }
 
-options=$(getopt -o hp:P:n:c:l:t: -l help,path:,packages:,name:,config:,network_type:,network_link: -- "${@}")
+options=$(getopt -o hp:P:n:c:l:t: -l help,rootfs:,path:,packages:,name:,config:,network_type:,network_link: -- "${@}")
 if [ ${?} -ne 0 ]; then
     usage $(basename ${0})
     exit 1
@@ -204,6 +205,7 @@ do
     -h|--help)          usage ${0} && exit 0;;
     -p|--path)          path=${2}; shift 2;;
     -n|--name)          name=${2}; shift 2;;
+    --rootfs)           rootfs_path=${2}; shift 2;;
     -P|--packages)      additional_packages=${2}; shift 2;;
     -c|--config)        pacman_config=${2}; shift 2;;
     -t|--network_type)  lxc_network_type=${2}; shift 2;;
@@ -238,7 +240,9 @@ if [ "${EUID}" != "0" ]; then
     exit 1
 fi
 
-rootfs_path="${path}/rootfs"
+if [ -z "$rootfs_path" ]; then
+    rootfs_path="${path}/rootfs"
+fi
 config_path="${default_path}/${name}"
 
 revert() {
diff --git a/templates/lxc-busybox.in b/templates/lxc-busybox.in
index db39b0e62..e224aadce 100644
--- a/templates/lxc-busybox.in
+++ b/templates/lxc-busybox.in
@@ -288,7 +288,7 @@ EOF
     return 0
 }
 
-options=$(getopt -o hp:n: -l help,path:,name: -- "$@")
+options=$(getopt -o hp:n: -l help,rootfs:,path:,name: -- "$@")
 if [ $? -ne 0 ]; then
     usage $(basename $0)
     exit 1
@@ -300,6 +300,7 @@ do
     case "$1" in
         -h|--help)      usage $0 && exit 0;;
         -p|--path)      path=$2; shift 2;;
+        --rootfs)       rootfs=$2; shift 2;;
         -n|--name)      name=$2; shift 2;;
         --)             shift 1; break ;;
         *)              break ;;
@@ -318,10 +319,12 @@ fi
 
 # detect rootfs
 config="$path/config"
-if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
-    rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
-else
-    rootfs=$path/rootfs
+if [ -z "$rootfs" ]; then
+    if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
+        rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
+    else
+        rootfs=$path/rootfs
+    fi
 fi
 
 install_busybox $rootfs $name
diff --git a/templates/lxc-debian.in b/templates/lxc-debian.in
index d4ea3de59..9ae1ea728 100644
--- a/templates/lxc-debian.in
+++ b/templates/lxc-debian.in
@@ -284,7 +284,7 @@ EOF
     return 0
 }
 
-options=$(getopt -o hp:n:c -l help,path:,name:,clean -- "$@")
+options=$(getopt -o hp:n:c -l help,rootfs:,path:,name:,clean -- "$@")
 if [ $? -ne 0 ]; then
         usage $(basename $0)
         exit 1
@@ -296,6 +296,7 @@ do
     case "$1" in
         -h|--help)      usage $0 && exit 0;;
         -p|--path)      path=$2; shift 2;;
+        --rootfs)       rootfs=$2; shift 2;;
         -n|--name)      name=$2; shift 2;;
         -c|--clean)     clean=$2; shift 2;;
         --)             shift 1; break ;;
@@ -326,10 +327,12 @@ fi
 
 # detect rootfs
 config="$path/config"
-if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
-    rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
-else
-    rootfs=$path/rootfs
+if [ -z "$rootfs" ]; then
+    if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
+        rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
+    else
+        rootfs=$path/rootfs
+    fi
 fi
 
 
diff --git a/templates/lxc-fedora.in b/templates/lxc-fedora.in
index 871ae7a71..f5da7b52c 100644
--- a/templates/lxc-fedora.in
+++ b/templates/lxc-fedora.in
@@ -360,7 +360,8 @@ usage:
 Mandatory args:
   -n,--name         container name, used to as an identifier for that container from now on
 Optional args:
-  -p,--path         path to where the container rootfs will be created, defaults to @LXCPATH@. The container config will go under @LXCPATH@ in that case
+  -p,--path         path to where the container will be created, defaults to @LXCPATH@. The container config will go under @LXCPATH@ in that case
+  --rootfs          path for actual rootfs.
   -c,--clean        clean the cache
   -R,--release      Fedora release for the new container. if the host is Fedora, then it will default to the host's release.
   -A,--arch         NOT USED YET. Define what arch the container will be [i686,x86_64]
@@ -369,7 +370,7 @@ EOF
     return 0
 }
 
-options=$(getopt -o hp:n:cR: -l help,path:,name:,clean,release: -- "$@")
+options=$(getopt -o hp:n:cR: -l help,path:,rootfs:,name:,clean,release: -- "$@")
 if [ $? -ne 0 ]; then
     usage $(basename $0)
     exit 1
@@ -381,6 +382,7 @@ do
     case "$1" in
         -h|--help)      usage $0 && exit 0;;
         -p|--path)      path=$2; shift 2;;
+        --rootfs)       rootfs=$2; shift 2;;
         -n|--name)      name=$2; shift 2;;
         -c|--clean)     clean=$2; shift 2;;
         -R|--release)   release=$2; shift 2;;
@@ -438,10 +440,12 @@ if [ "$(id -u)" != "0" ]; then
 fi
 
 
-rootfs_path=$path/rootfs
-# check for 'lxc.rootfs' passed in through default config by lxc-create
-if grep -q '^lxc.rootfs' $path/config 2>/dev/null ; then
-    rootfs_path=`grep 'lxc.rootfs =' $path/config | awk -F= '{ print $2 }'`
+if [ -z "$rootfs_path" ]; then
+    rootfs_path=$path/rootfs
+    # check for 'lxc.rootfs' passed in through default config by lxc-create
+    if grep -q '^lxc.rootfs' $path/config 2>/dev/null ; then
+        rootfs_path=`grep 'lxc.rootfs =' $path/config | awk -F= '{ print $2 }'`
+    fi
 fi
 config_path=$default_path/$name
 cache=$cache_base/$release
diff --git a/templates/lxc-opensuse.in b/templates/lxc-opensuse.in
index 7d3dd1cad..8d9523699 100644
--- a/templates/lxc-opensuse.in
+++ b/templates/lxc-opensuse.in
@@ -342,7 +342,7 @@ EOF
     return 0
 }
 
-options=$(getopt -o hp:n:c -l help,path:,name:,clean -- "$@")
+options=$(getopt -o hp:n:c -l help,rootfs:,path:,name:,clean -- "$@")
 if [ $? -ne 0 ]; then
     usage $(basename $0)
     exit 1
@@ -354,6 +354,7 @@ do
     case "$1" in
         -h|--help)      usage $0 && exit 0;;
         -p|--path)      path=$2; shift 2;;
+        --rootfs)       rootfs=$2; shift 2;;
         -n|--name)      name=$2; shift 2;;
         -c|--clean)     clean=$2; shift 2;;
         --)             shift 1; break ;;
@@ -384,10 +385,12 @@ fi
 
 # detect rootfs
 config="$path/config"
-if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
-    rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
-else
-    rootfs=$path/rootfs
+if [ -z "$rootfs" ]; then
+    if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
+        rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
+    else
+        rootfs=$path/rootfs
+    fi
 fi
 
 install_opensuse $rootfs
diff --git a/templates/lxc-oracle.in b/templates/lxc-oracle.in
index 70ef63245..b09131e09 100644
--- a/templates/lxc-oracle.in
+++ b/templates/lxc-oracle.in
@@ -348,7 +348,6 @@ lxc.utsname = $name
 lxc.devttydir = lxc
 lxc.tty = 4
 lxc.pts = 1024
-lxc.rootfs = $container_rootfs
 lxc.mount = $cfg_dir/fstab
 # Uncomment these if you don't run anything that needs the capability, and
 # would like the container to run with less privilege.
@@ -370,6 +369,7 @@ lxc.cap.drop = mac_admin mac_override setfcap setpcap
 lxc.cap.drop = sys_module sys_nice sys_pacct
 lxc.cap.drop = sys_rawio sys_time
 EOF
+    grep -q "^lxc.rootfs" $cfg_dir/config 2>/dev/null || echo "lxc.rootfs = $container_rootfs" >> $cfg_dir/config
 
     if [ $container_release_major != "4" ]; then
         echo "lxc.cap.drop = sys_resource" >>$cfg_dir/config
@@ -610,6 +610,7 @@ usage()
     cat <<EOF
   -a|--arch=<arch>        architecture (ie. i386, x86_64)
   -R|--release=<release>  release to download for the new container
+  --rootfs=<path>         rootfs path
   -r|--rpms=<rpm name>    additional rpms to install into container
   -u|--url=<url>          replace yum repo url (ie. local yum mirror)
   -t|--templatefs=<path>  copy/clone rootfs at path instead of downloading
@@ -620,7 +621,7 @@ EOF
     return 0
 }
 
-options=$(getopt -o hp:n:a:R:r:u:t: -l help,path:,name:,arch:,release:,rpms:,url:,templatefs: -- "$@")
+options=$(getopt -o hp:n:a:R:r:u:t: -l help,rootfs:,path:,name:,arch:,release:,rpms:,url:,templatefs: -- "$@")
 if [ $? -ne 0 ]; then
     usage $(basename $0)
     exit 1
@@ -633,6 +634,7 @@ do
     case "$1" in
         -h|--help)		usage $0 && exit 0;;
         -p|--path)		cfg_dir=$2; shift 2;;
+        --rootfs)		container_rootfs=$2; shift 2;;
         -n|--name)		name=$2; shift 2;;
         -a|--arch)		arch=$2; shift 2;;
         -R|--release)		container_release_version=$2; shift 2;;
@@ -696,7 +698,9 @@ else
 fi
 echo "Host is $host_distribution $host_release_version"
 
-container_rootfs="$cfg_dir/rootfs"
+if [ -z "$container_rootfs" ]; then
+    container_rootfs="$cfg_dir/rootfs"
+fi
 
 if [ -n "$template_rootfs" ]; then
     container_release_get $template_rootfs
diff --git a/templates/lxc-sshd.in b/templates/lxc-sshd.in
index 2927c9295..5400156ba 100644
--- a/templates/lxc-sshd.in
+++ b/templates/lxc-sshd.in
@@ -140,12 +140,12 @@ EOF
 usage()
 {
     cat <<EOF
-$1 -h|--help -p|--path=<path>
+$1 -h|--help -p|--path=<path> [--rootfs=<path>]
 EOF
     return 0
 }
 
-options=$(getopt -o hp:n:S: -l help,path:,name:,auth-key: -- "$@")
+options=$(getopt -o hp:n:S: -l help,rootfs:,path:,name:,auth-key: -- "$@")
 if [ $? -ne 0 ]; then
         usage $(basename $0)
     exit 1
@@ -157,6 +157,7 @@ do
     case "$1" in
         -h|--help)      usage $0 && exit 0;;
         -p|--path)      path=$2; shift 2;;
+        --rootfs)       rootfs=$2; shift 2;;
         -n|--name)      name=$2; shift 2;;
         -S|--auth-key)  auth_key=$2; shift 2;;
         --)             shift 1; break ;;
@@ -210,10 +211,12 @@ fi
 
 # detect rootfs
 config="$path/config"
-if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
-    rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
-else
-    rootfs=$path/rootfs
+if [ -z "$rootfs" ]; then
+    if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
+        rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
+    else
+        rootfs=$path/rootfs
+    fi
 fi
 
 install_sshd $rootfs
diff --git a/templates/lxc-ubuntu-cloud.in b/templates/lxc-ubuntu-cloud.in
index 7a56398d0..37a060549 100644
--- a/templates/lxc-ubuntu-cloud.in
+++ b/templates/lxc-ubuntu-cloud.in
@@ -116,6 +116,7 @@ LXC Container configuration for Ubuntu Cloud images.
 
 Generic Options
 [ -r | --release <release> ]: Release name of container, defaults to host
+[ --rootfs <path> ]: Path in which rootfs will be placed
 [ -a | --arch ]: Arhcitecture of container, defaults to host arcitecture
 [ -C | --cloud ]: Configure container for use with meta-data service, defaults to no
 [ -T | --tarball ]: Location of tarball
@@ -132,7 +133,7 @@ EOF
     return 0
 }
 
-options=$(getopt -o a:hp:r:n:Fi:CLS:T:ds:u: -l arch:,help,path:,release:,name:,flush-cache,hostid:,auth-key:,cloud,no_locales,tarball:,debug,stream:,userdata: -- "$@")
+options=$(getopt -o a:hp:r:n:Fi:CLS:T:ds:u: -l arch:,help,rootfs:,path:,release:,name:,flush-cache,hostid:,auth-key:,cloud,no_locales,tarball:,debug,stream:,userdata: -- "$@")
 if [ $? -ne 0 ]; then
     usage $(basename $0)
     exit 1
@@ -266,10 +267,12 @@ fi
 
 # detect rootfs
 config="$path/config"
-if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
-    rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
-else
-    rootfs=$path/rootfs
+if [ -z "$rootfs" ]; then
+    if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
+        rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
+    else
+        rootfs=$path/rootfs
+    fi
 fi
 
 type ubuntu-cloudimg-query
diff --git a/templates/lxc-ubuntu.in b/templates/lxc-ubuntu.in
index 02ffa199d..1623b750b 100644
--- a/templates/lxc-ubuntu.in
+++ b/templates/lxc-ubuntu.in
@@ -622,6 +622,7 @@ usage()
     cat <<EOF
 $1 -h|--help [-a|--arch] [-b|--bindhome <user>] [--trim] [-d|--debug]
    [-F | --flush-cache] [-r|--release <release>] [ -S | --auth-key <keyfile>]
+   [--rootfs <rootfs>]
 release: the ubuntu release (e.g. precise): defaults to host release on ubuntu, otherwise uses latest LTS
 trim: make a minimal (faster, but not upgrade-safe) container
 bindhome: bind <user>'s home into the container
@@ -633,7 +634,7 @@ EOF
     return 0
 }
 
-options=$(getopt -o a:b:hp:r:xn:FS:d -l arch:,bindhome:,help,path:,release:,trim,name:,flush-cache,auth-key:,debug -- "$@")
+options=$(getopt -o a:b:hp:r:xn:FS:d -l arch:,bindhome:,help,path:,release:,trim,name:,flush-cache,auth-key:,debug,rootfs: -- "$@")
 if [ $? -ne 0 ]; then
     usage $(basename $0)
     exit 1
@@ -674,6 +675,7 @@ while true
 do
     case "$1" in
     -h|--help)      usage $0 && exit 0;;
+    --rootfs)       rootfs=$2; shift 2;;
     -p|--path)      path=$2; shift 2;;
     -n|--name)      name=$2; shift 2;;
     -F|--flush-cache) flushcache=1; shift 1;;
@@ -735,10 +737,13 @@ fi
 
 # detect rootfs
 config="$path/config"
-if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
-    rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
-else
-    rootfs=$path/rootfs
+# if $rootfs exists here, it was passed in with --rootfs
+if [ -z "$rootfs" ]; then
+    if grep -q '^lxc.rootfs' $config 2>/dev/null ; then
+        rootfs=`grep 'lxc.rootfs =' $config | awk -F= '{ print $2 }'`
+    else
+        rootfs=$path/rootfs
+    fi
 fi
 
 install_ubuntu $rootfs $release $flushcache