From: Christian Brauner Date: Wed, 30 Dec 2015 16:02:54 +0000 (+0100) Subject: Split bdev into modules: lxcnbd X-Git-Tag: lxc-2.0.0.beta2~65^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bf76c012b3f0e49930d207fe77ba6cc3919728ac;p=thirdparty%2Flxc.git Split bdev into modules: lxcnbd The functions - clone_attach_nbd(); - do_attach_nbd(); - nbd_busy(); - nbd_detach(); - nbd_get_partition(); - wait_for_partition(); move from bdev.c to lxcnbd.c. They remain static The functions - attach_nbd(); - detach_nbd_idx(); - nbd_clonepaths(); - nbd_create(); - nbd_destroy(); - nbd_detect(); - nbd_mount(); - nbd_umount(); - requires_nbd(); move from bdev.c to lxcnbd.{c,h}. They all become extern. The struct - struct nbd_attach_data moves from bdev.c to lxcnbd.c. Adapt Makefile.am to include lxcnbd.{c,h}. The structs - struct bdev; /* defined in bdev.h */ - struct bdev_specs; /* defined in lxccontainer.h */ - struct lxc_conf; /* defined conf.h */ are forward declared/put as incomplete types into lxcnbd.h as the functions associated with nbd need access to it. Signed-off-by: Christian Brauner --- diff --git a/src/lxc/Makefile.am b/src/lxc/Makefile.am index 1816ac72d..6fc812f04 100644 --- a/src/lxc/Makefile.am +++ b/src/lxc/Makefile.am @@ -12,6 +12,7 @@ noinst_HEADERS = \ bdev/lxcdir.h \ bdev/lxcloop.h \ bdev/lxclvm.h \ + bdev/lxcnbd.h \ bdev/lxcoverlay.h \ bdev/lxcrbd.h \ bdev/lxcrsync.h \ @@ -74,6 +75,7 @@ liblxc_so_SOURCES = \ bdev/lxcdir.c bdev/lxcdir.h \ bdev/lxcloop.c bdev/lxcloop.h \ bdev/lxclvm.c bdev/lxclvm.h \ + bdev/lxcnbd.c bdev/lxcnbd.h \ bdev/lxcoverlay.c bdev/lxcoverlay.h \ bdev/lxcrbd.c bdev/lxcrbd.h \ bdev/lxcrsync.c bdev/lxcrsync.h \ diff --git a/src/lxc/bdev/bdev.c b/src/lxc/bdev/bdev.c index 54a2284f2..96b0a5967 100644 --- a/src/lxc/bdev/bdev.c +++ b/src/lxc/bdev/bdev.c @@ -58,6 +58,7 @@ #include "lxclock.h" #include "lxclvm.h" #include "lxcloop.h" +#include "lxcnbd.h" #include "lxcoverlay.h" #include "lxcrbd.h" #include "lxcrsync.h" @@ -140,6 +141,18 @@ static const struct bdev_ops lvm_ops = { .can_backup = false, }; +/* nbd */ +const struct bdev_ops nbd_ops = { + .detect = &nbd_detect, + .mount = &nbd_mount, + .umount = &nbd_umount, + .clone_paths = &nbd_clonepaths, + .destroy = &nbd_destroy, + .create = &nbd_create, + .can_snapshot = true, + .can_backup = false, +}; + /* overlay */ static const struct bdev_ops ovl_ops = { .detect = &ovl_detect, @@ -176,33 +189,13 @@ static const struct bdev_ops zfs_ops = { .can_backup = true, }; -/* functions associated with an nbd bdev struct */ -static bool attach_nbd(char *src, struct lxc_conf *conf); -static bool clone_attach_nbd(const char *nbd, const char *path); -static int do_attach_nbd(void *d); -static int nbd_get_partition(const char *src); -static bool nbd_busy(int idx); -static int nbd_clonepaths(struct bdev *orig, struct bdev *new, - const char *oldname, const char *cname, - const char *oldpath, const char *lxcpath, int snap, - uint64_t newsize, struct lxc_conf *conf); -static int nbd_create(struct bdev *bdev, const char *dest, const char *n, - struct bdev_specs *specs); -static void nbd_detach(const char *path); -static int nbd_destroy(struct bdev *orig); -static int nbd_detect(const char *path); -static int nbd_mount(struct bdev *bdev); -static int nbd_umount(struct bdev *bdev); -static bool requires_nbd(const char *path); -static bool wait_for_partition(const char *path); - /* helpers */ /* * These are copied from conf.c. However as conf.c will be moved to using * the callback system, they can be pulled from there eventually, so we * don't need to pollute utils.c with these low level functions */ -static int find_fstype_cb(char* buffer, void *data); +static int find_fstype_cb(char *buffer, void *data); static char *linkderef(char *path, char *dest); static bool unpriv_snap_allowed(struct bdev *b, const char *t, bool snap, bool maybesnap); @@ -512,317 +505,6 @@ int is_blktype(struct bdev *b) return 0; } -// -// nbd dev ops -// - -static int nbd_detect(const char *path) -{ - if (strncmp(path, "nbd:", 4) == 0) - return 1; - return 0; -} - -struct nbd_attach_data { - const char *nbd; - const char *path; -}; - -static void nbd_detach(const char *path) -{ - int ret; - pid_t pid = fork(); - - if (pid < 0) { - SYSERROR("Error forking to detach nbd"); - return; - } - if (pid) { - ret = wait_for_pid(pid); - if (ret < 0) - ERROR("nbd disconnect returned an error"); - return; - } - execlp("qemu-nbd", "qemu-nbd", "-d", path, NULL); - SYSERROR("Error executing qemu-nbd"); - exit(1); -} - -static int do_attach_nbd(void *d) -{ - struct nbd_attach_data *data = d; - const char *nbd, *path; - pid_t pid; - sigset_t mask; - int sfd; - ssize_t s; - struct signalfd_siginfo fdsi; - - sigemptyset(&mask); - sigaddset(&mask, SIGHUP); - sigaddset(&mask, SIGCHLD); - - nbd = data->nbd; - path = data->path; - - if (sigprocmask(SIG_BLOCK, &mask, NULL) == -1) { - SYSERROR("Error blocking signals for nbd watcher"); - exit(1); - } - - sfd = signalfd(-1, &mask, 0); - if (sfd == -1) { - SYSERROR("Error opening signalfd for nbd task"); - exit(1); - } - - if (prctl(PR_SET_PDEATHSIG, SIGHUP, 0, 0, 0) < 0) - SYSERROR("Error setting parent death signal for nbd watcher"); - - pid = fork(); - if (pid) { - for (;;) { - s = read(sfd, &fdsi, sizeof(struct signalfd_siginfo)); - if (s != sizeof(struct signalfd_siginfo)) - SYSERROR("Error reading from signalfd"); - - if (fdsi.ssi_signo == SIGHUP) { - /* container has exited */ - nbd_detach(nbd); - exit(0); - } else if (fdsi.ssi_signo == SIGCHLD) { - int status; - /* If qemu-nbd fails, or is killed by a signal, - * then exit */ - while (waitpid(-1, &status, WNOHANG) > 0) { - if ((WIFEXITED(status) && WEXITSTATUS(status) != 0) || - WIFSIGNALED(status)) { - nbd_detach(nbd); - exit(1); - } - } - } - } - } - - close(sfd); - if (sigprocmask(SIG_UNBLOCK, &mask, NULL) == -1) - WARN("Warning: unblocking signals for nbd watcher"); - - execlp("qemu-nbd", "qemu-nbd", "-c", nbd, path, NULL); - SYSERROR("Error executing qemu-nbd"); - exit(1); -} - -static bool clone_attach_nbd(const char *nbd, const char *path) -{ - pid_t pid; - struct nbd_attach_data data; - - data.nbd = nbd; - data.path = path; - - pid = lxc_clone(do_attach_nbd, &data, CLONE_NEWPID); - if (pid < 0) - return false; - return true; -} - -static bool nbd_busy(int idx) -{ - char path[100]; - int ret; - - ret = snprintf(path, 100, "/sys/block/nbd%d/pid", idx); - if (ret < 0 || ret >= 100) - return true; - return file_exists(path); -} - -static bool attach_nbd(char *src, struct lxc_conf *conf) -{ - char *orig = alloca(strlen(src)+1), *p, path[50]; - int i = 0; - - strcpy(orig, src); - /* if path is followed by a partition, drop that for now */ - p = strchr(orig, ':'); - if (p) - *p = '\0'; - while (1) { - sprintf(path, "/dev/nbd%d", i); - if (!file_exists(path)) - return false; - if (nbd_busy(i)) { - i++; - continue; - } - if (!clone_attach_nbd(path, orig)) - return false; - conf->nbd_idx = i; - return true; - } -} - -static bool requires_nbd(const char *path) -{ - if (strncmp(path, "nbd:", 4) == 0) - return true; - return false; -} - -/* - * attach_block_device returns true if all went well, - * meaning either a block device was attached or was not - * needed. It returns false if something went wrong and - * container startup should be stopped. - */ -bool attach_block_device(struct lxc_conf *conf) -{ - char *path; - - if (!conf->rootfs.path) - return true; - path = conf->rootfs.path; - if (!requires_nbd(path)) - return true; - path = strchr(path, ':'); - if (!path) - return false; - path++; - if (!attach_nbd(path, conf)) - return false; - return true; -} - -void detach_nbd_idx(int idx) -{ - int ret; - char path[50]; - - ret = snprintf(path, 50, "/dev/nbd%d", idx); - if (ret < 0 || ret >= 50) - return; - - nbd_detach(path); -} - -void detach_block_device(struct lxc_conf *conf) -{ - if (conf->nbd_idx != -1) - detach_nbd_idx(conf->nbd_idx); -} - -/* - * Pick the partition # off the end of a nbd:file:p - * description. Return 1-9 for the partition id, or 0 - * for no partition. - */ -static int nbd_get_partition(const char *src) -{ - char *p = strchr(src, ':'); - if (!p) - return 0; - p = strchr(p+1, ':'); - if (!p) - return 0; - p++; - if (*p < '1' || *p > '9') - return 0; - return *p - '0'; -} - -static bool wait_for_partition(const char *path) -{ - int count = 0; - while (count < 5) { - if (file_exists(path)) - return true; - sleep(1); - count++; - } - ERROR("Device %s did not show up after 5 seconds", path); - return false; -} - -static int nbd_mount(struct bdev *bdev) -{ - int ret = -1, partition; - char path[50]; - - if (strcmp(bdev->type, "nbd")) - return -22; - if (!bdev->src || !bdev->dest) - return -22; - - /* nbd_idx should have been copied by bdev_init from the lxc_conf */ - if (bdev->nbd_idx < 0) - return -22; - partition = nbd_get_partition(bdev->src); - if (partition) - ret = snprintf(path, 50, "/dev/nbd%dp%d", bdev->nbd_idx, - partition); - else - ret = snprintf(path, 50, "/dev/nbd%d", bdev->nbd_idx); - if (ret < 0 || ret >= 50) { - ERROR("Error setting up nbd device path"); - return ret; - } - - /* It might take awhile for the partition files to show up */ - if (partition) { - if (!wait_for_partition(path)) - return -2; - } - ret = mount_unknown_fs(path, bdev->dest, bdev->mntopts); - if (ret < 0) - ERROR("Error mounting %s", bdev->src); - - return ret; -} - -static int nbd_create(struct bdev *bdev, const char *dest, const char *n, - struct bdev_specs *specs) -{ - return -ENOSYS; -} - -static int nbd_clonepaths(struct bdev *orig, struct bdev *new, - const char *oldname, const char *cname, - const char *oldpath, const char *lxcpath, int snap, - uint64_t newsize, struct lxc_conf *conf) -{ - return -ENOSYS; -} - -static int nbd_destroy(struct bdev *orig) -{ - return -ENOSYS; -} - -static int nbd_umount(struct bdev *bdev) -{ - int ret; - - if (strcmp(bdev->type, "nbd")) - return -22; - if (!bdev->src || !bdev->dest) - return -22; - ret = umount(bdev->dest); - return ret; -} - -static const struct bdev_ops nbd_ops = { - .detect = &nbd_detect, - .mount = &nbd_mount, - .umount = &nbd_umount, - .clone_paths = &nbd_clonepaths, - .destroy = &nbd_destroy, - .create = &nbd_create, - .can_snapshot = true, - .can_backup = false, -}; - static const struct bdev_type bdevs[] = { {.name = "zfs", .ops = &zfs_ops,}, {.name = "lvm", .ops = &lvm_ops,}, @@ -1249,3 +931,33 @@ int bdev_destroy_wrapper(void *data) else return 0; } + +/* + * attach_block_device returns true if all went well, + * meaning either a block device was attached or was not + * needed. It returns false if something went wrong and + * container startup should be stopped. + */ +bool attach_block_device(struct lxc_conf *conf) +{ + char *path; + + if (!conf->rootfs.path) + return true; + path = conf->rootfs.path; + if (!requires_nbd(path)) + return true; + path = strchr(path, ':'); + if (!path) + return false; + path++; + if (!attach_nbd(path, conf)) + return false; + return true; +} + +void detach_block_device(struct lxc_conf *conf) +{ + if (conf->nbd_idx != -1) + detach_nbd_idx(conf->nbd_idx); +} diff --git a/src/lxc/bdev/bdev.h b/src/lxc/bdev/bdev.h index 8ad9272c4..91b9c8de0 100644 --- a/src/lxc/bdev/bdev.h +++ b/src/lxc/bdev/bdev.h @@ -128,15 +128,6 @@ bool bdev_destroy(struct lxc_conf *conf); /* callback function to be used with userns_exec_1() */ int bdev_destroy_wrapper(void *data); -/* - * these are really for qemu-nbd support, as container shutdown - * must explicitly request device detach. - */ -bool attach_block_device(struct lxc_conf *conf); -void detach_block_device(struct lxc_conf *conf); - -bool rootfs_is_blockdev(struct lxc_conf *conf); - /* Some helpers for lvm, rdb, and/or loop: * Maybe they should move to a separate implementation and header-file * (bdev_utils.{c,h}) which can be included in bdev.c? @@ -147,6 +138,12 @@ int do_mkfs(const char *path, const char *fstype); int is_blktype(struct bdev *b); int mount_unknown_fs(const char *rootfs, const char *target, const char *options); - +bool rootfs_is_blockdev(struct lxc_conf *conf); +/* + * these are really for qemu-nbd support, as container shutdown + * must explicitly request device detach. + */ +bool attach_block_device(struct lxc_conf *conf); +void detach_block_device(struct lxc_conf *conf); #endif // __LXC_BDEV_H diff --git a/src/lxc/bdev/lxcnbd.c b/src/lxc/bdev/lxcnbd.c new file mode 100644 index 000000000..2f68a00cf --- /dev/null +++ b/src/lxc/bdev/lxcnbd.c @@ -0,0 +1,311 @@ +/* + * lxc: linux Container library + * + * (C) Copyright IBM Corp. 2007, 2008 + * + * Authors: + * Daniel Lezcano + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bdev.h" +#include "log.h" +#include "lxcnbd.h" +#include "utils.h" + +lxc_log_define(lxcnbd, lxc); + +struct nbd_attach_data { + const char *nbd; + const char *path; +}; + +static bool clone_attach_nbd(const char *nbd, const char *path); +static int do_attach_nbd(void *d); +static bool nbd_busy(int idx); +static void nbd_detach(const char *path); +static int nbd_get_partition(const char *src); +static bool wait_for_partition(const char *path); + +bool attach_nbd(char *src, struct lxc_conf *conf) +{ + char *orig = alloca(strlen(src)+1), *p, path[50]; + int i = 0; + + strcpy(orig, src); + /* if path is followed by a partition, drop that for now */ + p = strchr(orig, ':'); + if (p) + *p = '\0'; + while (1) { + sprintf(path, "/dev/nbd%d", i); + if (!file_exists(path)) + return false; + if (nbd_busy(i)) { + i++; + continue; + } + if (!clone_attach_nbd(path, orig)) + return false; + conf->nbd_idx = i; + return true; + } +} + +void detach_nbd_idx(int idx) +{ + int ret; + char path[50]; + + ret = snprintf(path, 50, "/dev/nbd%d", idx); + if (ret < 0 || ret >= 50) + return; + + nbd_detach(path); +} + +int nbd_clonepaths(struct bdev *orig, struct bdev *new, const char *oldname, + const char *cname, const char *oldpath, const char *lxcpath, + int snap, uint64_t newsize, struct lxc_conf *conf) +{ + return -ENOSYS; +} + +int nbd_create(struct bdev *bdev, const char *dest, const char *n, + struct bdev_specs *specs) +{ + return -ENOSYS; +} + +int nbd_destroy(struct bdev *orig) +{ + return -ENOSYS; +} + +int nbd_detect(const char *path) +{ + if (strncmp(path, "nbd:", 4) == 0) + return 1; + return 0; +} + +int nbd_mount(struct bdev *bdev) +{ + int ret = -1, partition; + char path[50]; + + if (strcmp(bdev->type, "nbd")) + return -22; + if (!bdev->src || !bdev->dest) + return -22; + + /* nbd_idx should have been copied by bdev_init from the lxc_conf */ + if (bdev->nbd_idx < 0) + return -22; + partition = nbd_get_partition(bdev->src); + if (partition) + ret = snprintf(path, 50, "/dev/nbd%dp%d", bdev->nbd_idx, + partition); + else + ret = snprintf(path, 50, "/dev/nbd%d", bdev->nbd_idx); + if (ret < 0 || ret >= 50) { + ERROR("Error setting up nbd device path"); + return ret; + } + + /* It might take awhile for the partition files to show up */ + if (partition) { + if (!wait_for_partition(path)) + return -2; + } + ret = mount_unknown_fs(path, bdev->dest, bdev->mntopts); + if (ret < 0) + ERROR("Error mounting %s", bdev->src); + + return ret; +} + +int nbd_umount(struct bdev *bdev) +{ + int ret; + + if (strcmp(bdev->type, "nbd")) + return -22; + if (!bdev->src || !bdev->dest) + return -22; + ret = umount(bdev->dest); + return ret; +} + +bool requires_nbd(const char *path) +{ + if (strncmp(path, "nbd:", 4) == 0) + return true; + return false; +} + +static int do_attach_nbd(void *d) +{ + struct nbd_attach_data *data = d; + const char *nbd, *path; + pid_t pid; + sigset_t mask; + int sfd; + ssize_t s; + struct signalfd_siginfo fdsi; + + sigemptyset(&mask); + sigaddset(&mask, SIGHUP); + sigaddset(&mask, SIGCHLD); + + nbd = data->nbd; + path = data->path; + + if (sigprocmask(SIG_BLOCK, &mask, NULL) == -1) { + SYSERROR("Error blocking signals for nbd watcher"); + exit(1); + } + + sfd = signalfd(-1, &mask, 0); + if (sfd == -1) { + SYSERROR("Error opening signalfd for nbd task"); + exit(1); + } + + if (prctl(PR_SET_PDEATHSIG, SIGHUP, 0, 0, 0) < 0) + SYSERROR("Error setting parent death signal for nbd watcher"); + + pid = fork(); + if (pid) { + for (;;) { + s = read(sfd, &fdsi, sizeof(struct signalfd_siginfo)); + if (s != sizeof(struct signalfd_siginfo)) + SYSERROR("Error reading from signalfd"); + + if (fdsi.ssi_signo == SIGHUP) { + /* container has exited */ + nbd_detach(nbd); + exit(0); + } else if (fdsi.ssi_signo == SIGCHLD) { + int status; + /* If qemu-nbd fails, or is killed by a signal, + * then exit */ + while (waitpid(-1, &status, WNOHANG) > 0) { + if ((WIFEXITED(status) && WEXITSTATUS(status) != 0) || + WIFSIGNALED(status)) { + nbd_detach(nbd); + exit(1); + } + } + } + } + } + + close(sfd); + if (sigprocmask(SIG_UNBLOCK, &mask, NULL) == -1) + WARN("Warning: unblocking signals for nbd watcher"); + + execlp("qemu-nbd", "qemu-nbd", "-c", nbd, path, NULL); + SYSERROR("Error executing qemu-nbd"); + exit(1); +} + +static bool clone_attach_nbd(const char *nbd, const char *path) +{ + pid_t pid; + struct nbd_attach_data data; + + data.nbd = nbd; + data.path = path; + + pid = lxc_clone(do_attach_nbd, &data, CLONE_NEWPID); + if (pid < 0) + return false; + return true; +} + +static bool nbd_busy(int idx) +{ + char path[100]; + int ret; + + ret = snprintf(path, 100, "/sys/block/nbd%d/pid", idx); + if (ret < 0 || ret >= 100) + return true; + return file_exists(path); +} + +static void nbd_detach(const char *path) +{ + int ret; + pid_t pid = fork(); + + if (pid < 0) { + SYSERROR("Error forking to detach nbd"); + return; + } + if (pid) { + ret = wait_for_pid(pid); + if (ret < 0) + ERROR("nbd disconnect returned an error"); + return; + } + execlp("qemu-nbd", "qemu-nbd", "-d", path, NULL); + SYSERROR("Error executing qemu-nbd"); + exit(1); +} + +/* + * Pick the partition # off the end of a nbd:file:p + * description. Return 1-9 for the partition id, or 0 + * for no partition. + */ +static int nbd_get_partition(const char *src) +{ + char *p = strchr(src, ':'); + if (!p) + return 0; + p = strchr(p+1, ':'); + if (!p) + return 0; + p++; + if (*p < '1' || *p > '9') + return 0; + return *p - '0'; +} + +static bool wait_for_partition(const char *path) +{ + int count = 0; + while (count < 5) { + if (file_exists(path)) + return true; + sleep(1); + count++; + } + ERROR("Device %s did not show up after 5 seconds", path); + return false; +} diff --git a/src/lxc/bdev/lxcnbd.h b/src/lxc/bdev/lxcnbd.h new file mode 100644 index 000000000..1404d5a3b --- /dev/null +++ b/src/lxc/bdev/lxcnbd.h @@ -0,0 +1,58 @@ +/* + * lxc: linux Container library + * + * (C) Copyright IBM Corp. 2007, 2008 + * + * Authors: + * Daniel Lezcano + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef __LXC_NBD_H +#define __LXC_NBD_H + +#define _GNU_SOURCE +#include +#include + +/* defined in bdev.h */ +struct bdev; + +/* defined in lxccontainer.h */ +struct bdev_specs; + +/* defined conf.h */ +struct lxc_conf; + +/* + * Functions associated with an nbd bdev struct. + */ +int nbd_clonepaths(struct bdev *orig, struct bdev *new, const char *oldname, + const char *cname, const char *oldpath, const char *lxcpath, + int snap, uint64_t newsize, struct lxc_conf *conf); +int nbd_create(struct bdev *bdev, const char *dest, const char *n, + struct bdev_specs *specs); +int nbd_destroy(struct bdev *orig); +int nbd_detect(const char *path); +int nbd_mount(struct bdev *bdev); +int nbd_umount(struct bdev *bdev); + +/* helpers */ +bool attach_nbd(char *src, struct lxc_conf *conf); +void detach_nbd_idx(int idx); +bool requires_nbd(const char *path); + +#endif /* __LXC_NBD_H */