From: Peter Somogyi Date: Tue, 2 Jan 2007 17:16:39 +0000 (+0100) Subject: Joining ctdb and ibwrapper (incomplete). X-Git-Tag: tevent-0.9.20~348^2~2975^2~14 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=11fb3ef3b552556adb440774b24e5689cbe198bb;p=thirdparty%2Fsamba.git Joining ctdb and ibwrapper (incomplete). It compiles. TODO: modify ctdb as well. (This used to be ctdb commit a080b85a5d8e46946f8308bb6c76aa62b5a254db) --- 11fb3ef3b552556adb440774b24e5689cbe198bb diff --cc ctdb/common/ctdb.c index e4150f83ca9,e4150f83ca9..bd1e150d439 --- a/ctdb/common/ctdb.c +++ b/ctdb/common/ctdb.c @@@ -30,10 -30,10 +30,18 @@@ int ctdb_set_transport(struct ctdb_context *ctdb, const char *transport) { int ctdb_tcp_init(struct ctdb_context *ctdb); ++#ifdef HAVE_INFINIBAND ++ int ctdb_ibw_init(struct ctdb_context *ctdb); ++#endif /*HAVE_INFINIBAND*/ if (strcmp(transport, "tcp") == 0) { return ctdb_tcp_init(ctdb); } ++#ifdef HAVE_INFINIBAND ++ if (strcmp(transport, "ib") == 0) { ++ return ctdb_ibw_init(ctdb); ++ } ++#endif /*HAVE_INFINIBAND*/ ctdb_set_error(ctdb, "Unknown transport '%s'\n", transport); return -1; } diff --cc ctdb/ib/config.m4 index e84b7448a76,e84b7448a76..9d95ea7a5ab --- a/ctdb/ib/config.m4 +++ b/ctdb/ib/config.m4 @@@ -7,7 -7,7 +7,7 @@@ if eval "test x$enable_infiniband = xye AC_DEFINE(USE_INFINIBAND,1,[Use infiniband]) HAVE_INFINIBAND=yes -- INFINIBAND_WRAPPER_OBJ="ib/ibwrapper.o" ++ INFINIBAND_WRAPPER_OBJ="ib/ibwrapper.o ib/ibw_ctdb.o ib/ibw_ctdb_init.o" INFINIBAND_LIBS="-lrdmacm -libverbs" INFINIBAND_BINS="bin/ibwrapper_test" diff --cc ctdb/ib/ibw_ctdb.c index 00000000000,00000000000..4379cb75086 new file mode 100644 --- /dev/null +++ b/ctdb/ib/ibw_ctdb.c @@@ -1,0 -1,0 +1,105 @@@ ++/* ++ * Unix SMB/CIFS implementation. ++ * Join infiniband wrapper and ctdb. ++ * ++ * Copyright (C) Sven Oehme 2006 ++ * ++ * Major code contributions by Peter Somogyi ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, write to the Free Software ++ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. ++ */ ++ ++#include "includes.h" ++#include "lib/events/events.h" ++#include ++#include ++#include "ctdb_private.h" ++#include "ibwrapper.h" ++#include "ibw_ctdb.h" ++ ++int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn) ++{ ++ if (ctx!=NULL) { ++ /* ctx->state changed */ ++ switch(ctx->state) { ++ case IBWS_INIT: /* ctx start - after ibw_init */ ++ break; ++ case IBWS_READY: /* after ibw_bind & ibw_listen */ ++ break; ++ case IBWS_CONNECT_REQUEST: /* after [IBWS_READY + incoming request] */ ++ /* => [(ibw_accept)IBWS_READY | (ibw_disconnect)STOPPED | ERROR] */ ++ if (ibw_accept(ctx, conn, NULL)) { ++ DEBUG(0, ("connstate_handler/ibw_accept failed\n")); ++ return -1; ++ } /* else continue in IBWC_CONNECTED */ ++ break; ++ case IBWS_STOPPED: /* normal stop <= ibw_disconnect+(IBWS_READY | IBWS_CONNECT_REQUEST) */ ++ /* TODO: have a CTDB upcall for which CTDB should wait in a (final) loop */ ++ break; ++ case IBWS_ERROR: /* abnormal state; ibw_stop must be called after this */ ++ break; ++ default: ++ assert(0); ++ break; ++ } ++ } ++ ++ if (conn!=NULL) { ++ /* conn->state changed */ ++ switch(conn->state) { ++ case IBWC_INIT: /* conn start - internal state */ ++ break; ++ case IBWC_CONNECTED: { /* after ibw_accept or ibw_connect */ ++ struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node); ++ if (node!=NULL) /* after ibw_connect */ ++ node->ctdb->upcalls->node_connected(node); ++ else { /* after ibw_accept */ ++ /* NOP in CTDB case */ ++ } ++ } break; ++ case IBWC_DISCONNECTED: /* after ibw_disconnect */ ++ /* TODO: have a CTDB upcall */ ++ break; ++ case IBWC_ERROR: { ++ struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node); ++ if (node!=NULL) ++ node->ctdb->upcalls->node_connected(node); ++ } break; ++ default: ++ assert(0); ++ break; ++ } ++ } ++ ++ return 0; ++} ++ ++int ctdb_ibw_receive_handler(struct ibw_conn *conn, void *buf, int n) ++{ ++ struct ctdb_context *ctdb = talloc_get_type(conn->ctx->ctx_userdata, struct ctdb_context); ++ ++ assert(ctdb!=NULL); ++ assert(conn->state==IBWC_CONNECTED); ++ ++ /* TODO: shall I short-circuit this in ibwrapper? */ ++ /* maybe when everything go fine... */ ++ ++ /* TODO2: !!! here I can provide conn->conn_userdata (with no perf. penalty) - ++ * as struct ctdb_node in case the connection ++ * has been built up by ibw_connect !!! */ ++ ctdb->upcalls->recv_pkt(ctdb, (uint8_t *)buf, (uint32_t)n); ++ ++ return 0; ++} diff --cc ctdb/ib/ibw_ctdb.h index 00000000000,00000000000..c43aca7df15 new file mode 100644 --- /dev/null +++ b/ctdb/ib/ibw_ctdb.h @@@ -1,0 -1,0 +1,25 @@@ ++/* ++ * Unix SMB/CIFS implementation. ++ * Join infiniband wrapper and ctdb. ++ * ++ * Copyright (C) Sven Oehme 2006 ++ * ++ * Major code contributions by Peter Somogyi ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, write to the Free Software ++ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. ++ */ ++ ++int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn); ++int ctdb_ibw_receive_handler(struct ibw_conn *conn, void *buf, int n); diff --cc ctdb/ib/ibw_ctdb_init.c index 00000000000,00000000000..a6a3a8beffa new file mode 100644 --- /dev/null +++ b/ctdb/ib/ibw_ctdb_init.c @@@ -1,0 -1,0 +1,164 @@@ ++/* ++ * Unix SMB/CIFS implementation. ++ * Join infiniband wrapper and ctdb. ++ * ++ * Copyright (C) Sven Oehme 2006 ++ * ++ * Major code contributions by Peter Somogyi ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, write to the Free Software ++ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. ++ */ ++ ++#include "includes.h" ++#include "lib/events/events.h" ++#include ++#include ++#include "ctdb_private.h" ++#include "ibwrapper.h" ++#include "ibw_ctdb.h" ++ ++static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog) ++{ ++ struct ibw_ctx *ictx = talloc_get_type(ctdb->private, struct ibw_ctx); ++ struct sockaddr_in my_addr; ++ ++ assert(ictx!=NULL); ++ memset(&my_addr, 0, sizeof(struct sockaddr_in)); ++ my_addr.sin_port = htons(ctdb->address.port); ++ my_addr.sin_family = PF_INET; ++ inet_pton(AF_INET, ctdb->address.address, &my_addr.sin_addr); ++ ++ if (ibw_bind(ictx, &my_addr)) { ++ DEBUG(0, ("ctdb_ibw_listen: ibw_bind failed\n")); ++ return -1; ++ } ++ ++ if (ibw_listen(ictx, backlog)) { ++ DEBUG(0, ("ctdb_ibw_listen: ibw_listen failed\n")); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++int ctdb_ibw_node_connect(struct ibw_ctx *ictx, struct ctdb_node *node) ++{ ++ struct sockaddr_in sock_out; ++ ++ inet_pton(AF_INET, node->address.address, &sock_out.sin_addr); ++ sock_out.sin_port = htons(node->address.port); ++ sock_out.sin_family = PF_INET; ++ ++ if (ibw_connect(ictx, &sock_out, node)) { ++ DEBUG(0, ("ctdb_ibw_node_connect: ibw_connect failed\n")); ++ return -1; ++ } ++ ++ /* continues at ibw_ctdb.c/IBWC_CONNECTED in good case */ ++ return 0; ++} ++ ++/* ++ * Start infiniband ++ */ ++static int ctdb_ibw_start(struct ctdb_context *ctdb) ++{ ++ struct ibw_ctx *ictx = talloc_get_type(ctdb->private, struct ibw_ctx); ++ int i; ++ ++ /* listen on our own address */ ++ if (ctdb_ibw_listen(ctdb, 10)) /* TODO: backlog as param */ ++ return -1; ++ ++ /* everything async here */ ++ for (i=0;inum_nodes;i++) { ++ struct ctdb_node *node = ctdb->nodes[i]; ++ if (!(ctdb->flags & CTDB_FLAG_SELF_CONNECT) && ++ ctdb_same_address(&ctdb->address, &node->address)) ++ continue; ++ ctdb_ibw_node_connect(ictx, node); ++ } ++ ++ return 0; ++} ++ ++ ++/* ++ * initialise ibw portion of a ctdb node ++ */ ++static int ctdb_ibw_add_node(struct ctdb_node *node) ++{ ++ /* TODO: clarify whether is this necessary for us ? ++ - why not enough doing such thing internally at connect time ? */ ++ return 0; ++} ++ ++/* ++ * transport packet allocator - allows transport to control memory for packets ++ */ ++static void *ctdb_ibw_allocate_pkt(struct ctdb_context *ctdb, size_t size) ++{ ++ struct ibw_conn *conn = NULL; ++ void *buf = NULL; ++ void *key; /* TODO: expand the param list with this */ ++ ++ /* TODO2: !!! I need "node" or ibw_conn here */ ++ if (ibw_alloc_send_buf(conn, &buf, &key, (int)size)) ++ return NULL; ++ ++ return buf; ++} ++ ++static int ctdb_ibw_queue_pkt(struct ctdb_node *node, uint8_t *data, uint32_t length) ++{ ++ struct ibw_conn *conn = talloc_get_type(node->private, struct ibw_conn); ++ void *key = NULL; /* TODO: expand the param list with this */ ++ ++ assert(conn!=NULL); ++ return ibw_send(conn, data, key, length); ++} ++ ++static const struct ctdb_methods ctdb_ibw_methods = { ++ .start = ctdb_ibw_start, ++ .add_node = ctdb_ibw_add_node, ++ .queue_pkt = ctdb_ibw_queue_pkt, ++ .allocate_pkt = ctdb_ibw_allocate_pkt ++ /* TODO: missing node_disconnect & final_stop upcalls */ ++}; ++ ++/* ++ * initialise ibw portion of ctdb ++ */ ++int ctdb_ibw_init(struct ctdb_context *ctdb) ++{ ++ struct ibw_ctx *ictx; ++ ++ ictx = ibw_init( ++ NULL, //struct ibw_initattr *attr, /* TODO */ ++ 0, //int nattr, /* TODO */ ++ ctdb, ++ ctdb_ibw_connstate_handler, ++ ctdb_ibw_receive_handler, ++ ctdb->ev); ++ ++ if (ictx==NULL) { ++ DEBUG(0, ("ctdb_ibw_init: ibw_init failed\n")); ++ return -1; ++ } ++ ++ ctdb->methods = &ctdb_ibw_methods; ++ ctdb->private = ictx; ++ return 0; ++}