- changed the REQ_REGISTER PDU to be a control

- allow controls to know which client invoked them

- added a client_id to clients, so they can be identified remotely

- added the ability to remove registered srvids

- in the list_keys code, register a temp srvid, then remove it afterwards

(This used to be ctdb commit 29603c51cc6d81362532cd8e50f75c8360c5f5ef)

diff --git a/ctdb/common/ctdb_client.c b/ctdb/common/ctdb_client.c
index c8480ec378629167bede011867668071ba77649e..ee195569183f029198a421dfc82d3c0301262c6d 100644 (file)
--- a/ctdb/common/ctdb_client.c
+++ b/ctdb/common/ctdb_client.c
@@ -376,27 +376,38 @@ int ctdb_set_message_handler(struct ctdb_context *ctdb, uint64_t srvid,
                             void *private_data)
                                    
 {
-       struct ctdb_req_register *c;
        int res;
-
-       /* if the domain socket is not yet open, open it */
-       if (ctdb->daemon.sd==-1) {
-               ctdb_socket_connect(ctdb);
+       int32_t status;
+       
+       res = ctdb_control(ctdb, CTDB_CURRENT_NODE, srvid, CTDB_CONTROL_REGISTER_SRVID, 0, 
+                          tdb_null, NULL, NULL, &status, NULL);
+       if (res != 0 || status != 0) {
+               DEBUG(0,("Failed to register srvid %llu\n", (unsigned long long)srvid));
+               return -1;
        }
 
-       c = ctdbd_allocate_pkt(ctdb, ctdb, CTDB_REQ_REGISTER, sizeof(*c), 
-                              struct ctdb_req_register);
-       CTDB_NO_MEMORY(ctdb, c);
-       c->srvid            = srvid;
+       /* also need to register the handler with our own ctdb structure */
+       return ctdb_register_message_handler(ctdb, ctdb, srvid, handler, private_data);
+}
 
-       res = ctdb_client_queue_pkt(ctdb, &c->hdr);
-       talloc_free(c);
-       if (res != 0) {
-               return res;
+/*
+  tell the daemon we no longer want a srvid
+*/
+int ctdb_remove_message_handler(struct ctdb_context *ctdb, uint64_t srvid, void *private_data)
+{
+       int res;
+       int32_t status;
+       
+       res = ctdb_control(ctdb, CTDB_CURRENT_NODE, srvid, CTDB_CONTROL_DEREGISTER_SRVID, 0, 
+                          tdb_null, NULL, NULL, &status, NULL);
+       if (res != 0 || status != 0) {
+               DEBUG(0,("Failed to deregister srvid %llu\n", (unsigned long long)srvid));
+               return -1;
        }
 
-       /* also need to register the handler with our ctdb structure */
-       return ctdb_register_message_handler(ctdb, ctdb, srvid, handler, private_data);
+       /* also need to register the handler with our own ctdb structure */
+       ctdb_deregister_message_handler(ctdb, srvid, private_data);
+       return 0;
 }
 
 
@@ -689,6 +700,7 @@ int ctdb_control(struct ctdb_context *ctdb, uint32_t destnode, uint64_t srvid,
        c->hdr.destnode     = destnode;
        c->hdr.reqid        = state->reqid;
        c->opcode           = opcode;
+       c->client_id        = 0;
        c->flags            = flags;
        c->srvid            = srvid;
        c->datalen          = data.dsize;
@@ -1493,5 +1505,11 @@ int ctdb_list_keys(struct ctdb_db_context *ctdb_db, FILE *f)
                event_loop_once(ctdb_db->ctdb->ev);
        }
 
+       ret = ctdb_remove_message_handler(ctdb_db->ctdb, srvid, &state);
+       if (ret != 0) {
+               DEBUG(0,("Failed to remove list keys handler\n"));
+               return -1;
+       }
+
        return state.count;
 }

diff --git a/ctdb/common/ctdb_control.c b/ctdb/common/ctdb_control.c
index cd24973d40fe51202fe0e2a022ac55c0d2811084..bd52d60453af550747539bd0ca828d1c92171191 100644 (file)
--- a/ctdb/common/ctdb_control.c
+++ b/ctdb/common/ctdb_control.c
@@ -130,7 +130,9 @@ static int traverse_getkeys(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data
   process a control request
  */
 static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb, 
-                                    uint32_t opcode, TDB_DATA indata,
+                                    uint32_t opcode, 
+                                    uint64_t srvid, uint32_t client_id,
+                                    TDB_DATA indata,
                                     TDB_DATA *outdata, uint32_t srcnode)
 {
        switch (opcode) {
@@ -387,6 +389,12 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
        case CTDB_CONTROL_TRAVERSE_DATA:
                return ctdb_control_traverse_data(ctdb, indata, outdata);
 
+       case CTDB_CONTROL_REGISTER_SRVID:
+               return daemon_register_message_handler(ctdb, client_id, srvid);
+
+       case CTDB_CONTROL_DEREGISTER_SRVID:
+               return daemon_deregister_message_handler(ctdb, client_id, srvid);
+
        default:
                DEBUG(0,(__location__ " Unknown CTDB control opcode %u\n", opcode));
                return -1;
@@ -408,7 +416,8 @@ void ctdb_request_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr
        data.dsize = c->datalen;
 
        outdata = talloc_zero(c, TDB_DATA);
-       status = ctdb_control_dispatch(ctdb, c->opcode, data, outdata, hdr->srcnode);
+       status = ctdb_control_dispatch(ctdb, c->opcode, c->srvid, c->client_id,
+                                      data, outdata, hdr->srcnode);
 
        /* some controls send no reply */
        if (c->flags & CTDB_CTRL_FLAG_NOREPLY) {
@@ -471,7 +480,8 @@ static int ctdb_control_destructor(struct ctdb_control_state *state)
   send a control message to a node
  */
 int ctdb_daemon_send_control(struct ctdb_context *ctdb, uint32_t destnode,
-                            uint64_t srvid, uint32_t opcode, uint32_t flags,
+                            uint64_t srvid, uint32_t opcode, uint32_t client_id,
+                            uint32_t flags,
                             TDB_DATA data,
                             ctdb_control_callback_fn_t callback,
                             void *private_data)
@@ -504,6 +514,7 @@ int ctdb_daemon_send_control(struct ctdb_context *ctdb, uint32_t destnode,
        c->hdr.destnode     = destnode;
        c->hdr.reqid        = state->reqid;
        c->opcode           = opcode;
+       c->client_id        = client_id;
        c->flags            = flags;
        c->srvid            = srvid;
        c->datalen          = data.dsize;

diff --git a/ctdb/common/ctdb_daemon.c b/ctdb/common/ctdb_daemon.c
index 55af203c581ffce76962236f750cdc34824f172d..f29d7d3c1225aaed2076e9e0785f4fa87034349f 100644 (file)
--- a/ctdb/common/ctdb_daemon.c
+++ b/ctdb/common/ctdb_daemon.c
@@ -36,10 +36,10 @@ struct ctdb_client {
        struct ctdb_context *ctdb;
        int fd;
        struct ctdb_queue *queue;
+       uint32_t client_id;
 };
 
 
-
 static void daemon_incoming_packet(void *, uint8_t *, uint32_t );
 
 static void ctdb_main_loop(struct ctdb_context *ctdb)
@@ -134,20 +134,35 @@ static void daemon_message_handler(struct ctdb_context *ctdb, uint64_t srvid,
   this is called when the ctdb daemon received a ctdb request to 
   set the srvid from the client
  */
-static void daemon_request_register_message_handler(struct ctdb_client *client, 
-                                                   struct ctdb_req_register *c)
+int daemon_register_message_handler(struct ctdb_context *ctdb, uint32_t client_id, uint64_t srvid)
 {
+       struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
        int res;
-       res = ctdb_register_message_handler(client->ctdb, client, 
-                                           c->srvid, daemon_message_handler, 
-                                           client);
+       if (client == NULL) {
+               DEBUG(0,("Bad client_id in daemon_request_register_message_handler\n"));
+               return -1;
+       }
+       res = ctdb_register_message_handler(ctdb, client, srvid, daemon_message_handler, client);
        if (res != 0) {
-               DEBUG(0,(__location__ " Failed to register handler %llu in daemon\n", 
-                        c->srvid));
+               DEBUG(0,(__location__ " Failed to register handler %llu in daemon\n", srvid));
        } else {
-               DEBUG(2,(__location__ " Registered message handler for srvid=%llu\n", 
-                        c->srvid));
+               DEBUG(2,(__location__ " Registered message handler for srvid=%llu\n", srvid));
        }
+       return res;
+}
+
+/*
+  this is called when the ctdb daemon received a ctdb request to 
+  remove a srvid from the client
+ */
+int daemon_deregister_message_handler(struct ctdb_context *ctdb, uint32_t client_id, uint64_t srvid)
+{
+       struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
+       if (client == NULL) {
+               DEBUG(0,("Bad client_id in daemon_request_deregister_message_handler\n"));
+               return -1;
+       }
+       return ctdb_deregister_message_handler(ctdb, srvid, client);
 }
 
 
@@ -232,6 +247,7 @@ static void daemon_request_connect_wait(struct ctdb_client *client,
 */
 static int ctdb_client_destructor(struct ctdb_client *client)
 {
+       ctdb_reqid_remove(client->ctdb, client->client_id);
        client->ctdb->num_clients--;
        close(client->fd);
        client->fd = -1;
@@ -448,12 +464,6 @@ static void daemon_incoming_packet(void *p, uint8_t *data, uint32_t nread)
                daemon_request_call_from_client(client, (struct ctdb_req_call *)hdr);
                break;
 
-       case CTDB_REQ_REGISTER:
-               ctdb->status.client.req_register++;
-               daemon_request_register_message_handler(client, 
-                                                       (struct ctdb_req_register *)hdr);
-               break;
-
        case CTDB_REQ_MESSAGE:
                ctdb->status.client.req_message++;
                daemon_request_message_from_client(client, (struct ctdb_req_message *)hdr);
@@ -548,6 +558,7 @@ static void ctdb_accept_client(struct event_context *ev, struct fd_event *fde,
        client = talloc_zero(ctdb, struct ctdb_client);
        client->ctdb = ctdb;
        client->fd = fd;
+       client->client_id = ctdb_reqid_new(ctdb, client);
        ctdb->num_clients++;
 
        client->queue = ctdb_queue_setup(ctdb, client, fd, CTDB_DS_ALIGNMENT, 
@@ -854,7 +865,8 @@ static void daemon_request_control_from_client(struct ctdb_client *client,
        data.dptr = &c->data[0];
        data.dsize = c->datalen;
        res = ctdb_daemon_send_control(client->ctdb, c->hdr.destnode,
-                                      c->srvid, c->opcode, c->flags,
+                                      c->srvid, c->opcode, client->client_id,
+                                      c->flags,
                                       data, daemon_control_callback,
                                       state);
        if (res != 0) {

diff --git a/ctdb/common/ctdb_ltdb.c b/ctdb/common/ctdb_ltdb.c
index f8141180f2f3655c6a7babdbe01309f10d5d9d22..5222e0a7aa415a2c12f9fe080755bf5ec8418d05 100644 (file)
--- a/ctdb/common/ctdb_ltdb.c
+++ b/ctdb/common/ctdb_ltdb.c
@@ -389,7 +389,7 @@ int32_t ctdb_control_db_attach(struct ctdb_context *ctdb, TDB_DATA indata,
        
        /* tell all the other nodes about this database */
        ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_VNN, 0,
-                                CTDB_CONTROL_DB_ATTACH, CTDB_CTRL_FLAG_NOREPLY,
+                                CTDB_CONTROL_DB_ATTACH, 0, CTDB_CTRL_FLAG_NOREPLY,
                                 indata, NULL, NULL);
 
        DEBUG(1,("Attached to database '%s'\n", ctdb_db->db_path));

diff --git a/ctdb/common/ctdb_message.c b/ctdb/common/ctdb_message.c
index 4d75ac15fc4ee7c4ce486aec1a93194da8ac95cd..1a7a29508f0136593448cd47cf32592ca449eb4c 100644 (file)
--- a/ctdb/common/ctdb_message.c
+++ b/ctdb/common/ctdb_message.c
@@ -45,7 +45,6 @@ int ctdb_dispatch_message(struct ctdb_context *ctdb, uint64_t srvid, TDB_DATA da
        return 0;
 }
 
-
 /*
   called when a CTDB_REQ_MESSAGE packet comes in
 */
@@ -170,3 +169,20 @@ int ctdb_register_message_handler(struct ctdb_context *ctdb,
 
        return 0;
 }
+
+
+/*
+  setup handler for receipt of ctdb messages from ctdb_send_message()
+*/
+int ctdb_deregister_message_handler(struct ctdb_context *ctdb, uint64_t srvid, void *private_data)
+{
+       struct ctdb_message_list *m;
+
+       for (m=ctdb->message_list;m;m=m->next) {
+               if (m->srvid == srvid && m->message_private == private_data) {
+                       talloc_free(m);
+                       return 0;
+               }
+       }
+       return -1;
+}

diff --git a/ctdb/common/ctdb_traverse.c b/ctdb/common/ctdb_traverse.c
index 2972c70ac5c71bb006f2feb1d59b4fbaf27f19a7..c71457d717ed1c188e8688bd0f81506e44fc5a67 100644 (file)
--- a/ctdb/common/ctdb_traverse.c
+++ b/ctdb/common/ctdb_traverse.c
@@ -264,7 +264,7 @@ struct ctdb_traverse_all_handle *ctdb_daemon_traverse_all(struct ctdb_db_context
 
        /* tell all the nodes in the cluster to start sending records to this node */
        ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_VNN, 0, CTDB_CONTROL_TRAVERSE_ALL,
-                                      CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+                                      0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
        if (ret != 0) {
                talloc_free(state);
                return NULL;
@@ -300,7 +300,7 @@ static void traverse_all_callback(void *p, TDB_DATA key, TDB_DATA data)
        data.dsize = d->length;
 
        ret = ctdb_daemon_send_control(state->ctdb, state->srcnode, 0, CTDB_CONTROL_TRAVERSE_DATA,
-                                      CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+                                      0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
        if (ret != 0) {
                DEBUG(0,("Failed to send traverse data\n"));
        }

diff --git a/ctdb/direct/ctdbd_test.c b/ctdb/direct/ctdbd_test.c
index 1a3c2054b9d8719d77e4f714ea2a2a96c1cb0f34..1df52d08361d2f046df484434df1a9b299c258c0 100644 (file)
--- a/ctdb/direct/ctdbd_test.c
+++ b/ctdb/direct/ctdbd_test.c
@@ -51,22 +51,6 @@ static int ux_socket_connect(const char *name)
        return fd;
 }
 
-void register_pid_with_daemon(int fd, int pid)
-{
-       struct ctdb_req_register r;
-
-       bzero(&r, sizeof(r));
-       r.hdr.length       = sizeof(r);
-       r.hdr.ctdb_magic   = CTDB_MAGIC;
-       r.hdr.ctdb_version = CTDB_VERSION;
-       r.hdr.generation   = 1;
-       r.hdr.operation    = CTDB_REQ_REGISTER;
-       r.srvid            = pid;
-
-       /* XXX must deal with partial writes here */
-       write(fd, &r, sizeof(r));
-}
-
 /* send a command to the cluster to wait until all nodes are connected
    and the cluster is fully operational
  */
@@ -259,13 +243,6 @@ int main(int argc, const char *argv[])
        }
 
 
-       /* register our local server id with the daemon so that it knows
-          where to send messages addressed to our local pid.
-        */
-       pid=getpid();
-       register_pid_with_daemon(fd, pid);
-
-
        /* do a connect wait to ensure that all nodes in the cluster are up 
           and operational.
           this also tells us the vnn of the local cluster.

diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index 22bcedf197c8fec795f679a2d8eea718b06e2f50..518df2900b8d22387ec3975287ac1d420549f460 100644 (file)
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -149,7 +149,6 @@ struct ctdb_status {
                uint32_t req_call;
                uint32_t req_message;
                uint32_t req_finished;
-               uint32_t req_register;
                uint32_t req_connect_wait;
                uint32_t req_shutdown;
                uint32_t req_control;
@@ -281,6 +280,8 @@ enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS,
                    CTDB_CONTROL_TRAVERSE_START,
                    CTDB_CONTROL_TRAVERSE_ALL,
                    CTDB_CONTROL_TRAVERSE_DATA,
+                   CTDB_CONTROL_REGISTER_SRVID,
+                   CTDB_CONTROL_DEREGISTER_SRVID,
 };
 
 
@@ -337,8 +338,7 @@ enum ctdb_operation {
        CTDB_REPLY_CONTROL,
        
        /* only used on the domain socket */
-       CTDB_REQ_REGISTER       = 1000,     
-       CTDB_REQ_CONNECT_WAIT,
+       CTDB_REQ_CONNECT_WAIT   = 1000,
        CTDB_REPLY_CONNECT_WAIT,
        CTDB_REQ_SHUTDOWN
 };
@@ -404,11 +404,6 @@ struct ctdb_reply_dmaster {
        uint8_t  data[1];
 };
 
-struct ctdb_req_register {
-       struct ctdb_req_header hdr;
-       uint64_t srvid;
-};
-
 struct ctdb_req_message {
        struct ctdb_req_header hdr;
        uint64_t srvid;
@@ -449,6 +444,7 @@ struct ctdb_req_control {
        struct ctdb_req_header hdr;
        uint32_t opcode;
        uint64_t srvid;
+       uint32_t client_id;
 #define CTDB_CTRL_FLAG_NOREPLY 1
        uint32_t flags;
        uint32_t datalen;
@@ -629,7 +625,7 @@ void ctdb_request_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr
 void ctdb_reply_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
 
 int ctdb_daemon_send_control(struct ctdb_context *ctdb, uint32_t destnode,
-                            uint64_t srvid, uint32_t opcode, uint32_t flags,
+                            uint64_t srvid, uint32_t opcode, uint32_t client_id, uint32_t flags,
                             TDB_DATA data,
                             ctdb_control_callback_fn_t callback,
                             void *private_data);
@@ -688,5 +684,8 @@ int32_t ctdb_control_traverse_data(struct ctdb_context *ctdb, TDB_DATA data, TDB
 
 int ctdb_dispatch_message(struct ctdb_context *ctdb, uint64_t srvid, TDB_DATA data);
 
+int daemon_register_message_handler(struct ctdb_context *ctdb, uint32_t client_id, uint64_t srvid);
+int ctdb_deregister_message_handler(struct ctdb_context *ctdb, uint64_t srvid, void *private_data);
+int daemon_deregister_message_handler(struct ctdb_context *ctdb, uint32_t client_id, uint64_t srvid);
 
 #endif

diff --git a/ctdb/tests/ctdbd.sh b/ctdb/tests/ctdbd.sh
index 1664a68d639aa13032448283f82cb70066c32fb0..53857e2442b87fbe6b467f328e9c1384177dfcde 100755 (executable)
--- a/ctdb/tests/ctdbd.sh
+++ b/ctdb/tests/ctdbd.sh
@@ -8,6 +8,7 @@ $VALGRIND bin/ctdbd --nlist direct/nodes.txt
 
 echo "Testing ping"
 $VALGRIND bin/ctdb_control ping || exit 1
+exit 0
 
 echo "Testing status"
 $VALGRIND bin/ctdb_control status all || exit 1

diff --git a/ctdb/tools/ctdb_control.c b/ctdb/tools/ctdb_control.c
index 8a8e8c3d8fdffa2cf3f140b67a899211f27c6fa7..1961572a5301944c251d00bf48a5ba9ac17021cf 100644 (file)
--- a/ctdb/tools/ctdb_control.c
+++ b/ctdb/tools/ctdb_control.c
@@ -94,7 +94,6 @@ static void show_status(struct ctdb_status *s)
        printf("   req_call              %u\n", s->client.req_call);
        printf("   req_message           %u\n", s->client.req_message);
        printf("   req_finished          %u\n", s->client.req_finished);
-       printf("   req_register          %u\n", s->client.req_register);
        printf("   req_connect_wait      %u\n", s->client.req_connect_wait);
        printf("   req_shutdown          %u\n", s->client.req_shutdown);
        printf("   req_control           %u\n", s->client.req_control);