May 2012 - cluster-commits - Fedora mailing-lists

cluster: STABLE32 - Add some additional debugging logging for try-locks
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=24… Commit: 24838db6f1e47802188765783f7b1e18ed16a880 Parent: c3e2d472b80f8d1c47c142b5f5baeb250b0edb9e Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Wed Apr 25 13:07:54 2012 -0400 Committer: Ryan McCabe <rmccabe(a)redhat.com> CommitterDate: Wed Apr 25 13:07:54 2012 -0400 Add some additional debugging logging for try-locks Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/src/daemons/cpglockd.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 689c494..1f69ef8 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -478,6 +478,8 @@ send_grant(struct request_node *n) static int send_nak(struct cpg_lock_msg *m) { + logt_print(LOG_DEBUG, "-> sending NAK %s to %d:%d:%d\n", + m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid); m->request = MSG_NAK; return send_lock_msg(m); @@ -1013,6 +1015,9 @@ process_nak(struct cpg_lock_msg *m, uint32_t nodeid) if (!joined) return 0; + logt_print(LOG_DEBUG, "NAK %s for %d:%d:%d\n", m->resource, + m->owner_nodeid, m->owner_pid, m->owner_tid); + list_for(&requests, r, y) { if (strcmp(r->l.resource, m->resource)) continue;

1 0

cluster: STABLE32 - cpglockd: show that we are alive while waiting for quorum (debug info)
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=37… Commit: 37e637601091d32f7608f7eda74d63e0fadff451 Parent: b0512e7c50c4c9618f751c88fd48c2a573511ace Author: Fabio M. Di Nitto <fdinitto(a)redhat.com> AuthorDate: Wed Apr 25 05:21:51 2012 +0200 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Wed Apr 25 05:21:51 2012 +0200 cpglockd: show that we are alive while waiting for quorum (debug info) Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/src/daemons/cpglockd.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 2e35f7f..d4d43b0 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -248,9 +248,10 @@ wait_for_fencing_join(int nodeid) static void wait_for_quorum_formation(cman_handle_t ch) { - logt_print(LOG_DEBUG, "Waiting for quorum to form\n"); - while (!shutdown_pending && !(cluster_quorate = cman_is_quorate(ch))) + while (!shutdown_pending && !(cluster_quorate = cman_is_quorate(ch))) { + logt_print(LOG_DEBUG, "Waiting for quorum to form\n"); sleep(1); + } logt_print(LOG_DEBUG, "Quorum formed\n"); }

1 0

cluster: STABLE32 - Fix operation when quorum is lost: - NAK all pending requests - Drop all locks - Until quorum is regained, NAK all new lock requests
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=44… Commit: 44a2a2f0b39b90154d36eeaeac8debeb291c9097 Parent: 37e637601091d32f7608f7eda74d63e0fadff451 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Tue Apr 24 23:54:36 2012 -0400 Committer: Ryan McCabe <rmccabe(a)redhat.com> CommitterDate: Tue Apr 24 23:56:05 2012 -0400 Fix operation when quorum is lost: - NAK all pending requests - Drop all locks - Until quorum is regained, NAK all new lock requests --- rgmanager/src/daemons/cpglockd.c | 110 ++++++++++++++++++++++++++++++++------ 1 files changed, 94 insertions(+), 16 deletions(-) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index d4d43b0..a3943a2 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -588,15 +588,20 @@ del_client(int fd) logt_print(LOG_DEBUG, "RECOVERY: Looking for locks held by PID %d\n", pid); - /* This may not be needed */ - purge_requests(my_node_id, pid); + if (cluster_quorate) { + /* This may not be needed */ + purge_requests(my_node_id, pid); - memset(&m, 0, sizeof(m)); - m.request = MSG_PURGE; - m.owner_nodeid = my_node_id; - m.owner_pid = pid; + memset(&m, 0, sizeof(m)); + m.request = MSG_PURGE; + m.owner_nodeid = my_node_id; + m.owner_pid = pid; - send_lock_msg(&m); + send_lock_msg(&m); + } + + if (!cluster_quorate && locks != NULL) + logt_print(LOG_DEBUG, "RECOVERY: not quorate but locks exist\n"); list_for(&locks, l, x) { if (l->l.owner_nodeid != my_node_id || @@ -1385,6 +1390,36 @@ static void init_logging(int reconf) logt_conf("cpglockd", mode, syslog_facility, syslog_priority, logfile_priority, logfile); } +static void +clear_local_state(void) +{ + struct request_node *cur_req = NULL; + struct lock_node *cur_lock = NULL; + + while ((cur_lock = locks) != NULL) { + logt_print(LOG_DEBUG, + "CLEAR STATE: Dropping lock %s [%d:%d]\n", + cur_lock->l.resource, + cur_lock->l.owner_nodeid, cur_lock->l.owner_pid); + list_remove(&locks, cur_lock); + free(cur_lock); + } + + while ((cur_req = requests) != NULL) { + if (cur_req->l.owner_nodeid == my_node_id) { + nak_client(cur_req); + logt_print(LOG_DEBUG, "CLEAR STATE: sent NAK %s [%d:%d:%d]\n", + cur_req->l.resource, cur_req->l.owner_nodeid, + cur_req->l.owner_pid, cur_req->l.owner_tid); + } + logt_print(LOG_DEBUG, "CLEAR STATE: Dropping request %s [%d:%d:%d]\n", + cur_req->l.resource, cur_req->l.owner_nodeid, + cur_req->l.owner_pid, cur_req->l.owner_tid); + list_remove(&requests, cur_req); + free(cur_req); + } +} + int main(int argc, char **argv) { @@ -1394,6 +1429,7 @@ main(int argc, char **argv) int afd = -1; int cman_fd; int n,x; + int quorate_last = 0; int wait_for_quorum = 1; int wait_for_fencing = 1; int opt; @@ -1525,6 +1561,10 @@ main(int argc, char **argv) goto out; } + if (wait_for_quorum) + wait_for_quorum_formation(cman_handle); + quorate_last = cluster_quorate; + logt_print(LOG_INFO, "cpglockd entering normal operation\n"); while (!shutdown_pending) { struct timeval tv; @@ -1569,6 +1609,16 @@ main(int argc, char **argv) goto out; } + if (!quorate_last && cluster_quorate) + logt_print(LOG_INFO, "Cluster quorum regained\n"); + + if (quorate_last && !cluster_quorate) { + logt_print(LOG_INFO, + "Cluster lost quorum. Clearing local state.\n"); + clear_local_state(); + } + quorate_last = cluster_quorate; + fence_check: list_for(&pending_fencing, pf_node, x) { int victim; @@ -1640,16 +1690,23 @@ main(int argc, char **argv) if (FD_ISSET(fd, &rfds)) { afd = accept(fd, NULL, NULL); - insert_client(afd); + if (afd < 0) { + logt_print(LOG_DEBUG, "Error accepting new client: %s\n", + strerror(errno)); + } else { + insert_client(afd); + } --n; } if (FD_ISSET(cpgfd, &rfds)) { - if (cpg_dispatch(cpg, CPG_DISPATCH_ALL) != CPG_OK) { + --n; + if (cluster_quorate && + cpg_dispatch(cpg, CPG_DISPATCH_ALL) != CPG_OK) + { logt_print(LOG_ERR, "Fatal: Lost CPG connection.\n"); return -1; } - --n; } if (n <= 0) @@ -1660,9 +1717,10 @@ main(int argc, char **argv) if (!FD_ISSET(client->fd, &rfds)) continue; --n; + if (read_retry(client->fd, &m, sizeof(m), NULL) < 0) { logt_print(LOG_DEBUG, "Closing client fd %d pid %d: %d\n", - client->fd, client->pid, errno); + client->fd, client->pid, errno); del_client(client->fd); break; @@ -1671,15 +1729,35 @@ main(int argc, char **argv) /* send lock request */ /* XXX check for dup connection */ if (m.request == MSG_LOCK) { - client->pid = m.owner_pid; - send_lock(&m); + if (!cluster_quorate) { + logt_print(LOG_DEBUG, "Sending NAK for new lock request while not quorate (%s [%d:%d:%d])\n", + m.resource, m.owner_nodeid, + m.owner_pid, m.owner_tid); + + m.request = MSG_NAK; + if (write_retry(client->fd, &m, sizeof(m), NULL) < 0) { + logt_print(LOG_DEBUG, + "Error sending NAK for %s [%d:%d:%d]: %s\n", + m.resource, m.owner_nodeid, + m.owner_pid, m.owner_tid, strerror(errno)); + } + } else { + client->pid = m.owner_pid; + send_lock(&m); + } } if (m.request == MSG_UNLOCK) { //logt_print(LOG_DEBUG, "Unlock from fd %d\n", client->fd); - find_lock(&m); - if (grant_next(&m) == 0) - send_unlock(&m); + /* + ** When not quorate, all locks and requests have already + ** been dropped. + */ + if (cluster_quorate) { + find_lock(&m); + if (grant_next(&m) == 0) + send_unlock(&m); + } } if (m.request == MSG_DUMP) {

1 0

cluster: STABLE32 - rgmanager: die hard if we cpglockd dies
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=4f… Commit: 4fd8e8cc76af3152a1520ea4acbc6733ba06f2a3 Parent: 564207f3f2699a567c01f1b395d7a760989b36a0 Author: Fabio M. Di Nitto <fdinitto(a)redhat.com> AuthorDate: Tue Apr 24 16:30:32 2012 +0200 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:10 2012 +0200 rgmanager: die hard if we cpglockd dies Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/src/clulib/cpg_lock.c | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diff --git a/rgmanager/src/clulib/cpg_lock.c b/rgmanager/src/clulib/cpg_lock.c index 7cff2a3..d257cf4 100644 --- a/rgmanager/src/clulib/cpg_lock.c +++ b/rgmanager/src/clulib/cpg_lock.c @@ -12,6 +12,7 @@ #include <sys/types.h> #include <sys/select.h> #include <pthread.h> +#include <signal.h> #include <cpglock.h> @@ -69,9 +70,13 @@ _cpg_lock(int mode, ret = cpg_lock(_cpgh, resource, 1, &l); else ret = cpg_lock(_cpgh, resource, 0, &l); - + if (ret == 0) { cpg2dlm(&l, lksb); + } else { + if (errno == EPIPE) { + raise(SIGSEGV); + } } return ret;

1 0

cluster: STABLE32 - If we see a JOIN message from ourselves, move ourselves to the end of the membership list.
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=b0… Commit: b0512e7c50c4c9618f751c88fd48c2a573511ace Parent: c2d306b73a5194f7689f0250bbf2ef2c7e66baa8 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Tue Apr 24 12:56:24 2012 -0400 Committer: Ryan McCabe <rmccabe(a)redhat.com> CommitterDate: Tue Apr 24 12:56:24 2012 -0400 If we see a JOIN message from ourselves, move ourselves to the end of the membership list. Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/src/daemons/cpglockd.c | 12 ++++++++++-- 1 files changed, 10 insertions(+), 2 deletions(-) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 7d4882d..2e35f7f 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -1094,8 +1094,16 @@ process_join(struct cpg_lock_msg *m, uint32_t nodeid, uint32_t pid) list_for(&group_members, n, x) { if (n->nodeid == nodeid) { - logt_print(LOG_DEBUG, "IGNORING JOIN from existing member %d.%d (%d.%d)\n", - nodeid, pid, nodeid, n->pid); + if (n->pid == pid) { + logt_print(LOG_DEBUG, "Saw JOIN from self (%d.%d)\n", + nodeid, pid); + list_remove(&group_members, n); + list_append(&group_members, n); + } else { + logt_print(LOG_DEBUG, + "IGNORING JOIN from existing member %d.%d (%d.%d)\n", + nodeid, pid, nodeid, n->pid); + } return 0; } }

1 0

cluster: STABLE32 - Convert all remaining calls to printf and perror for error reporting to logt_print()
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=c2… Commit: c2d306b73a5194f7689f0250bbf2ef2c7e66baa8 Parent: 4fd8e8cc76af3152a1520ea4acbc6733ba06f2a3 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Tue Apr 24 12:54:51 2012 -0400 Committer: Ryan McCabe <rmccabe(a)redhat.com> CommitterDate: Tue Apr 24 12:54:51 2012 -0400 Convert all remaining calls to printf and perror for error reporting to logt_print() Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/src/daemons/cpglockd.c | 38 +++++++++++++++++++++++--------------- 1 files changed, 23 insertions(+), 15 deletions(-) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 7e1ae33..7d4882d 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -1314,12 +1314,12 @@ cpg_init(void) memset(&cpg, 0, sizeof(cpg)); if (cpg_initialize(&cpg, &my_callbacks) != CPG_OK) { - perror("cpg_initialize"); + logt_print(LOG_DEBUG, "cpg_initialize failed"); return -1; } if (cpg_join(cpg, &gname) != CPG_OK) { - perror("cpg_join"); + logt_print(LOG_DEBUG, "cpg_join failed"); return -1; } @@ -1327,7 +1327,8 @@ cpg_init(void) ret = cpg_membership_get(cpg, &gname, member_list, &cpg_member_list_len); if (ret != CPG_OK) { - fprintf(stderr, "cpg_membership_get() failed: %s", strerror(errno)); + logt_print(LOG_DEBUG, + "cpg_membership_get() failed: %s", strerror(errno)); cpg_fin(); return -1; } @@ -1335,7 +1336,7 @@ cpg_init(void) for (i = 0 ; i < cpg_member_list_len ; i++) { if (member_list[i].nodeid == my_node_id) { if (member_list[i].pid != getpid()) { - fprintf(stderr, "nodeid %d already in group with PID %u %u\n", + logt_print(LOG_DEBUG, "nodeid %d already in group with PID %u %u\n", member_list[i].nodeid, member_list[i].pid, getpid()); cpg_fin(); return -1; @@ -1437,7 +1438,7 @@ main(int argc, char **argv) cman_connect(&cman_handle); if (cman_handle == NULL) { - fprintf(stderr, "Unable to connect to cman\n"); + logt_print(LOG_ERR, "Unable to connect to cman\n"); exit_status = -1; goto out4; } @@ -1455,7 +1456,7 @@ main(int argc, char **argv) cman_get_node(cman_handle, CMAN_NODEID_US, &my_node); if (my_node.cn_nodeid == 0) { - fprintf(stderr, "Unable to get our cluster node ID\n"); + logt_print(LOG_ERR, "Unable to get our cluster node ID\n"); exit_status = -1; goto out3; } @@ -1473,14 +1474,14 @@ main(int argc, char **argv) sizeof(cman_nodes) / sizeof(cman_nodes[0]), &cman_node_count, cman_nodes); if (ret < 0) { - fprintf(stderr, "Unable to get cman nodes list\n"); + logt_print(LOG_ERR, "Unable to get cman nodes list\n"); exit_status = -1; goto out3; } cman_fd = cman_get_fd(cman_handle); if (cman_fd < 0) { - fprintf(stderr, "Error: cman fd is %d\n", cman_fd); + logt_print(LOG_ERR, "Error: cman fd is %d\n", cman_fd); exit_status = -1; goto out3; } @@ -1488,16 +1489,21 @@ main(int argc, char **argv) cman_start_notification(cman_handle, cman_callback); if (cpg_init() < 0) { - fprintf(stderr, "Unable to join CPG group\n"); + logt_print(LOG_ERR, "Unable to join CPG group\n"); exit_status = -1; goto out2; } - assert(my_node.cn_nodeid == my_node_id); + if (my_node.cn_nodeid != my_node_id) { + logt_print(LOG_ERR, "cman nodeid and CPG node id differ: %d != %d\n", + my_node.cn_nodeid, my_node_id); + exit_status = -1; + goto out1; + } fd = sock_listen(CPG_LOCKD_SOCK); if (fd < 0) { - fprintf(stderr, "Error connecting to %s: %s\n", + logt_print(LOG_ERR, "Error connecting to %s: %s\n", CPG_LOCKD_SOCK, strerror(errno)); exit_status = -1; goto out1; @@ -1505,10 +1511,12 @@ main(int argc, char **argv) cpg_fd_get(cpg, &cpgfd); if (cpgfd < 0 || send_join() < 0) { + logt_print(LOG_ERR, "Unable to complete join to CPG group\n"); exit_status = -1; goto out; } + logt_print(LOG_INFO, "cpglockd entering normal operation\n"); while (!shutdown_pending) { struct timeval tv; struct pending_fence_node *pf_node; @@ -1531,14 +1539,14 @@ main(int argc, char **argv) n = select_retry(x+1, &rfds, NULL, NULL, &tv); if (n < 0) { - fprintf(stderr, "Error: select: %s\n", strerror(errno)); + logt_print(LOG_ERR, "Error: select: %s\n", strerror(errno)); exit_status = -1; goto out; } if (FD_ISSET(cman_fd, &rfds)) { if (cman_dispatch(cman_handle, CMAN_DISPATCH_ALL) < 0) { - fprintf(stderr, "Fatal: cman_dispatch() failed: %s\n", + logt_print(LOG_ERR, "Fatal: cman_dispatch() failed: %s\n", strerror(errno)); exit_status = -1; goto out; @@ -1547,7 +1555,7 @@ main(int argc, char **argv) } if (cman_shutdown_requested) { - fprintf(stderr, "Fatal: cman requested shutdown.\n"); + logt_print(LOG_INFO, "cman requested shutdown. Exiting.\n"); cman_replyto_shutdown(cman_handle, 1); goto out; } @@ -1629,7 +1637,7 @@ main(int argc, char **argv) if (FD_ISSET(cpgfd, &rfds)) { if (cpg_dispatch(cpg, CPG_DISPATCH_ALL) != CPG_OK) { - fprintf(stderr, "Fatal: Lost CPG connection.\n"); + logt_print(LOG_ERR, "Fatal: Lost CPG connection.\n"); return -1; } --n;

1 0

cluster: STABLE32 - Avoid a free() of an uninitialized variable when central processing is enabled.
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=b1… Commit: b18d4c2d5282302cf37e752f17aad8ce168acf81 Parent: 381ad4cbb476f67a7dbebac856d3aa49b8d9e061 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Tue Apr 24 10:10:48 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:10 2012 +0200 Avoid a free() of an uninitialized variable when central processing is enabled. Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/src/daemons/rg_event.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/rgmanager/src/daemons/rg_event.c b/rgmanager/src/daemons/rg_event.c index 6106e70..a3c12c5 100644 --- a/rgmanager/src/daemons/rg_event.c +++ b/rgmanager/src/daemons/rg_event.c @@ -247,7 +247,7 @@ static int find_master(void) { event_master_t *masterinfo = NULL; - void *data; + void *data = NULL; uint32_t sz; cluster_member_list_t *m; uint64_t vn;

1 0

cluster: STABLE32 - rgmanager: Transmogrify states to expected values
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=56… Commit: 564207f3f2699a567c01f1b395d7a760989b36a0 Parent: b18d4c2d5282302cf37e752f17aad8ce168acf81 Author: Lon Hohberger <lhh(a)redhat.com> AuthorDate: Tue Apr 24 10:23:33 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:10 2012 +0200 rgmanager: Transmogrify states to expected values Because central_processing looks for state == 0 and LOCK_HELD is 2, we need to flip the states to match DLM states. Signed-off-by: Lon Hohberger <lhh(a)redhat.com> --- rgmanager/src/clulib/cpg_lock.c | 23 +++++++++++++++++++++-- 1 files changed, 21 insertions(+), 2 deletions(-) diff --git a/rgmanager/src/clulib/cpg_lock.c b/rgmanager/src/clulib/cpg_lock.c index a747419..7cff2a3 100644 --- a/rgmanager/src/clulib/cpg_lock.c +++ b/rgmanager/src/clulib/cpg_lock.c @@ -24,15 +24,34 @@ dlm2cpg(struct dlm_lksb *dlm, struct cpg_lock *cpg) { memset(cpg, 0, sizeof(*cpg)); cpg->local_id = dlm->sb_lkid; - cpg->state = dlm->sb_status; + switch(dlm->sb_status) { + case 0: + cpg->state = LOCK_HELD; + break; + case EINPROG: + cpg->state = LOCK_PENDING; + break; + default: + cpg->state = LOCK_FREE; + break; + } } static void cpg2dlm(struct cpg_lock *cpg, struct dlm_lksb *dlm) { memset(dlm, 0, sizeof(*dlm)); - dlm->sb_status = cpg->state; dlm->sb_lkid = cpg->local_id; + switch(cpg->state) { + case LOCK_HELD: + dlm->sb_status = 0; + break; + case LOCK_PENDING: + default: + /* XXX LOCK_FREE -> DLM state? */ + dlm->sb_status = EINPROG; + break; + } }

1 0

cluster: STABLE32 - cpglockd: refuse to shutdown if clients are connected
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=38… Commit: 381ad4cbb476f67a7dbebac856d3aa49b8d9e061 Parent: abe0bc61232f32b3b6757ff2b13e6e33c9894285 Author: Fabio M. Di Nitto <fdinitto(a)redhat.com> AuthorDate: Tue Apr 24 15:58:31 2012 +0200 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:10 2012 +0200 cpglockd: refuse to shutdown if clients are connected or bad things will happen Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/src/daemons/cpglockd.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 47cbcdc..7e1ae33 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -89,6 +89,10 @@ static void init_logging(int reconf); static void flag_shutdown(int __attribute__ ((unused)) sig) { + if (clients) { + logt_print(LOG_INFO, "Clients are connected to cpglockd. Refusing to shutdown\n"); + return; + } shutdown_pending = 1; } @@ -1429,6 +1433,7 @@ main(int argc, char **argv) setup_signal(SIGPIPE, SIG_IGN); setup_signal(SIGTERM, flag_shutdown); + setup_signal(SIGINT, flag_shutdown); cman_connect(&cman_handle); if (cman_handle == NULL) {

1 0

cluster: STABLE32 - cpglockd: use standard logging infrastructure
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=ab… Commit: abe0bc61232f32b3b6757ff2b13e6e33c9894285 Parent: d6212e80ad256dd8695feca7b50ee21dc1f681fa Author: Fabio M. Di Nitto <fdinitto(a)redhat.com> AuthorDate: Tue Apr 24 15:21:34 2012 +0200 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:10 2012 +0200 cpglockd: use standard logging infrastructure Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/include/cpglock-internal.h | 7 -- rgmanager/src/clulib/libcpglock.c | 4 - rgmanager/src/daemons/Makefile | 2 +- rgmanager/src/daemons/cpglockd.c | 171 ++++++++++++++++++++-------------- 4 files changed, 104 insertions(+), 80 deletions(-) diff --git a/rgmanager/include/cpglock-internal.h b/rgmanager/include/cpglock-internal.h index 904df37..e05b90a 100644 --- a/rgmanager/include/cpglock-internal.h +++ b/rgmanager/include/cpglock-internal.h @@ -1,12 +1,6 @@ #ifndef _CPGLOCK_INT_H #define _CPGLOCK_INT_H -#ifdef CPGLOCK_DEBUG -# define cpgl_debug(format, args...) do { printf(format, ##args); } while (0) -#else -# define cpgl_debug(format, args...) do { } while (0) -#endif - #ifndef CPG_LOCKD_SOCK #define CPG_LOCKD_SOCK "/var/run/cpglockd.sk" #endif @@ -25,7 +19,6 @@ typedef enum { MSG_HALT = 999 } cpg_lock_req_t; -/* Mixed architecture not supported yet */ struct cpg_lock_msg { int32_t request; uint32_t owner_nodeid; diff --git a/rgmanager/src/clulib/libcpglock.c b/rgmanager/src/clulib/libcpglock.c index d554281..f99edd4 100644 --- a/rgmanager/src/clulib/libcpglock.c +++ b/rgmanager/src/clulib/libcpglock.c @@ -145,8 +145,6 @@ cpg_lock(void *handle, const char *resource, lock_flag_t flags, struct cpg_lock } if (strcmp(r.resource, l.resource)) { - cpgl_debug("NOTE: msg for wrong lock want: %s got: %s\n", - l.resource, r.resource); add_pending(&h->pending, &r); pthread_mutex_unlock(&h->mutex); usleep(random() & 16383); @@ -154,8 +152,6 @@ cpg_lock(void *handle, const char *resource, lock_flag_t flags, struct cpg_lock continue; } if (r.owner_tid != l.owner_tid) { - cpgl_debug("NOTE: msg for wrong seq want: %d got: %d\n", - l.owner_tid, r.owner_tid ); add_pending(&h->pending, &r); pthread_mutex_unlock(&h->mutex); usleep(random() & 16383); diff --git a/rgmanager/src/daemons/Makefile b/rgmanager/src/daemons/Makefile index d125eaa..a178523 100644 --- a/rgmanager/src/daemons/Makefile +++ b/rgmanager/src/daemons/Makefile @@ -107,7 +107,7 @@ ${TARGET3}: ${TARGET1} ${TARGET4}: ${OBJS4} ${LDDEPS} $(CC) -o $@ $^ $(CPG_LDFLAGS) $(CMAN_LDFLAGS) $(CCS_LDFLAGS) \ - $(FENCED_LDFLAGS) $(EXTRA_LDFLAGS) $(LDFLAGS) + $(FENCED_LDFLAGS) $(LOGSYS_LDFLAGS) $(EXTRA_LDFLAGS) $(LDFLAGS) ${TARGET5}: ${OBJS5} ${LDDEPS} $(CC) -o $@ $^ $(CPG_LDFLAGS) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 9f9f635..47cbcdc 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -18,6 +18,7 @@ #include <ccs.h> #include <libcman.h> #include <libfenced.h> +#include <liblogthread.h> #include "daemon_init.h" #include "sock.h" @@ -75,6 +76,7 @@ static int joined = 0; static int cluster_quorate = 0; static int cman_shutdown_requested = 0; static int shutdown_pending = 0; +static int nofork = 0; static cman_node_t cman_nodes[CPG_MEMBERS_MAX]; static int cman_node_count; @@ -82,6 +84,8 @@ static int cman_node_count; static cman_node_t old_cman_nodes[CPG_MEMBERS_MAX]; static int old_cman_node_count; +static void init_logging(int reconf); + static void flag_shutdown(int __attribute__ ((unused)) sig) { @@ -100,7 +104,7 @@ cman_nodes_lost(cman_node_t *old_nodes, size_t lost_nodes_len = 0; if (lost_nodes_size < old_node_len) { - cpgl_debug("Lost nodes array smaller than old nodes array"); + logt_print(LOG_DEBUG, "Lost nodes array smaller than old nodes array"); return -1; } @@ -140,7 +144,7 @@ node_has_fencing(int nodeid) ccs_desc = ccs_connect(); if (ccs_desc < 0) { - cpgl_debug("Unable to connect to ccsd\n"); + logt_print(LOG_DEBUG, "Unable to connect to ccsd\n"); /* Assume node has fencing */ return 1; } @@ -164,7 +168,7 @@ fence_domain_joined(void) { ret = fenced_node_info(FENCED_NODEID_US, &fn); if (ret < 0) { - cpgl_debug("Unable to determine fence domain membership\n"); + logt_print(LOG_DEBUG, "Unable to determine fence domain membership\n"); return 0; } @@ -192,14 +196,14 @@ cman_callback(cman_handle_t ch, void *privdata, int reason, int arg) ret = cman_get_nodes(ch, nodes_elem, &cman_node_count, cman_nodes); if (ret < 0) { - cpgl_debug("Unable to get cman nodes list\n"); + logt_print(LOG_DEBUG, "Unable to get cman nodes list\n"); return; } nl = cman_nodes_lost(old_cman_nodes, old_cman_node_count, cman_nodes, cman_node_count, lost_nodes, nodes_elem); if (nl < 0) { - cpgl_debug("Unable to get list of lost nodes"); + logt_print(LOG_DEBUG, "Unable to get list of lost nodes"); return; } @@ -209,18 +213,20 @@ cman_callback(cman_handle_t ch, void *privdata, int reason, int arg) if (node_has_fencing(cur_nodeid)) { struct pending_fence_node *pf; - cpgl_debug("Lost node %d\n", cur_nodeid); + logt_print(LOG_DEBUG, "Lost node %d\n", cur_nodeid); pf = do_alloc(sizeof(*pf)); pf->nodeid = cur_nodeid; pf->fail_time = cur_time; list_append(&pending_fencing, pf); } else { - cpgl_debug("Lost node %d but fencing not configured\n", + logt_print(LOG_DEBUG, "Lost node %d but fencing not configured\n", cur_nodeid); } } } else if (reason == CMAN_REASON_TRY_SHUTDOWN) { cman_shutdown_requested = 1; + } else if (reason == CMAN_REASON_CONFIG_UPDATE) { + init_logging(1); } } @@ -228,20 +234,20 @@ static void wait_for_fencing_join(int nodeid) { if (node_has_fencing(nodeid)) { - cpgl_debug("Waiting for fence domain join operation to complete\n"); + logt_print(LOG_DEBUG, "Waiting for fence domain join operation to complete\n"); while (!shutdown_pending && !fence_domain_joined()) sleep(1); - cpgl_debug("Fence domain joined\n"); + logt_print(LOG_DEBUG, "Fence domain joined\n"); } else - cpgl_debug("No fencing is configured, not waiting\n"); + logt_print(LOG_DEBUG, "No fencing is configured, not waiting\n"); } static void wait_for_quorum_formation(cman_handle_t ch) { - cpgl_debug("Waiting for quorum to form\n"); + logt_print(LOG_DEBUG, "Waiting for quorum to form\n"); while (!shutdown_pending && !(cluster_quorate = cman_is_quorate(ch))) sleep(1); - cpgl_debug("Quorum formed\n"); + logt_print(LOG_DEBUG, "Quorum formed\n"); } static int @@ -251,10 +257,10 @@ cman_connect(cman_handle_t *ch) *ch = cman_init(NULL); if (!*ch) { - cpgl_debug("Waiting for CMAN to start\n"); + logt_print(LOG_DEBUG, "Waiting for CMAN to start\n"); while (!shutdown_pending && !(*ch = cman_init(NULL))) sleep(1); - cpgl_debug("CMAN started\n"); + logt_print(LOG_DEBUG, "CMAN started\n"); } return 0; @@ -425,7 +431,7 @@ send_lock_msg(struct cpg_lock_msg *m) do { ret = cpg_mcast_joined(cpg, CPG_TYPE_AGREED, &iov, 1); if (ret != CPG_OK) { - cpgl_debug("send_lock_msg() failed\n"); + logt_print(LOG_DEBUG, "send_lock_msg() failed\n"); usleep(250000); } } while (ret != CPG_OK && !shutdown_pending); @@ -449,7 +455,7 @@ send_grant(struct request_node *n) { struct cpg_lock_msg m; - cpgl_debug("-> sending grant %s to %d:%d:%d\n", + logt_print(LOG_DEBUG, "-> sending grant %s to %d:%d:%d\n", n->l.resource, n->l.owner_nodeid, n->l.owner_pid, n->l.owner_tid); memset(&m, 0, sizeof(m)); @@ -508,7 +514,7 @@ grant_next(struct cpg_lock_msg *m) /* Send grant */ if (r->l.state == LOCK_PENDING) { - cpgl_debug("LOCK %s: grant to %d:%d:%d\n", m->resource, + logt_print(LOG_DEBUG, "LOCK %s: grant to %d:%d:%d\n", m->resource, r->l.owner_nodeid, r->l.owner_pid, r->l.owner_tid); /* don't send dup grants */ r->l.state = LOCK_HELD; @@ -545,9 +551,9 @@ purge_requests(uint32_t nodeid, uint32_t pid) if (count) { if (pid) { - cpgl_debug("RECOVERY: purged %d requests from %d:%d\n", count, nodeid, pid); + logt_print(LOG_DEBUG, "RECOVERY: purged %d requests from %d:%d\n", count, nodeid, pid); } else { - cpgl_debug("RECOVERY: purged %d requests from node %d\n", count, nodeid); + logt_print(LOG_DEBUG, "RECOVERY: purged %d requests from node %d\n", count, nodeid); } } } @@ -575,7 +581,7 @@ del_client(int fd) if (!pid) return; - cpgl_debug("RECOVERY: Looking for locks held by PID %d\n", pid); + logt_print(LOG_DEBUG, "RECOVERY: Looking for locks held by PID %d\n", pid); /* This may not be needed */ purge_requests(my_node_id, pid); @@ -593,7 +599,7 @@ del_client(int fd) l->l.state != LOCK_HELD) continue; - cpgl_debug("RECOVERY: Releasing %s \n", l->l.resource); + logt_print(LOG_DEBUG, "RECOVERY: Releasing %s \n", l->l.resource); l->l.state = LOCK_FREE; strncpy(m.resource, l->l.resource, sizeof(m.resource)); @@ -602,9 +608,9 @@ del_client(int fd) } if (recovered) { - cpgl_debug("RECOVERY: %d locks from local PID %d\n", recovered, pid); + logt_print(LOG_DEBUG, "RECOVERY: %d locks from local PID %d\n", recovered, pid); } - cpgl_debug("RECOVERY: Complete\n"); + logt_print(LOG_DEBUG, "RECOVERY: Complete\n"); } @@ -622,7 +628,7 @@ del_node(uint32_t nodeid) */ list_for(&locks, l, x) { if (l->l.owner_nodeid == nodeid) { - cpgl_debug("RECOVERY: LOCK UPDATED: %s [%d:%d]=>[%d:%d]\n", + logt_print(LOG_DEBUG, "RECOVERY: LOCK UPDATED: %s [%d:%d]=>[%d:%d]\n", l->l.resource, l->l.owner_nodeid, l->l.owner_pid, group_members->nodeid, group_members->pid); @@ -633,7 +639,7 @@ del_node(uint32_t nodeid) return; } - cpgl_debug("RECOVERY: I am oldest node in the group, recovering locks\n"); + logt_print(LOG_DEBUG, "RECOVERY: I am oldest node in the group, recovering locks\n"); /* pass 1: purge outstanding requests from this node. */ @@ -649,13 +655,13 @@ del_node(uint32_t nodeid) list_for(&locks, l, x) { if (l->l.owner_nodeid == nodeid && l->l.state == LOCK_HELD) { - cpgl_debug("RECOVERY: Releasing %s held by dead node %d\n", + logt_print(LOG_DEBUG, "RECOVERY: Releasing %s held by dead node %d\n", l->l.resource, nodeid); l->l.state = LOCK_FREE; strncpy(m.resource, l->l.resource, sizeof(m.resource)); if (grant_next(&m) == 0) { - cpgl_debug("RECOVERY: HELD LOCK UPDATED: %s [%d:%d]=>[%d:%d]\n", + logt_print(LOG_DEBUG, "RECOVERY: HELD LOCK UPDATED: %s [%d:%d]=>[%d:%d]\n", l->l.resource, l->l.owner_nodeid, l->l.owner_pid, my_node_id, getpid()); @@ -669,7 +675,7 @@ del_node(uint32_t nodeid) } else if (l->l.owner_nodeid == nodeid && l->l.state == LOCK_FREE) { strncpy(m.resource, l->l.resource, sizeof(m.resource)); if (grant_next(&m) == 0) { - cpgl_debug("RECOVERY: FREE LOCK UPDATED: %s [%d:%d]=>[%d:%d]\n", + logt_print(LOG_DEBUG, "RECOVERY: FREE LOCK UPDATED: %s [%d:%d]=>[%d:%d]\n", l->l.resource, l->l.owner_nodeid, l->l.owner_pid, my_node_id, getpid()); @@ -681,13 +687,13 @@ del_node(uint32_t nodeid) } if (recovered) { - cpgl_debug("RECOVERY: %d locks from node %d\n", recovered, nodeid); + logt_print(LOG_DEBUG, "RECOVERY: %d locks from node %d\n", recovered, nodeid); } if (granted) { - cpgl_debug("RECOVERY: %d pending locks granted\n", granted); + logt_print(LOG_DEBUG, "RECOVERY: %d pending locks granted\n", granted); } - cpgl_debug("RECOVERY: Complete\n"); + logt_print(LOG_DEBUG, "RECOVERY: Complete\n"); } @@ -760,19 +766,19 @@ grant_client(struct lock_node *l) c = find_client(l->l.owner_pid); if (!c) { - cpgl_debug("can't find client for pid %d\n", l->l.owner_pid); + logt_print(LOG_DEBUG, "can't find client for pid %d\n", l->l.owner_pid); return 1; } if (c->fd < 0) { - cpgl_debug(" Client has bad fd\n"); + logt_print(LOG_DEBUG, " Client has bad fd\n"); return -1; } if (write_retry(c->fd, &m, sizeof(m), NULL) < 0) { /* no client anymore; drop and send to next guy XXX */ /* This should be handled by our main loop */ - //cpgl_debug("Failed to notify client!\n"); + //logt_print(LOG_DEBUG, "Failed to notify client!\n"); } return 0; @@ -794,19 +800,19 @@ nak_client(struct request_node *l) c = find_client(l->l.owner_pid); if (!c) { - cpgl_debug("can't find client for pid %d\n", l->l.owner_pid); + logt_print(LOG_DEBUG, "can't find client for pid %d\n", l->l.owner_pid); return 1; } if (c->fd < 0) { - cpgl_debug(" Client has bad fd\n"); + logt_print(LOG_DEBUG, " Client has bad fd\n"); return -1; } if (write_retry(c->fd, &m, sizeof(m), NULL) < 0) { /* no client anymore; drop and send to next guy XXX */ /* This should be handled by our main loop */ - //cpgl_debug("Failed to notify client!\n"); + //logt_print(LOG_DEBUG, "Failed to notify client!\n"); } return 0; @@ -838,7 +844,7 @@ process_lock(struct cpg_lock_msg *m) if (!joined) return 0; - cpgl_debug("LOCK %s: queue for %d:%d:%d\n", m->resource, + logt_print(LOG_DEBUG, "LOCK %s: queue for %d:%d:%d\n", m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid); queue_request(m); @@ -914,14 +920,14 @@ process_grant(struct cpg_lock_msg *m, uint32_t nodeid) if (l->l.state == LOCK_HELD) { if (m->owner_pid == 0 || m->owner_nodeid == 0) { - cpgl_debug("GRANT averted\n"); + logt_print(LOG_DEBUG, "GRANT averted\n"); return 0; } } else { l->l.state = LOCK_HELD; } - cpgl_debug("GRANT %s: to %d:%d:%d\n", + logt_print(LOG_DEBUG, "GRANT %s: to %d:%d:%d\n", m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid); @@ -962,7 +968,7 @@ process_grant(struct cpg_lock_msg *m, uint32_t nodeid) if (group_members->nodeid == my_node_id && !is_member(l->l.owner_nodeid)) { - cpgl_debug("GRANT to non-member %d; giving to next requestor\n", + logt_print(LOG_DEBUG, "GRANT to non-member %d; giving to next requestor\n", l->l.owner_nodeid); l->l.state = LOCK_FREE; @@ -1037,7 +1043,7 @@ process_unlock(struct cpg_lock_msg *m, uint32_t nodeid) GRANT to the first node on the request queue */ if (l->l.owner_nodeid == m->owner_nodeid && l->l.owner_pid == m->owner_pid) { - cpgl_debug("UNLOCK %s: %d:%d:%d\n", + logt_print(LOG_DEBUG, "UNLOCK %s: %d:%d:%d\n", m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid); l->l.state = LOCK_FREE; if (l->l.owner_nodeid == my_node_id) { @@ -1063,7 +1069,7 @@ find_lock(struct cpg_lock_msg *m) list_for(&locks, l, x) { if (m->lockid == l->l.local_id) { strncpy(m->resource, l->l.resource, sizeof(m->resource)); - cpgl_debug("LOCK %d -> %s\n", m->lockid, m->resource); + logt_print(LOG_DEBUG, "LOCK %d -> %s\n", m->lockid, m->resource); m->owner_nodeid = l->l.owner_nodeid; m->owner_pid = l->l.owner_pid; m->owner_tid = l->l.owner_tid; @@ -1084,7 +1090,7 @@ process_join(struct cpg_lock_msg *m, uint32_t nodeid, uint32_t pid) list_for(&group_members, n, x) { if (n->nodeid == nodeid) { - cpgl_debug("IGNORING JOIN from existing member %d.%d (%d.%d)\n", + logt_print(LOG_DEBUG, "IGNORING JOIN from existing member %d.%d (%d.%d)\n", nodeid, pid, nodeid, n->pid); return 0; } @@ -1093,13 +1099,13 @@ process_join(struct cpg_lock_msg *m, uint32_t nodeid, uint32_t pid) n = do_alloc(sizeof(*n)); n->nodeid = nodeid; n->pid = pid; - cpgl_debug("JOIN: node %d.%u", n->nodeid, n->pid); + logt_print(LOG_DEBUG, "JOIN: node %d.%u", n->nodeid, n->pid); if (nodeid == my_node_id) { - cpgl_debug(" (self)"); + logt_print(LOG_DEBUG, " (self)"); joined = 1; } total_members++; - cpgl_debug("\n"); + logt_print(LOG_DEBUG, "\n"); list_insert(&group_members, n); return 0; @@ -1112,7 +1118,7 @@ process_request(struct cpg_lock_msg *m, uint32_t nodeid, uint32_t pid) swab_cpg_lock_msg(m); if (m->request == MSG_HALT) { - cpgl_debug("FAULT: Halting operations; see node %d\n", m->owner_nodeid); + logt_print(LOG_DEBUG, "FAULT: Halting operations; see node %d\n", m->owner_nodeid); while (1) sleep(30); } @@ -1154,7 +1160,7 @@ cpg_deliver_func(cpg_handle_t h, { if (msglen != sizeof(struct cpg_lock_msg)) { - cpgl_debug("Invalid message size %d\n", (int)msglen); + logt_print(LOG_DEBUG, "Invalid message size %d\n", (int)msglen); } process_request((struct cpg_lock_msg *)msg, nodeid, pid); @@ -1180,7 +1186,7 @@ cpg_config_change(cpg_handle_t h, old_msg(&m); if (total_members == 0) { - cpgl_debug("JOIN: Setting up initial node list\n"); + logt_print(LOG_DEBUG, "JOIN: Setting up initial node list\n"); for (x = 0; x < memberlen; x++) { for (y = 0; y < joinlen; y++) { if (join[y].nodeid == members[x].nodeid) @@ -1191,16 +1197,16 @@ cpg_config_change(cpg_handle_t h, n = do_alloc(sizeof(*n)); n->nodeid = members[x].nodeid; n->pid = members[x].pid; - cpgl_debug("JOIN: node %d.%u\n", n->nodeid, n->pid); + logt_print(LOG_DEBUG, "JOIN: node %d.%u\n", n->nodeid, n->pid); list_insert(&group_members, n); } } - cpgl_debug("JOIN: Done\n"); + logt_print(LOG_DEBUG, "JOIN: Done\n"); total_members = memberlen; } - //cpgl_debug("members %d now, %d joined, %d left\n", memberlen, joinlen, leftlen); + //logt_print(LOG_DEBUG, "members %d now, %d joined, %d left\n", memberlen, joinlen, leftlen); #if 0 /* XXX process join on receipt of JOIN message rather than here @@ -1209,7 +1215,7 @@ cpg_config_change(cpg_handle_t h, for (x = 0; x < joinlen; x++) { n = do_alloc(sizeof(*n)); n->nodeid = join[x].nodeid; - cpgl_debug("ADD: node %d\n", n->nodeid); + logt_print(LOG_DEBUG, "ADD: node %d\n", n->nodeid); list_insert(&group_members, n); } #endif @@ -1219,17 +1225,17 @@ cpg_config_change(cpg_handle_t h, if (n->nodeid == left[x].nodeid) { if (n->pid == left[x].pid) { if (n->nodeid == my_node_id) { - cpgl_debug("Received DELETE message for self. Exiting."); + logt_print(LOG_DEBUG, "Received DELETE message for self. Exiting."); exit(0); } list_remove(&group_members, n); - cpgl_debug("DELETE: node %d.%u\n", n->nodeid, n->pid); + logt_print(LOG_DEBUG, "DELETE: node %d.%u\n", n->nodeid, n->pid); del_node(n->nodeid); cpglock_members_removed++; free(n); break; } else { - cpgl_debug("DUPE NODE %d LEFT (%u != %u)\n", + logt_print(LOG_DEBUG, "DUPE NODE %d LEFT (%u != %u)\n", n->nodeid, n->pid, left[x].pid); } } @@ -1241,11 +1247,11 @@ cpg_config_change(cpg_handle_t h, total_members = 0; #if 0 - cpgl_debug("MEMBERS:"); + logt_print(LOG_DEBUG, "MEMBERS:"); list_for(&group_members, n, y) { - cpgl_debug(" %d.%u", n->nodeid, n->pid); + logt_print(LOG_DEBUG, " %d.%u", n->nodeid, n->pid); } - cpgl_debug("\n"); + logt_print(LOG_DEBUG, "\n"); #endif return; @@ -1336,6 +1342,35 @@ cpg_init(void) return 0; } +static void init_logging(int reconf) +{ + int ccs_handle; + int mode = LOG_MODE_OUTPUT_FILE | LOG_MODE_OUTPUT_SYSLOG; + int syslog_facility = SYSLOGFACILITY; + int syslog_priority = SYSLOGLEVEL; + char logfile[PATH_MAX]; + int logfile_priority = SYSLOGLEVEL; + int debug = 0; + + memset(logfile, 0, PATH_MAX); + snprintf(logfile, sizeof(logfile) - 1, LOGDIR "/cpglockd.log"); + + ccs_handle = ccs_connect(); + if (ccs_handle > 0) { + ccs_read_logging(ccs_handle, "cpglockd", &debug, &mode, + &syslog_facility, &syslog_priority, &logfile_priority, logfile); + ccs_disconnect(ccs_handle); + } + + if (nofork) + mode |= LOG_MODE_OUTPUT_STDERR; + + if (!reconf) + logt_init("cpglockd", mode, syslog_facility, syslog_priority, logfile_priority, logfile); + else + logt_conf("cpglockd", mode, syslog_facility, syslog_priority, logfile_priority, logfile); +} + int main(int argc, char **argv) { @@ -1345,7 +1380,6 @@ main(int argc, char **argv) int afd = -1; int cman_fd; int n,x; - int nofork = 0; int wait_for_quorum = 1; int wait_for_fencing = 1; int opt; @@ -1391,6 +1425,7 @@ main(int argc, char **argv) } update_pidfile((char *) "cpglockd"); } + init_logging(0); setup_signal(SIGPIPE, SIG_IGN); setup_signal(SIGTERM, flag_shutdown); @@ -1520,7 +1555,7 @@ main(int argc, char **argv) ret = fenced_node_info(pf_node->nodeid, &fn); if (ret < 0) { - cpgl_debug("Unable to get last fenced time for node %d\n", + logt_print(LOG_DEBUG, "Unable to get last fenced time for node %d\n", pf_node->nodeid); victim = 1; lft = 0; @@ -1530,7 +1565,7 @@ main(int argc, char **argv) } if (lft > pf_node->fail_time) { - cpgl_debug("Fencing for node %d finished at %ld (>%ld)\n", + logt_print(LOG_DEBUG, "Fencing for node %d finished at %ld (>%ld)\n", pf_node->nodeid, lft, pf_node->fail_time); list_remove(&pending_fencing, pf_node); free(pf_node); @@ -1554,18 +1589,18 @@ main(int argc, char **argv) do { usleep(250000); if (fenced_node_info(pf_node->nodeid, &fn) < 0) { - cpgl_debug("Unable to get fenced data for node %d\n", + logt_print(LOG_DEBUG, "Unable to get fenced data for node %d\n", pf_node->nodeid); } else victim = fn.victim; } while (!victim && retries++ < 4); if (!victim) { - cpgl_debug("First entry in list and victim == 0, removing %d from pending fencing\n", pf_node->nodeid); + logt_print(LOG_DEBUG, "First entry in list and victim == 0, removing %d from pending fencing\n", pf_node->nodeid); list_remove(&pending_fencing, pf_node); free(pf_node); } else { - cpgl_debug("fenced_node.victim flipped to 1 for %d\n", + logt_print(LOG_DEBUG, "fenced_node.victim flipped to 1 for %d\n", pf_node->nodeid); } goto fence_check; @@ -1604,7 +1639,7 @@ main(int argc, char **argv) continue; --n; if (read_retry(client->fd, &m, sizeof(m), NULL) < 0) { - cpgl_debug("Closing client fd %d pid %d: %d\n", + logt_print(LOG_DEBUG, "Closing client fd %d pid %d: %d\n", client->fd, client->pid, errno); del_client(client->fd); @@ -1619,7 +1654,7 @@ main(int argc, char **argv) } if (m.request == MSG_UNLOCK) { - //cpgl_debug("Unlock from fd %d\n", client->fd); + //logt_print(LOG_DEBUG, "Unlock from fd %d\n", client->fd); find_lock(&m); if (grant_next(&m) == 0) send_unlock(&m);

1 0

cluster: STABLE32 - cpglockd: make network comm endian free
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=07… Commit: 07b8bd6d379430d3050bfee680c466d3dc96f6ab Parent: 1cd7dfd41c74fe140c46833c336c37200015c252 Author: Fabio M. Di Nitto <fdinitto(a)redhat.com> AuthorDate: Tue Apr 24 14:55:01 2012 +0200 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 cpglockd: make network comm endian free Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/include/cpglock-internal.h | 6 +++--- rgmanager/src/daemons/cpglockd.c | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/rgmanager/include/cpglock-internal.h b/rgmanager/include/cpglock-internal.h index 25725c6..904df37 100644 --- a/rgmanager/include/cpglock-internal.h +++ b/rgmanager/include/cpglock-internal.h @@ -26,16 +26,16 @@ typedef enum { } cpg_lock_req_t; /* Mixed architecture not supported yet */ -struct __attribute__((packed)) cpg_lock_msg { - char resource[96]; +struct cpg_lock_msg { int32_t request; uint32_t owner_nodeid; uint32_t owner_pid; uint32_t flags; uint32_t lockid; uint32_t owner_tid; + char resource[96]; char pad[8]; -}; /* 128 */ +} __attribute__((packed)); /* 128 */ #define CPG_LOCKD_NAME "cpglockd" diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 1bd53d3..aa052b3 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -24,6 +24,7 @@ #include "cpglock.h" #include "cpglock-internal.h" #include "list.h" +#include "platform.h" struct request_node { list_head(); @@ -398,6 +399,16 @@ insert_client(int fd) list_append(&clients, n); } +static void +swab_cpg_lock_msg(struct cpg_lock_msg *m) +{ + swab32(m->request); + swab32(m->owner_nodeid); + swab32(m->owner_pid); + swab32(m->flags); + swab32(m->lockid); + swab32(m->owner_tid); +} /* forward request from client */ static int @@ -406,6 +417,8 @@ send_lock_msg(struct cpg_lock_msg *m) struct iovec iov; int ret; + swab_cpg_lock_msg(m); + iov.iov_base = m; iov.iov_len = sizeof (*m); @@ -1096,6 +1109,8 @@ process_join(struct cpg_lock_msg *m, uint32_t nodeid, uint32_t pid) static int process_request(struct cpg_lock_msg *m, uint32_t nodeid, uint32_t pid) { + swab_cpg_lock_msg(m); + if (m->request == MSG_HALT) { cpgl_debug("FAULT: Halting operations; see node %d\n", m->owner_nodeid); while (1)

1 0

cluster: STABLE32 - Clean up the pid file when exiting after running with -f
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=d6… Commit: d6212e80ad256dd8695feca7b50ee21dc1f681fa Parent: 07b8bd6d379430d3050bfee680c466d3dc96f6ab Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Tue Apr 24 09:16:31 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:10 2012 +0200 Clean up the pid file when exiting after running with -f Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/src/daemons/cpglockd.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index aa052b3..9f9f635 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -1649,7 +1649,6 @@ out2: out3: cman_finish(cman_handle); out4: - if (!nofork) - daemon_cleanup(); + daemon_cleanup(); return exit_status; }

1 0

cluster: STABLE32 - rgmanager: cleanup binary linking and include dirs
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=28… Commit: 281412810bd0ef44e48acd274af924a33a978a15 Parent: 237d91e9c292ae0e997f777934eb20c745369b4f Author: Fabio M. Di Nitto <fdinitto(a)redhat.com> AuthorDate: Tue Apr 24 13:51:36 2012 +0200 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 rgmanager: cleanup binary linking and include dirs Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/src/daemons/Makefile | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diff --git a/rgmanager/src/daemons/Makefile b/rgmanager/src/daemons/Makefile index 336b6cf..d125eaa 100644 --- a/rgmanager/src/daemons/Makefile +++ b/rgmanager/src/daemons/Makefile @@ -49,7 +49,8 @@ OBJS5= cpglockdump.o CFLAGS += -DSHAREDIR=\"${sharedir}\" -D_GNU_SOURCE CFLAGS += -fPIC -CFLAGS += -I${ccsincdir} -I${cmanincdir} -I${dlmincdir} -I${logtincdir} +CFLAGS += -I${ccsincdir} -I${cmanincdir} -I${corosyncincdir} +CFLAGS += -I${dlmincdir} -I${fencedincdir} -I${logtincdir} CFLAGS += `xml2-config --cflags` -I${slangincdir} CFLAGS += -I$(S)/../../include CFLAGS += -I${incdir} @@ -63,8 +64,8 @@ CCS_LDFLAGS += -L${ccslibdir} -lccs CMAN_LDFLAGS += -L${cmanlibdir} -lcman LOGSYS_LDFLAGS += -L${logtlibdir} -llogthread DLM_LDFLAGS += -L${dlmlibdir} -ldlm -CPG_LDFLAGS += -lcpg -FENCED_LDFLAGS += -lfenced +CPG_LDFLAGS += -L${corosynclibdir} -lcpg +FENCED_LDFLAGS += -L${fencedlibdir} -lfenced XML2_LDFLAGS += `xml2-config --libs` SLANG_LDFLAGS += -L${slanglibdir} -lslang EXTRA_LDFLAGS += -lpthread

1 0

cluster: STABLE32 - cpglock: drop unused define
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=1c… Commit: 1cd7dfd41c74fe140c46833c336c37200015c252 Parent: 281412810bd0ef44e48acd274af924a33a978a15 Author: Fabio M. Di Nitto <fdinitto(a)redhat.com> AuthorDate: Tue Apr 24 14:36:50 2012 +0200 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 cpglock: drop unused define Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/include/cpglock-internal.h | 2 -- rgmanager/src/clulib/libcpglock.c | 1 - 2 files changed, 0 insertions(+), 3 deletions(-) diff --git a/rgmanager/include/cpglock-internal.h b/rgmanager/include/cpglock-internal.h index d6734eb..25725c6 100644 --- a/rgmanager/include/cpglock-internal.h +++ b/rgmanager/include/cpglock-internal.h @@ -37,8 +37,6 @@ struct __attribute__((packed)) cpg_lock_msg { char pad[8]; }; /* 128 */ -pid_t _gettid (void); - #define CPG_LOCKD_NAME "cpglockd" #endif diff --git a/rgmanager/src/clulib/libcpglock.c b/rgmanager/src/clulib/libcpglock.c index 9fe5ead..d554281 100644 --- a/rgmanager/src/clulib/libcpglock.c +++ b/rgmanager/src/clulib/libcpglock.c @@ -226,7 +226,6 @@ cpg_unlock(void *handle, struct cpg_lock *lock) l.owner_nodeid = lock->owner_nodeid; l.owner_pid = lock->owner_pid; l.lockid = lock->local_id; - //l.owner_tid = _gettid(); l.owner_tid = 0; lock->state = LOCK_FREE;

1 0

cluster: STABLE32 - cpglockd: fix a startup race condition
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=23… Commit: 237d91e9c292ae0e997f777934eb20c745369b4f Parent: dfc510ae98605633e86d27eec584c7768af70065 Author: Fabio M. Di Nitto <fdinitto(a)redhat.com> AuthorDate: Tue Apr 24 12:07:25 2012 +0200 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 cpglockd: fix a startup race condition [root@clusternet-node1 shm]# cpglockd -f nodeid 1 already in group with PID 8126 8126 Unable to join CPG group in some cases the cpg membership is updated between cpglockd joining and cpglockd requesting current membership, triggering the above error incorrectly. check if the pids are the same, we can be rather sure that it's us (given previous fix for startup pid/lock check) Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/src/daemons/cpglockd.c | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 5b6ea79..1bd53d3 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -1309,10 +1309,12 @@ cpg_init(void) for (i = 0 ; i < cpg_member_list_len ; i++) { if (member_list[i].nodeid == my_node_id) { - fprintf(stderr, "nodeid %d already in group with PID %u\n", - member_list[i].nodeid, member_list[i].pid); - cpg_fin(); - return -1; + if (member_list[i].pid != getpid()) { + fprintf(stderr, "nodeid %d already in group with PID %u %u\n", + member_list[i].nodeid, member_list[i].pid, getpid()); + cpg_fin(); + return -1; + } } }

1 0

cluster: STABLE32 - cpglockd: fix pid file creation and check if daemon is already running
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=df… Commit: dfc510ae98605633e86d27eec584c7768af70065 Parent: 3e92e3dbf9a423b4f9ddb50094969bea0b1d178a Author: Fabio M. Di Nitto <fdinitto(a)redhat.com> AuthorDate: Tue Apr 24 12:03:37 2012 +0200 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 cpglockd: fix pid file creation and check if daemon is already running Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/include/daemon_init.h | 1 + rgmanager/src/clulib/daemon_init.c | 3 +-- rgmanager/src/daemons/cpglockd.c | 11 ++++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/rgmanager/include/daemon_init.h b/rgmanager/include/daemon_init.h index 0252386..99e87c2 100644 --- a/rgmanager/include/daemon_init.h +++ b/rgmanager/include/daemon_init.h @@ -3,6 +3,7 @@ int check_pid_valid(pid_t pid, char *prog); int check_process_running(char *prog, pid_t * pid); +void update_pidfile(char *prog); void daemon_init(char *prog); void daemon_cleanup(void); diff --git a/rgmanager/src/clulib/daemon_init.c b/rgmanager/src/clulib/daemon_init.c index 7703d34..7d68ffd 100644 --- a/rgmanager/src/clulib/daemon_init.c +++ b/rgmanager/src/clulib/daemon_init.c @@ -30,7 +30,6 @@ /* * Local prototypes. */ -static void update_pidfile(char *prog); static int setup_sigmask(void); static char pid_filename[PATH_MAX]; @@ -138,7 +137,7 @@ check_process_running(char *prog, pid_t * pid) } -static void +void update_pidfile(char *prog) { FILE *fp = NULL; diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 0cba6f3..5b6ea79 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -1363,8 +1363,17 @@ main(int argc, char **argv) } } - if (!nofork) + if (!nofork) { daemon_init((char *) "cpglockd"); + } else { + pid_t pid; + if (check_process_running((char *) "cpglockd", &pid) && (pid != getpid())) { + fprintf(stderr, + "cpglockd is already running\n"); + return -1; + } + update_pidfile((char *) "cpglockd"); + } setup_signal(SIGPIPE, SIG_IGN); setup_signal(SIGTERM, flag_shutdown);

1 0

cluster: STABLE32 - cpglockd: cleanup startup init scripts and drop code from rgmanager
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=3e… Commit: 3e92e3dbf9a423b4f9ddb50094969bea0b1d178a Parent: f046d797bd2c0774642274db5cc9b063bda8efec Author: Fabio M. Di Nitto <fdinitto(a)redhat.com> AuthorDate: Tue Apr 24 11:38:50 2012 +0200 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 cpglockd: cleanup startup init scripts and drop code from rgmanager also fix conditionals to require or not cpglockd within rgmanager Signed-off-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/include/cpglock-internal.h | 4 -- rgmanager/init.d/rgmanager.in | 18 +++++++- rgmanager/src/daemons/main.c | 79 ++++++++++++---------------------- 3 files changed, 44 insertions(+), 57 deletions(-) diff --git a/rgmanager/include/cpglock-internal.h b/rgmanager/include/cpglock-internal.h index 27917c7..d6734eb 100644 --- a/rgmanager/include/cpglock-internal.h +++ b/rgmanager/include/cpglock-internal.h @@ -11,10 +11,6 @@ #define CPG_LOCKD_SOCK "/var/run/cpglockd.sk" #endif -#ifndef CPGLOCKD_BIN_PATH -# define CPGLOCKD_BIN_PATH "/usr/sbin/cpglockd" -#endif - #include <stdint.h> typedef enum { diff --git a/rgmanager/init.d/rgmanager.in b/rgmanager/init.d/rgmanager.in index 5b7219c..e7ee916 100644 --- a/rgmanager/init.d/rgmanager.in +++ b/rgmanager/init.d/rgmanager.in @@ -6,8 +6,8 @@ # ### BEGIN INIT INFO # Provides: rgmanager -# Required-Start: cman -# Required-Stop: cman +# Required-Start: cman cpglockd +# Required-Stop: cman cpglockd # Default-Start: # Default-Stop: # Short-Description: Starts and stops Red Hat Service (resource group) Manager @@ -72,6 +72,15 @@ stop_cluster() done } +# +# start cpglock if necessary +# +start_cpglockd() +{ + rings="$(corosync-objctl 2>/dev/null |grep ringnumber | wc -l)" + [ "$rings" -gt "1" ] && service cpglockd start +} + rtrn=0 if [ "$EUID" != "0" ]; then @@ -89,6 +98,11 @@ start) # required subdirectories for proper operations mkdir -p /var/run/cluster + # failure to start cpglockd should not be fatal here + # rgmanager will take care to report the correct errors + # later + start_cpglockd || true + if status $RGMGRD > /dev/null 2>&1; then success else diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c index 769b695..15f6215 100644 --- a/rgmanager/src/daemons/main.c +++ b/rgmanager/src/daemons/main.c @@ -798,73 +798,43 @@ statedump(int __attribute__ ((unused)) sig) signalled++; } +/* + * return -1 on error + * 0 cpglock is not needed + * 1 cpglock is needed + */ static int cpglockd_needed(void) { char *v; int ccsfd; - int start_cpglockd = 0; - + int need = 0; ccsfd = ccs_force_connect(NULL, 0); if (ccsfd < 0) return -1; - if (ccs_get(ccsfd, "/cluster/cman/altmulticast/@addr", &v) == 0) { - if (v != NULL) { - int ret; - free(v); - ret = ccs_get(ccsfd, - "/cluster/clusternodes/clusternode/altname/@name", &v); - if (ret == 0) { - if (v != NULL) { - free(v); - start_cpglockd = 1; - } - } + if (ccs_get(ccsfd, + "/cluster/clusternodes/clusternode/altname/@name", &v) == 0) { + if (v) { + need = 1; + free(v); } } - if (start_cpglockd) { + if (need) { if (ccs_get(ccsfd, "/cluster/totem/@rrp_mode", &v) == 0) { if (v != NULL) { - if (!strcasecmp(v, "none")) - start_cpglockd = 0; + if (!strcasecmp(v, "none")) { + need = 0; + } free(v); } } } ccs_disconnect(ccsfd); - return start_cpglockd; -} - -static int -cpglockd_start(void) { - int pid; - int status; - - pid = fork(); - if (pid == -1) - return -1; - - if (pid) { - waitpid(pid, &status, 0); - if (!WIFEXITED(status) || WEXITSTATUS(status)) { - logt_print(LOG_NOTICE, "Unable to start cpglockd\n"); - return -1; - } - } else { - int i; - - for (i = 3 ; i < __FD_SETSIZE ; i++) - close(i); - - execl(CPGLOCKD_BIN_PATH, "cpglockd", NULL); - exit(-1); - } - - return 0; + return need; } static int @@ -1147,11 +1117,18 @@ main(int argc, char **argv) } xmlInitParser(); - if (cpg_lock_opt == 1 || (cpg_lock_opt != -1 && cpglockd_needed() == 1)) { - cpglockd_start(); + + cpg_locks = 0; + if (cpg_lock_opt == 1) { /* force enable */ cpg_locks = 1; - } else - cpg_locks = 0; + } else if (cpg_lock_opt == 0) { /* autodetect */ + cpg_locks = cpglockd_needed(); + if (cpg_locks < 0) { + printf("Unable to determine if cpglock is required!\n"); + cman_finish(clu); + return -1; + } + } if (!cpg_locks) { if (clu_lock_init(rgmanager_lsname) != 0) { @@ -1160,12 +1137,12 @@ main(int argc, char **argv) return -1; } } else { + logt_print(LOG_INFO, "Using CPG for locking (EXPERIMENTAL)\n"); if (cpg_lock_initialize() != 0) { printf("Locks not working!\n"); cman_finish(clu); return -1; } - logt_print(LOG_INFO, "Using CPG for locking (EXPERIMENTAL)\n"); } memset(&me, 0, sizeof(me));

1 0

cluster: STABLE32 - Update Makefile for cpglockd init script
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=fe… Commit: fe28e52bd0e7596e90fd71b1f1ceee8aaa935481 Parent: 73369096429f6f67ba15389f50aab16bfbf668c2 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Mon Apr 23 18:27:55 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 Update Makefile for cpglockd init script Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/init.d/Makefile | 19 +++++++++++++------ 1 files changed, 13 insertions(+), 6 deletions(-) diff --git a/rgmanager/init.d/Makefile b/rgmanager/init.d/Makefile index 4c9a44f..7346884 100644 --- a/rgmanager/init.d/Makefile +++ b/rgmanager/init.d/Makefile @@ -1,19 +1,26 @@ -TARGET= rgmanager +TARGET1=rgmanager +TARGET2=cpglockd -INITDT=$(TARGET) +INITDT=$(TARGET1) $(TARGET2) -all: $(TARGET) +all: $(TARGET1) $(TARGET2) include ../../make/defines.mk include $(OBJDIR)/make/clean.mk include $(OBJDIR)/make/install.mk include $(OBJDIR)/make/uninstall.mk -$(TARGET): $(S)/$(TARGET).in - cat $(S)/$(TARGET).in | sed \ +$(TARGET1): $(S)/$(TARGET1).in + cat $(S)/$(TARGET1).in | sed \ -e 's#@SBINDIR@#${sbindir}#g' \ -e 's#@INITDDIR@#${initddir}#g' \ - > $(TARGET) + > $(TARGET1) + +$(TARGET2): $(S)/$(TARGET2).in + cat $(S)/$(TARGET2).in | sed \ + -e 's#@SBINDIR@#${sbindir}#g' \ + -e 's#@INITDDIR@#${initddir}#g' \ + > $(TARGET2) clean: generalclean

1 0

cluster: STABLE32 - Add a simple init script for cpglockd. The rgmanager init script still needs to be updated to call this when RRP mode is enabled and CPG locks are not explicitly disabled.
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=73… Commit: 73369096429f6f67ba15389f50aab16bfbf668c2 Parent: 61a1cfe97578fc75783bf9592e52fc5a04f2d3af Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Mon Apr 23 18:25:40 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 Add a simple init script for cpglockd. The rgmanager init script still needs to be updated to call this when RRP mode is enabled and CPG locks are not explicitly disabled. Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/init.d/cpglockd.in | 142 ++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 142 insertions(+), 0 deletions(-) diff --git a/rgmanager/init.d/cpglockd.in b/rgmanager/init.d/cpglockd.in new file mode 100644 index 0000000..9fbb43c --- /dev/null +++ b/rgmanager/init.d/cpglockd.in @@ -0,0 +1,142 @@ +#!/bin/bash +# +# chkconfig: - 99 01 +# description: Starts and stops the CPG lock daemon +# +# +### BEGIN INIT INFO +# Provides: cpglockd +# Required-Start: cman +# Required-Stop: cman +# Default-Start: +# Default-Stop: +# Short-Description: Starts and stops the CPG lock daemon +# Description: Starts and stops the CPG lock daemon +### END INIT INFO + +ID="CPG lock daemon" +CPGLOCKD="cpglockd" + +# set secure PATH +PATH="/sbin:/bin:/usr/sbin:/usr/bin:@SBINDIR@" + +success() +{ + echo -ne "[ OK ]\r" +} + +failure() +{ + echo -ne "[FAILED]\r" +} + +status() +{ + pid=$(pidof $1 2>/dev/null) + statusrtrn=$? + if [ $statusrtrn -ne 0 ]; then + echo "$1 is stopped" + else + echo "$1 (pid $pid) is running..." + fi + return $statusrtrn +} + +# rpm based distros +if [ -d /etc/sysconfig ]; then + [ -f @INITDDIR@/functions ] && . @INITDDIR@/functions + [ -f /etc/sysconfig/cluster ] && . /etc/sysconfig/cluster + [ -f /etc/sysconfig/rgmanager ] && . /etc/sysconfig/rgmanager + [ -f /etc/sysconfig/cpglockd ] && . /etc/sysconfig/cpglockd + [ -z "$LOCK_FILE" ] && LOCK_FILE="/var/lock/subsys/cpglockd" +fi + +# deb based distros +if [ -d /etc/default ]; then + [ -f /etc/default/cluster ] && . /etc/default/cluster + [ -f /etc/default/rgmanager ] && . /etc/default/rgmanager + [ -f /etc/default/cpglockd ] && . /etc/default/cpglockd + [ -z "$LOCK_FILE" ] && LOCK_FILE="/var/lock/cpglockd" +fi + +# +# Stop cpglockd +# +stop_cpglockd() +{ + kill -TERM $(pidof $CPGLOCKD) +} + +rtrn=0 + +if [ "$EUID" != "0" ]; then + echo "Only root can execute $0 script" + exit 4 +fi + +case "$1" in +start) + echo -n "Starting $ID: " + + # most recent distributions use tmpfs for /var/run + # to avoid to clean it up on every boot. + # they also assume that init scripts will create + # required subdirectories for proper operations + mkdir -p /var/run/cluster + + if status $CPGLOCKD > /dev/null 2>&1; then + success + else + if $CPGLOCKD $CPGLOCKD_OPTS; then + touch $LOCK_FILE + success + else + failure + rtrn=1 + fi + fi + echo +;; +restart) + $0 stop + $0 start +;; +condrestart|try-restart) + if status $CPGLOCKD > /dev/null 2>&1; then + $0 stop + $0 start + rtrn=$? + fi +;; +reload|force-reload) + # not required anymore + # return not implemented + rtrn=3 +;; +status) + status $CPGLOCKD + rtrn=$? +;; +stop) + echo -n "Stopping $ID: " + + if status $CPGLOCKD > /dev/null 2>&1; then + if stop_cpglockd; then + success + else + failure + rtrn=1 + fi + else + success + fi + echo + rm -f $LOCK_FILE +;; +*) + echo "usage: $0 {start|stop|restart|condrestart|try-restart|reload|force-reload|status}" + rtrn=2 +;; +esac + +exit $rtrn

1 0

cluster: STABLE32 - Cleanup the Makefile and add support to the cpglockd init script for options set in /etc/sysconfig/cpglockd
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=f0… Commit: f046d797bd2c0774642274db5cc9b063bda8efec Parent: fe28e52bd0e7596e90fd71b1f1ceee8aaa935481 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Mon Apr 23 22:09:44 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 Cleanup the Makefile and add support to the cpglockd init script for options set in /etc/sysconfig/cpglockd Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/init.d/Makefile | 19 ++++++------------- rgmanager/init.d/cpglockd.in | 5 +++++ rgmanager/init.d/cpglockd.init.defaults.in | 7 +++++++ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/rgmanager/init.d/Makefile b/rgmanager/init.d/Makefile index 7346884..959ddb3 100644 --- a/rgmanager/init.d/Makefile +++ b/rgmanager/init.d/Makefile @@ -1,26 +1,19 @@ -TARGET1=rgmanager -TARGET2=cpglockd +TARGET=rgmanager cpglockd cpglockd.init.defaults -INITDT=$(TARGET1) $(TARGET2) +INITDT=rgmanager cpglockd -all: $(TARGET1) $(TARGET2) +all: $(TARGET) include ../../make/defines.mk include $(OBJDIR)/make/clean.mk include $(OBJDIR)/make/install.mk include $(OBJDIR)/make/uninstall.mk -$(TARGET1): $(S)/$(TARGET1).in - cat $(S)/$(TARGET1).in | sed \ +%: $(S)/%.in + cat $^ | sed \ -e 's#@SBINDIR@#${sbindir}#g' \ -e 's#@INITDDIR@#${initddir}#g' \ - > $(TARGET1) - -$(TARGET2): $(S)/$(TARGET2).in - cat $(S)/$(TARGET2).in | sed \ - -e 's#@SBINDIR@#${sbindir}#g' \ - -e 's#@INITDDIR@#${initddir}#g' \ - > $(TARGET2) + > $@ clean: generalclean diff --git a/rgmanager/init.d/cpglockd.in b/rgmanager/init.d/cpglockd.in index 9fbb43c..772e0e8 100644 --- a/rgmanager/init.d/cpglockd.in +++ b/rgmanager/init.d/cpglockd.in @@ -59,6 +59,9 @@ if [ -d /etc/default ]; then [ -z "$LOCK_FILE" ] && LOCK_FILE="/var/lock/cpglockd" fi +[ -z "$CPGLOCKD_WAIT_FOR_QUORUM" ] && CPGLOCKD_WAIT_FOR_QUORUM="yes" +[ -z "$CPGLOCKD_WAIT_FOR_FENCE_JOIN" ] && CPGLOCKD_WAIT_FOR_FENCE_JOIN="yes" + # # Stop cpglockd # @@ -83,6 +86,8 @@ start) # they also assume that init scripts will create # required subdirectories for proper operations mkdir -p /var/run/cluster + [ "$CPGLOCKD_WAIT_FOR_QUORUM" = "no" ] && CPGLOCKD_OPTS+=" -Q" + [ "$CPGLOCKD_WAIT_FOR_FENCE_JOIN" = "no" ] && CPGLOCKD_OPTS+=" -F" if status $CPGLOCKD > /dev/null 2>&1; then success diff --git a/rgmanager/init.d/cpglockd.init.defaults.in b/rgmanager/init.d/cpglockd.init.defaults.in new file mode 100644 index 0000000..29caf27 --- /dev/null +++ b/rgmanager/init.d/cpglockd.init.defaults.in @@ -0,0 +1,7 @@ +# CPGLOCKD_WAIT_FOR_QUORUM -- Wait for cluster quorum formation +# at startup. +#CPGLOCKD_WAIT_FOR_QUORUM=yes +# +# CPGLOCKD_WAIT_FOR_FENCE_JOIN -- Wait for the current node to join the +# fence domain at startup. +#CPGLOCKD_WAIT_FOR_FENCE_JOIN=yes

1 0

cluster: STABLE32 - Cleanup and exit if we receive a SIGTERM
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=61… Commit: 61a1cfe97578fc75783bf9592e52fc5a04f2d3af Parent: d14707dd26e1a8031e0afd4bd4214777a6af5814 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Mon Apr 23 16:46:33 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 Cleanup and exit if we receive a SIGTERM Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/src/daemons/cpglockd.c | 103 ++++++++++++++++++++++++------------- 1 files changed, 67 insertions(+), 36 deletions(-) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 2163f82..0cba6f3 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -1,5 +1,6 @@ #include <stdio.h> #include <signal.h> +#include <signals.h> #include <sys/types.h> #include <sys/time.h> #include <sys/select.h> @@ -71,7 +72,8 @@ static int local_lockid = 0; static int message_count = 0; static int joined = 0; static int cluster_quorate = 0; -static int shutdown_requested = 0; +static int cman_shutdown_requested = 0; +static int shutdown_pending = 0; static cman_node_t cman_nodes[CPG_MEMBERS_MAX]; static int cman_node_count; @@ -79,6 +81,12 @@ static int cman_node_count; static cman_node_t old_cman_nodes[CPG_MEMBERS_MAX]; static int old_cman_node_count; +static void +flag_shutdown(int __attribute__ ((unused)) sig) +{ + shutdown_pending = 1; +} + static int cman_nodes_lost(cman_node_t *old_nodes, size_t old_node_len, @@ -211,7 +219,7 @@ cman_callback(cman_handle_t ch, void *privdata, int reason, int arg) } } } else if (reason == CMAN_REASON_TRY_SHUTDOWN) { - shutdown_requested = 1; + cman_shutdown_requested = 1; } } @@ -220,7 +228,7 @@ wait_for_fencing_join(int nodeid) { if (node_has_fencing(nodeid)) { cpgl_debug("Waiting for fence domain join operation to complete\n"); - while (!fence_domain_joined()) + while (!shutdown_pending && !fence_domain_joined()) sleep(1); cpgl_debug("Fence domain joined\n"); } else @@ -230,7 +238,7 @@ wait_for_fencing_join(int nodeid) static void wait_for_quorum_formation(cman_handle_t ch) { cpgl_debug("Waiting for quorum to form\n"); - while (!(cluster_quorate = cman_is_quorate(ch))) + while (!shutdown_pending && !(cluster_quorate = cman_is_quorate(ch))) sleep(1); cpgl_debug("Quorum formed\n"); } @@ -243,7 +251,7 @@ cman_connect(cman_handle_t *ch) *ch = cman_init(NULL); if (!*ch) { cpgl_debug("Waiting for CMAN to start\n"); - while (!(*ch = cman_init(NULL))) + while (!shutdown_pending && !(*ch = cman_init(NULL))) sleep(1); cpgl_debug("CMAN started\n"); } @@ -407,7 +415,7 @@ send_lock_msg(struct cpg_lock_msg *m) cpgl_debug("send_lock_msg() failed\n"); usleep(250000); } - } while (ret != CPG_OK); + } while (ret != CPG_OK && !shutdown_pending); return 0; } @@ -1325,6 +1333,7 @@ main(int argc, char **argv) int wait_for_fencing = 1; int opt; int ret; + int exit_status = 0; struct cpg_lock_msg m; struct client_node *client; cman_handle_t cman_handle = NULL; @@ -1354,29 +1363,43 @@ main(int argc, char **argv) } } - signal(SIGPIPE, SIG_IGN); + if (!nofork) + daemon_init((char *) "cpglockd"); + + setup_signal(SIGPIPE, SIG_IGN); + setup_signal(SIGTERM, flag_shutdown); cman_connect(&cman_handle); if (cman_handle == NULL) { fprintf(stderr, "Unable to connect to cman\n"); - return -1; + exit_status = -1; + goto out4; } + if (shutdown_pending) + goto out3; + if (wait_for_quorum) wait_for_quorum_formation(cman_handle); + if (shutdown_pending) + goto out3; + memset(&my_node, 0, sizeof(my_node)); cman_get_node(cman_handle, CMAN_NODEID_US, &my_node); if (my_node.cn_nodeid == 0) { fprintf(stderr, "Unable to get our cluster node ID\n"); - cman_finish(cman_handle); - return -1; + exit_status = -1; + goto out3; } if (wait_for_fencing) wait_for_fencing_join(my_node.cn_nodeid); + if (shutdown_pending) + goto out3; + memset(&cman_nodes, 0, sizeof(cman_nodes)); cman_node_count = 0; @@ -1385,24 +1408,23 @@ main(int argc, char **argv) &cman_node_count, cman_nodes); if (ret < 0) { fprintf(stderr, "Unable to get cman nodes list\n"); - cman_finish(cman_handle); - return -1; + exit_status = -1; + goto out3; } cman_fd = cman_get_fd(cman_handle); if (cman_fd < 0) { fprintf(stderr, "Error: cman fd is %d\n", cman_fd); - cman_finish(cman_handle); - return -1; + exit_status = -1; + goto out3; } cman_start_notification(cman_handle, cman_callback); if (cpg_init() < 0) { fprintf(stderr, "Unable to join CPG group\n"); - cman_stop_notification(cman_handle); - cman_finish(cman_handle); - return -1; + exit_status = -1; + goto out2; } assert(my_node.cn_nodeid == my_node_id); @@ -1411,21 +1433,17 @@ main(int argc, char **argv) if (fd < 0) { fprintf(stderr, "Error connecting to %s: %s\n", CPG_LOCKD_SOCK, strerror(errno)); - cman_stop_notification(cman_handle); - cman_finish(cman_handle); - cpg_fin(); - return -1; + exit_status = -1; + goto out1; } - if (!nofork) - daemon_init((char *) "cpglockd"); - - cpg_local_get(cpg, &my_node_id); cpg_fd_get(cpg, &cpgfd); - if (send_join() < 0) - return -1; + if (cpgfd < 0 || send_join() < 0) { + exit_status = -1; + goto out; + } - while (1) { + while (!shutdown_pending) { struct timeval tv; struct pending_fence_node *pf_node; @@ -1448,22 +1466,24 @@ main(int argc, char **argv) n = select_retry(x+1, &rfds, NULL, NULL, &tv); if (n < 0) { fprintf(stderr, "Error: select: %s\n", strerror(errno)); - return -1; + exit_status = -1; + goto out; } if (FD_ISSET(cman_fd, &rfds)) { if (cman_dispatch(cman_handle, CMAN_DISPATCH_ALL) < 0) { fprintf(stderr, "Fatal: cman_dispatch() failed: %s\n", strerror(errno)); - return -1; + exit_status = -1; + goto out; } --n; } - if (shutdown_requested) { - cman_replyto_shutdown(cman_handle, 1); + if (cman_shutdown_requested) { fprintf(stderr, "Fatal: cman requested shutdown.\n"); - return -1; + cman_replyto_shutdown(cman_handle, 1); + goto out; } fence_check: @@ -1526,6 +1546,9 @@ main(int argc, char **argv) } } + if (shutdown_pending) + goto out; + /* While fencing is pending for any nodes pause lock activity. */ if (pending_fencing != NULL) { usleep(500000); @@ -1590,9 +1613,17 @@ main(int argc, char **argv) } while (n); } +out: + close(fd); + unlink(CPG_LOCKD_SOCK); +out1: + cpg_fin(); +out2: cman_stop_notification(cman_handle); +out3: cman_finish(cman_handle); - cpg_fin(); - - return 0; +out4: + if (!nofork) + daemon_cleanup(); + return exit_status; }

1 0

cluster: STABLE32 - Allow -C to take a 1 or 0 argument to respectively force enable and force disable of CPG locks.
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=d1… Commit: d14707dd26e1a8031e0afd4bd4214777a6af5814 Parent: 096fe8d673933d2830aa0d6f37de9f863200a823 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Mon Apr 23 14:48:35 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 Allow -C to take a 1 or 0 argument to respectively force enable and force disable of CPG locks. Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/src/daemons/main.c | 27 +++++++++++++++++++++++---- 1 files changed, 23 insertions(+), 4 deletions(-) diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c index 4b88df2..769b695 100644 --- a/rgmanager/src/daemons/main.c +++ b/rgmanager/src/daemons/main.c @@ -1057,14 +1057,15 @@ int main(int argc, char **argv) { int rv, do_init = 1; - char foreground = 0, wd = 1, cpg_locks = 0; + int cpg_locks = 0; + char foreground = 0, wd = 1, cpg_lock_opt = 0; cman_node_t me; msgctx_t *cluster_ctx; msgctx_t *local_ctx; pthread_t th; cman_handle_t clu = NULL; - while ((rv = getopt(argc, argv, "wfdNqC")) != EOF) { + while ((rv = getopt(argc, argv, "wfdNqC::")) != EOF) { switch (rv) { case 'w': wd = 0; @@ -1082,7 +1083,22 @@ main(int argc, char **argv) rgm_dbus_notify = 0; break; case 'C': - cpg_locks = 1; + if (optarg) { + if (!strcmp(optarg, "0")) { + /* force disable */ + cpg_lock_opt = -1; + } else if (!strcmp(optarg, "1")) { + /* force enable */ + cpg_lock_opt = 1; + } else { + fprintf(stderr, + "Unknown argument for -C: argument must be 1 or 0\n"); + return -1; + } + } else { + /* auto handling: enable when RRP mode is enabled */ + cpg_lock_opt = 0; + } break; default: return 1; @@ -1131,8 +1147,11 @@ main(int argc, char **argv) } xmlInitParser(); - if (cpg_locks || cpglockd_needed() == 1) + if (cpg_lock_opt == 1 || (cpg_lock_opt != -1 && cpglockd_needed() == 1)) { cpglockd_start(); + cpg_locks = 1; + } else + cpg_locks = 0; if (!cpg_locks) { if (clu_lock_init(rgmanager_lsname) != 0) {

1 0

cluster: STABLE32 - rgmanager: Sync cpglock bits with upstream
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=d3… Commit: d347b183dac60754351566ed82c0844d9c9425f9 Parent: 92f177068cb324124f2d9ed7f117c6e3c2ed2a4a Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Mon Apr 23 10:44:26 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 rgmanager: Sync cpglock bits with upstream Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/src/clulib/libcpglock.c | 4 +- rgmanager/src/daemons/Makefile | 5 +- rgmanager/src/daemons/cpglockd.c | 423 +++++++++++++++++++++++++++++++++++-- 3 files changed, 413 insertions(+), 19 deletions(-) diff --git a/rgmanager/src/clulib/libcpglock.c b/rgmanager/src/clulib/libcpglock.c index 2db65a5..9fe5ead 100644 --- a/rgmanager/src/clulib/libcpglock.c +++ b/rgmanager/src/clulib/libcpglock.c @@ -250,8 +250,10 @@ cpg_lock_dump(FILE *fp) memset(&l, 0, sizeof(l)); l.request = MSG_DUMP; - if (write_retry(fd, &l, sizeof(l), 0) < 0) + if (write_retry(fd, &l, sizeof(l), 0) < 0) { + close(fd); return -1; + } while (read_retry(fd, &c, 1, 0) == 1) fprintf(fp, "%c", c); diff --git a/rgmanager/src/daemons/Makefile b/rgmanager/src/daemons/Makefile index dbd75ea..61ae6e4 100644 --- a/rgmanager/src/daemons/Makefile +++ b/rgmanager/src/daemons/Makefile @@ -61,6 +61,7 @@ CMAN_LDFLAGS += -L${cmanlibdir} -lcman LOGSYS_LDFLAGS += -L${logtlibdir} -llogthread DLM_LDFLAGS += -L${dlmlibdir} -ldlm CPG_LDFLAGS += -lcpg +FENCED_LDFLAGS += -lfenced XML2_LDFLAGS += `xml2-config --libs` SLANG_LDFLAGS += -L${slanglibdir} -lslang EXTRA_LDFLAGS += -lpthread @@ -101,8 +102,8 @@ ${TARGET3}: ${TARGET1} ln -sf ${TARGET1} ${TARGET3} ${TARGET4}: ${OBJS4} ${LDDEPS} - $(CC) -o $@ $^ $(CPG_LDFLAGS) $(EXTRA_LDFLAGS) $(LDFLAGS) - + $(CC) -o $@ $^ $(CPG_LDFLAGS) $(CMAN_LDFLAGS) $(CCS_LDFLAGS) \ + $(FENCED_LDFLAGS) $(EXTRA_LDFLAGS) $(LDFLAGS) check: rg_test cd tests && ./runtests.sh diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 80d05a1..2163f82 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -5,19 +5,25 @@ #include <sys/select.h> #include <errno.h> #include <unistd.h> +#include <stdlib.h> #include <malloc.h> #include <string.h> #include <time.h> +#include <assert.h> #include <sys/uio.h> + #include <corosync/cpg.h> +#include <ccs.h> +#include <libcman.h> +#include <libfenced.h> + #include "daemon_init.h" #include "sock.h" #include "cpglock.h" #include "cpglock-internal.h" #include "list.h" - struct request_node { list_head(); struct cpg_lock l; @@ -28,6 +34,12 @@ struct lock_node { struct cpg_lock l; }; +struct pending_fence_node { + list_head(); + int nodeid; + uint64_t fail_time; +}; + struct client_node { list_head(); int fd; @@ -48,6 +60,7 @@ struct msg_node { /* Local vars */ static cpg_handle_t cpg; static uint32_t my_node_id = 0; +static struct pending_fence_node *pending_fencing = NULL; static struct request_node *requests = NULL; static struct lock_node *locks = NULL; static struct client_node *clients = NULL; @@ -57,7 +70,186 @@ static int total_members = 0; static int local_lockid = 0; static int message_count = 0; static int joined = 0; +static int cluster_quorate = 0; +static int shutdown_requested = 0; + +static cman_node_t cman_nodes[CPG_MEMBERS_MAX]; +static int cman_node_count; + +static cman_node_t old_cman_nodes[CPG_MEMBERS_MAX]; +static int old_cman_node_count; + +static int +cman_nodes_lost(cman_node_t *old_nodes, + size_t old_node_len, + cman_node_t *new_nodes, + size_t new_node_len, + cman_node_t *lost_nodes, + size_t lost_nodes_size) +{ + int i; + size_t lost_nodes_len = 0; + + if (lost_nodes_size < old_node_len) { + cpgl_debug("Lost nodes array smaller than old nodes array"); + return -1; + } + + for (i = 0 ; i < old_node_len ; i++) { + if (old_nodes[i].cn_member) { + int present_in_new_list = 0; + int j; + + for (j = 0 ; j < new_node_len ; j++) { + if (new_nodes[j].cn_nodeid == old_nodes[i].cn_nodeid) { + present_in_new_list = 1; + if (!new_nodes[j].cn_member) { + memcpy(&lost_nodes[lost_nodes_len++], + &old_nodes[i], sizeof(*lost_nodes)); + break; + } + } + } + + if (!present_in_new_list) { + memcpy(&lost_nodes[lost_nodes_len++], + &old_nodes[i], sizeof(*lost_nodes)); + } + } + } + + return lost_nodes_len; +} + +static int +node_has_fencing(int nodeid) +{ + int ccs_desc; + char *val = NULL; + char buf[1024]; + int ret = 1; + + ccs_desc = ccs_connect(); + if (ccs_desc < 0) { + cpgl_debug("Unable to connect to ccsd\n"); + /* Assume node has fencing */ + return 1; + } + + snprintf(buf, sizeof(buf), + "/cluster/clusternodes/clusternode[@nodeid=\"%d\"]" + "/fence/method/device/@name", nodeid); + + if (ccs_get(ccs_desc, buf, &val) != 0) + ret = 0; + if (val) + free(val); + ccs_disconnect(ccs_desc); + return ret; +} + +static int +fence_domain_joined(void) { + int ret; + struct fenced_node fn; + + ret = fenced_node_info(FENCED_NODEID_US, &fn); + if (ret < 0) { + cpgl_debug("Unable to determine fence domain membership\n"); + return 0; + } + + return fn.member; +} + +static void +cman_callback(cman_handle_t ch, void *privdata, int reason, int arg) +{ + if (reason == CMAN_REASON_STATECHANGE) { + int i; + int ret; + int nl; + time_t cur_time = time(NULL); + size_t nodes_elem = sizeof(cman_nodes) / sizeof(cman_nodes[0]); + cman_node_t lost_nodes[nodes_elem]; + + cluster_quorate = arg; + + memcpy(&old_cman_nodes, &cman_nodes, sizeof(old_cman_nodes)); + old_cman_node_count = cman_node_count; + + memset(&cman_nodes, 0, sizeof(cman_nodes)); + cman_node_count = 0; + + ret = cman_get_nodes(ch, nodes_elem, &cman_node_count, cman_nodes); + if (ret < 0) { + cpgl_debug("Unable to get cman nodes list\n"); + return; + } + + nl = cman_nodes_lost(old_cman_nodes, old_cman_node_count, + cman_nodes, cman_node_count, lost_nodes, nodes_elem); + if (nl < 0) { + cpgl_debug("Unable to get list of lost nodes"); + return; + } + + for (i = 0 ; i < nl ; i++) { + int cur_nodeid = lost_nodes[i].cn_nodeid; + + if (node_has_fencing(cur_nodeid)) { + struct pending_fence_node *pf; + + cpgl_debug("Lost node %d\n", cur_nodeid); + pf = do_alloc(sizeof(*pf)); + pf->nodeid = cur_nodeid; + pf->fail_time = cur_time; + list_append(&pending_fencing, pf); + } else { + cpgl_debug("Lost node %d but fencing not configured\n", + cur_nodeid); + } + } + } else if (reason == CMAN_REASON_TRY_SHUTDOWN) { + shutdown_requested = 1; + } +} + +static void +wait_for_fencing_join(int nodeid) +{ + if (node_has_fencing(nodeid)) { + cpgl_debug("Waiting for fence domain join operation to complete\n"); + while (!fence_domain_joined()) + sleep(1); + cpgl_debug("Fence domain joined\n"); + } else + cpgl_debug("No fencing is configured, not waiting\n"); +} + +static void +wait_for_quorum_formation(cman_handle_t ch) { + cpgl_debug("Waiting for quorum to form\n"); + while (!(cluster_quorate = cman_is_quorate(ch))) + sleep(1); + cpgl_debug("Quorum formed\n"); +} + +static int +cman_connect(cman_handle_t *ch) +{ + assert(ch != NULL); + + *ch = cman_init(NULL); + if (!*ch) { + cpgl_debug("Waiting for CMAN to start\n"); + while (!(*ch = cman_init(NULL))) + sleep(1); + cpgl_debug("CMAN started\n"); + } + return 0; +} static const char * ls2str(int x) @@ -209,9 +401,14 @@ send_lock_msg(struct cpg_lock_msg *m) iov.iov_base = m; iov.iov_len = sizeof (*m); - ret = cpg_mcast_joined(cpg, CPG_TYPE_AGREED, &iov, 1); - if (ret != CPG_OK) - return -1; + do { + ret = cpg_mcast_joined(cpg, CPG_TYPE_AGREED, &iov, 1); + if (ret != CPG_OK) { + cpgl_debug("send_lock_msg() failed\n"); + usleep(250000); + } + } while (ret != CPG_OK); + return 0; } @@ -397,8 +594,23 @@ del_node(uint32_t nodeid) struct lock_node *l; int x, recovered = 0, granted = 0; - if (group_members->nodeid != my_node_id) + if (group_members->nodeid != my_node_id) { + /* + ** Update the owner and pid of any locks owned by the deleted + ** node to those of the oldest node in the group. + */ + list_for(&locks, l, x) { + if (l->l.owner_nodeid == nodeid) { + cpgl_debug("RECOVERY: LOCK UPDATED: %s [%d:%d]=>[%d:%d]\n", + l->l.resource, + l->l.owner_nodeid, l->l.owner_pid, + group_members->nodeid, group_members->pid); + l->l.owner_nodeid = group_members->nodeid; + l->l.owner_pid = group_members->pid; + } + } return; + } cpgl_debug("RECOVERY: I am oldest node in the group, recovering locks\n"); @@ -415,19 +627,34 @@ del_node(uint32_t nodeid) send_lock_msg(&m); list_for(&locks, l, x) { - if (l->l.owner_nodeid == nodeid && - l->l.state == LOCK_HELD) { + if (l->l.owner_nodeid == nodeid && l->l.state == LOCK_HELD) { cpgl_debug("RECOVERY: Releasing %s held by dead node %d\n", l->l.resource, nodeid); l->l.state = LOCK_FREE; strncpy(m.resource, l->l.resource, sizeof(m.resource)); - if (grant_next(&m) == 0) + if (grant_next(&m) == 0) { + cpgl_debug("RECOVERY: HELD LOCK UPDATED: %s [%d:%d]=>[%d:%d]\n", + l->l.resource, + l->l.owner_nodeid, l->l.owner_pid, + my_node_id, getpid()); + l->l.owner_nodeid = my_node_id; + l->l.owner_pid = getpid(); + m.owner_nodeid = my_node_id; + m.owner_pid = getpid(); send_unlock(&m); + } ++recovered; - } else if (l->l.state == LOCK_FREE) { - if (grant_next(&m) == 0) - send_unlock(&m); + } else if (l->l.owner_nodeid == nodeid && l->l.state == LOCK_FREE) { + strncpy(m.resource, l->l.resource, sizeof(m.resource)); + if (grant_next(&m) == 0) { + cpgl_debug("RECOVERY: FREE LOCK UPDATED: %s [%d:%d]=>[%d:%d]\n", + l->l.resource, + l->l.owner_nodeid, l->l.owner_pid, + my_node_id, getpid()); + l->l.owner_nodeid = my_node_id; + l->l.owner_pid = getpid(); + } ++granted; } } @@ -968,6 +1195,10 @@ cpg_config_change(cpg_handle_t h, list_for(&group_members, n, y) { if (n->nodeid == left[x].nodeid) { if (n->pid == left[x].pid) { + if (n->nodeid == my_node_id) { + cpgl_debug("Received DELETE message for self. Exiting."); + exit(0); + } list_remove(&group_members, n); cpgl_debug("DELETE: node %d.%u\n", n->nodeid, n->pid); del_node(n->nodeid); @@ -1087,37 +1318,101 @@ main(int argc, char **argv) int fd; int cpgfd; int afd = -1; + int cman_fd; int n,x; int nofork = 0; + int wait_for_quorum = 1; + int wait_for_fencing = 1; int opt; + int ret; struct cpg_lock_msg m; struct client_node *client; + cman_handle_t cman_handle = NULL; + cman_node_t my_node; - while ((opt = getopt(argc, argv, "fh")) != EOF) { + while ((opt = getopt(argc, argv, "FQfh")) != EOF) { switch (opt) { + case 'F': + wait_for_fencing = 0; + break; + case 'Q': + wait_for_quorum = 0; + break; case 'f': nofork = 1; break; case 'h': printf("Usage: %s [options]\n\ + -F Don't wait for the current node to join the fencing domain at startup\n\ + -Q Don't wait for quorum formation at startup\n\ -f Don't daemonize\n\ -h Print this help message\n", argv[0]); return 0; + default: + return -1; } } signal(SIGPIPE, SIG_IGN); + cman_connect(&cman_handle); + if (cman_handle == NULL) { + fprintf(stderr, "Unable to connect to cman\n"); + return -1; + } + + if (wait_for_quorum) + wait_for_quorum_formation(cman_handle); + + memset(&my_node, 0, sizeof(my_node)); + cman_get_node(cman_handle, CMAN_NODEID_US, &my_node); + + if (my_node.cn_nodeid == 0) { + fprintf(stderr, "Unable to get our cluster node ID\n"); + cman_finish(cman_handle); + return -1; + } + + if (wait_for_fencing) + wait_for_fencing_join(my_node.cn_nodeid); + + memset(&cman_nodes, 0, sizeof(cman_nodes)); + cman_node_count = 0; + + ret = cman_get_nodes(cman_handle, + sizeof(cman_nodes) / sizeof(cman_nodes[0]), + &cman_node_count, cman_nodes); + if (ret < 0) { + fprintf(stderr, "Unable to get cman nodes list\n"); + cman_finish(cman_handle); + return -1; + } + + cman_fd = cman_get_fd(cman_handle); + if (cman_fd < 0) { + fprintf(stderr, "Error: cman fd is %d\n", cman_fd); + cman_finish(cman_handle); + return -1; + } + + cman_start_notification(cman_handle, cman_callback); + if (cpg_init() < 0) { fprintf(stderr, "Unable to join CPG group\n"); + cman_stop_notification(cman_handle); + cman_finish(cman_handle); return -1; } + assert(my_node.cn_nodeid == my_node_id); + fd = sock_listen(CPG_LOCKD_SOCK); if (fd < 0) { fprintf(stderr, "Error connecting to %s: %s\n", CPG_LOCKD_SOCK, strerror(errno)); + cman_stop_notification(cman_handle); + cman_finish(cman_handle); cpg_fin(); return -1; } @@ -1131,6 +1426,9 @@ main(int argc, char **argv) return -1; while (1) { + struct timeval tv; + struct pending_fence_node *pf_node; + FD_ZERO(&rfds); x = client_fdset(&rfds); FD_SET(fd, &rfds); @@ -1139,13 +1437,101 @@ main(int argc, char **argv) FD_SET(cpgfd, &rfds); if (cpgfd > x) x = cpgfd; - - n = select_retry(x+1, &rfds, NULL, NULL, NULL); + cman_fd = cman_get_fd(cman_handle); + FD_SET(cman_fd, &rfds); + if (cman_fd > x) + x = cman_fd; + + tv.tv_sec = 0; + tv.tv_usec = 500000; + + n = select_retry(x+1, &rfds, NULL, NULL, &tv); if (n < 0) { - perror("select"); + fprintf(stderr, "Error: select: %s\n", strerror(errno)); return -1; } + if (FD_ISSET(cman_fd, &rfds)) { + if (cman_dispatch(cman_handle, CMAN_DISPATCH_ALL) < 0) { + fprintf(stderr, "Fatal: cman_dispatch() failed: %s\n", + strerror(errno)); + return -1; + } + --n; + } + + if (shutdown_requested) { + cman_replyto_shutdown(cman_handle, 1); + fprintf(stderr, "Fatal: cman requested shutdown.\n"); + return -1; + } + + fence_check: + list_for(&pending_fencing, pf_node, x) { + int victim; + uint64_t lft; + struct fenced_node fn; + + ret = fenced_node_info(pf_node->nodeid, &fn); + if (ret < 0) { + cpgl_debug("Unable to get last fenced time for node %d\n", + pf_node->nodeid); + victim = 1; + lft = 0; + } else { + victim = fn.victim; + lft = fn.last_fenced_time; + } + + if (lft > pf_node->fail_time) { + cpgl_debug("Fencing for node %d finished at %ld (>%ld)\n", + pf_node->nodeid, lft, pf_node->fail_time); + list_remove(&pending_fencing, pf_node); + free(pf_node); + goto fence_check; + } + + /* + ** If node A either fails or leaves the cluster cleanly while + ** fencing is outstanding on another node (or nodes), we aren't + ** able to determine whether A has failed or left cleanly until + ** fencing has completed for the nodes that have left or failed + ** prior to A while any fencing is outstanding. The first entry + ** in this list will be the node that has left or failed longer + ** ago than any others in the list. If fenced has not set + ** victim to 1 by now, we can deduce it has left cleanly, and we + ** don't need to wait for it. + */ + if (!victim && !x) { + int retries = 0; + /* Wait up to 1s for fenced to set victim */ + do { + usleep(250000); + if (fenced_node_info(pf_node->nodeid, &fn) < 0) { + cpgl_debug("Unable to get fenced data for node %d\n", + pf_node->nodeid); + } else + victim = fn.victim; + } while (!victim && retries++ < 4); + + if (!victim) { + cpgl_debug("First entry in list and victim == 0, removing %d from pending fencing\n", pf_node->nodeid); + list_remove(&pending_fencing, pf_node); + free(pf_node); + } else { + cpgl_debug("fenced_node.victim flipped to 1 for %d\n", + pf_node->nodeid); + } + goto fence_check; + } + } + + /* While fencing is pending for any nodes pause lock activity. */ + if (pending_fencing != NULL) { + usleep(500000); + continue; + } + if (FD_ISSET(fd, &rfds)) { afd = accept(fd, NULL, NULL); insert_client(afd); @@ -1153,7 +1539,10 @@ main(int argc, char **argv) } if (FD_ISSET(cpgfd, &rfds)) { - cpg_dispatch(cpg, CPG_DISPATCH_ALL); + if (cpg_dispatch(cpg, CPG_DISPATCH_ALL) != CPG_OK) { + fprintf(stderr, "Fatal: Lost CPG connection.\n"); + return -1; + } --n; } @@ -1201,6 +1590,8 @@ main(int argc, char **argv) } while (n); } + cman_stop_notification(cman_handle); + cman_finish(cman_handle); cpg_fin(); return 0;

1 0

cluster: STABLE32 - rgmanager: Sync cpglock bits with upstream
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=92… Commit: 92f177068cb324124f2d9ed7f117c6e3c2ed2a4a Parent: 009c89cb56610a5ba2da310a36d7bc2822e4d919 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Wed Mar 28 14:17:12 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 rgmanager: Sync cpglock bits with upstream Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/include/cpglock-internal.h | 6 + rgmanager/src/clulib/libcpglock.c | 11 +- rgmanager/src/daemons/cpglockd.c | 239 +++++++++++++++++++++------------- 3 files changed, 164 insertions(+), 92 deletions(-) diff --git a/rgmanager/include/cpglock-internal.h b/rgmanager/include/cpglock-internal.h index 1c322ae..d6734eb 100644 --- a/rgmanager/include/cpglock-internal.h +++ b/rgmanager/include/cpglock-internal.h @@ -1,6 +1,12 @@ #ifndef _CPGLOCK_INT_H #define _CPGLOCK_INT_H +#ifdef CPGLOCK_DEBUG +# define cpgl_debug(format, args...) do { printf(format, ##args); } while (0) +#else +# define cpgl_debug(format, args...) do { } while (0) +#endif + #ifndef CPG_LOCKD_SOCK #define CPG_LOCKD_SOCK "/var/run/cpglockd.sk" #endif diff --git a/rgmanager/src/clulib/libcpglock.c b/rgmanager/src/clulib/libcpglock.c index 8e5c7ff..2db65a5 100644 --- a/rgmanager/src/clulib/libcpglock.c +++ b/rgmanager/src/clulib/libcpglock.c @@ -97,7 +97,7 @@ cpg_lock(void *handle, const char *resource, lock_flag_t flags, struct cpg_lock if (!h) { errno = EINVAL; - goto out; + return -1; } pthread_mutex_lock(&h->mutex); @@ -145,7 +145,8 @@ cpg_lock(void *handle, const char *resource, lock_flag_t flags, struct cpg_lock } if (strcmp(r.resource, l.resource)) { - printf("NOTE: msg for wrong lock want: %s got: %s\n", l.resource, r.resource); + cpgl_debug("NOTE: msg for wrong lock want: %s got: %s\n", + l.resource, r.resource); add_pending(&h->pending, &r); pthread_mutex_unlock(&h->mutex); usleep(random() & 16383); @@ -153,7 +154,8 @@ cpg_lock(void *handle, const char *resource, lock_flag_t flags, struct cpg_lock continue; } if (r.owner_tid != l.owner_tid) { - printf("NOTE: msg for wrong seq want: %d got: %d\n", l.owner_tid, r.owner_tid ); + cpgl_debug("NOTE: msg for wrong seq want: %d got: %d\n", + l.owner_tid, r.owner_tid ); add_pending(&h->pending, &r); pthread_mutex_unlock(&h->mutex); usleep(random() & 16383); @@ -169,7 +171,8 @@ cpg_lock(void *handle, const char *resource, lock_flag_t flags, struct cpg_lock if (r.request == MSG_NAK) { errno = EAGAIN; - return -1; + ret = -1; + goto out; } if (r.request != MSG_GRANT) { diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 3ed132b..80d05a1 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -11,6 +11,7 @@ #include <sys/uio.h> #include <corosync/cpg.h> +#include "daemon_init.h" #include "sock.h" #include "cpglock.h" #include "cpglock-internal.h" @@ -36,6 +37,7 @@ struct client_node { struct member_node { list_head(); int nodeid; + uint32_t pid; }; struct msg_node { @@ -104,7 +106,7 @@ dump_state(FILE *fp) if (group_members) { fprintf(fp, "Participants:"); list_for(&group_members, m, x) { - fprintf(fp, " %d", m->nodeid); + fprintf(fp, " %d.%u", m->nodeid, m->pid); } fprintf(fp, "\n"); } @@ -202,11 +204,15 @@ static int send_lock_msg(struct cpg_lock_msg *m) { struct iovec iov; + int ret; iov.iov_base = m; iov.iov_len = sizeof (*m); - return cpg_mcast_joined(cpg, CPG_TYPE_AGREED, &iov, 1); + ret = cpg_mcast_joined(cpg, CPG_TYPE_AGREED, &iov, 1); + if (ret != CPG_OK) + return -1; + return 0; } @@ -225,7 +231,7 @@ send_grant(struct request_node *n) { struct cpg_lock_msg m; - printf("-> sending grant %s to %d:%d:%d\n", + cpgl_debug("-> sending grant %s to %d:%d:%d\n", n->l.resource, n->l.owner_nodeid, n->l.owner_pid, n->l.owner_tid); memset(&m, 0, sizeof(m)); @@ -284,7 +290,7 @@ grant_next(struct cpg_lock_msg *m) /* Send grant */ if (r->l.state == LOCK_PENDING) { - printf("LOCK %s: grant to %d:%d:%d\n", m->resource, + cpgl_debug("LOCK %s: grant to %d:%d:%d\n", m->resource, r->l.owner_nodeid, r->l.owner_pid, r->l.owner_tid); /* don't send dup grants */ r->l.state = LOCK_HELD; @@ -321,9 +327,9 @@ purge_requests(uint32_t nodeid, uint32_t pid) if (count) { if (pid) { - printf("RECOVERY: purged %d requests from %d:%d\n", count, nodeid, pid); + cpgl_debug("RECOVERY: purged %d requests from %d:%d\n", count, nodeid, pid); } else { - printf("RECOVERY: purged %d requests from node %d\n", count, nodeid); + cpgl_debug("RECOVERY: purged %d requests from node %d\n", count, nodeid); } } } @@ -351,11 +357,12 @@ del_client(int fd) if (!pid) return; - printf("RECOVERY: Looking for locks held by PID %d\n", pid); + cpgl_debug("RECOVERY: Looking for locks held by PID %d\n", pid); /* This may not be needed */ purge_requests(my_node_id, pid); + memset(&m, 0, sizeof(m)); m.request = MSG_PURGE; m.owner_nodeid = my_node_id; m.owner_pid = pid; @@ -368,7 +375,7 @@ del_client(int fd) l->l.state != LOCK_HELD) continue; - printf("RECOVERY: Releasing %s \n", l->l.resource); + cpgl_debug("RECOVERY: Releasing %s \n", l->l.resource); l->l.state = LOCK_FREE; strncpy(m.resource, l->l.resource, sizeof(m.resource)); @@ -377,9 +384,9 @@ del_client(int fd) } if (recovered) { - printf("RECOVERY: %d locks from local PID %d\n", recovered, pid); + cpgl_debug("RECOVERY: %d locks from local PID %d\n", recovered, pid); } - printf("RECOVERY: Complete\n"); + cpgl_debug("RECOVERY: Complete\n"); } @@ -393,13 +400,14 @@ del_node(uint32_t nodeid) if (group_members->nodeid != my_node_id) return; - printf("RECOVERY: I am oldest node in the group, recovering locks\n"); + cpgl_debug("RECOVERY: I am oldest node in the group, recovering locks\n"); /* pass 1: purge outstanding requests from this node. */ /* This may not be needed */ purge_requests(nodeid, 0); + memset(&m, 0, sizeof(m)); m.request = MSG_PURGE; m.owner_nodeid = nodeid; m.owner_pid = 0; @@ -409,8 +417,8 @@ del_node(uint32_t nodeid) list_for(&locks, l, x) { if (l->l.owner_nodeid == nodeid && l->l.state == LOCK_HELD) { - printf("RECOVERY: Releasing %s held by dead node %d\n", l->l.resource, - nodeid); + cpgl_debug("RECOVERY: Releasing %s held by dead node %d\n", + l->l.resource, nodeid); l->l.state = LOCK_FREE; strncpy(m.resource, l->l.resource, sizeof(m.resource)); @@ -425,13 +433,13 @@ del_node(uint32_t nodeid) } if (recovered) { - printf("RECOVERY: %d locks from node %d\n", recovered, nodeid); + cpgl_debug("RECOVERY: %d locks from node %d\n", recovered, nodeid); } if (granted) { - printf("RECOVERY: %d pending locks granted\n", granted); + cpgl_debug("RECOVERY: %d pending locks granted\n", granted); } - printf("RECOVERY: Complete\n"); + cpgl_debug("RECOVERY: Complete\n"); } @@ -493,7 +501,7 @@ grant_client(struct lock_node *l) struct client_node *c; struct cpg_lock_msg m; - //memset(&m, 0, sizeof(m)); + memset(&m, 0, sizeof(m)); strncpy(m.resource, l->l.resource, sizeof(m.resource)); m.request = MSG_GRANT; m.owner_pid = l->l.owner_pid; @@ -504,19 +512,19 @@ grant_client(struct lock_node *l) c = find_client(l->l.owner_pid); if (!c) { - printf("can't find client for pid %d\n", l->l.owner_pid); + cpgl_debug("can't find client for pid %d\n", l->l.owner_pid); return 1; } if (c->fd < 0) { - printf(" Client has bad fd\n"); + cpgl_debug(" Client has bad fd\n"); return -1; } if (write_retry(c->fd, &m, sizeof(m), NULL) < 0) { /* no client anymore; drop and send to next guy XXX */ /* This should be handled by our main loop */ - //printf("Failed to notify client!\n"); + //cpgl_debug("Failed to notify client!\n"); } return 0; @@ -529,6 +537,7 @@ nak_client(struct request_node *l) struct client_node *c; struct cpg_lock_msg m; + memset(&m, 0, sizeof(m)); strncpy(m.resource, l->l.resource, sizeof(m.resource)); m.request = MSG_NAK; m.owner_pid = l->l.owner_pid; @@ -537,19 +546,19 @@ nak_client(struct request_node *l) c = find_client(l->l.owner_pid); if (!c) { - printf("can't find client for pid %d\n", l->l.owner_pid); + cpgl_debug("can't find client for pid %d\n", l->l.owner_pid); return 1; } if (c->fd < 0) { - printf(" Client has bad fd\n"); + cpgl_debug(" Client has bad fd\n"); return -1; } if (write_retry(c->fd, &m, sizeof(m), NULL) < 0) { /* no client anymore; drop and send to next guy XXX */ /* This should be handled by our main loop */ - //printf("Failed to notify client!\n"); + //cpgl_debug("Failed to notify client!\n"); } return 0; @@ -581,7 +590,7 @@ process_lock(struct cpg_lock_msg *m) if (!joined) return 0; - printf("LOCK %s: queue for %d:%d:%d\n", m->resource, + cpgl_debug("LOCK %s: queue for %d:%d:%d\n", m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid); queue_request(m); @@ -657,14 +666,14 @@ process_grant(struct cpg_lock_msg *m, uint32_t nodeid) if (l->l.state == LOCK_HELD) { if (m->owner_pid == 0 || m->owner_nodeid == 0) { - printf("GRANT averted\n"); + cpgl_debug("GRANT averted\n"); return 0; } } else { l->l.state = LOCK_HELD; } - printf("GRANT %s: to %d:%d:%d\n", + cpgl_debug("GRANT %s: to %d:%d:%d\n", m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid); @@ -705,7 +714,8 @@ process_grant(struct cpg_lock_msg *m, uint32_t nodeid) if (group_members->nodeid == my_node_id && !is_member(l->l.owner_nodeid)) { - printf("GRANT to non-member %d; giving to next requestor\n", l->l.owner_nodeid); + cpgl_debug("GRANT to non-member %d; giving to next requestor\n", + l->l.owner_nodeid); l->l.state = LOCK_FREE; if (grant_next(m) == 0) @@ -717,7 +727,7 @@ process_grant(struct cpg_lock_msg *m, uint32_t nodeid) /* Record lock state since we now know it */ /* Allocate a lock structure */ - l = do_alloc(sizeof(*m)); + l = do_alloc(sizeof(*l)); strncpy(l->l.resource, m->resource, sizeof(l->l.resource)); l->l.state = LOCK_HELD; l->l.owner_nodeid = m->owner_nodeid; @@ -779,10 +789,13 @@ process_unlock(struct cpg_lock_msg *m, uint32_t nodeid) GRANT to the first node on the request queue */ if (l->l.owner_nodeid == m->owner_nodeid && l->l.owner_pid == m->owner_pid) { - printf("UNLOCK %s: %d:%d:%d\n", m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid); + cpgl_debug("UNLOCK %s: %d:%d:%d\n", + m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid); l->l.state = LOCK_FREE; - //if (grant_next(m) != 0) - //l->l.state = LOCK_PENDING; + if (l->l.owner_nodeid == my_node_id) { + if (grant_next(m) != 0) + l->l.state = LOCK_PENDING; + } } } @@ -802,7 +815,7 @@ find_lock(struct cpg_lock_msg *m) list_for(&locks, l, x) { if (m->lockid == l->l.local_id) { strncpy(m->resource, l->l.resource, sizeof(m->resource)); - printf("LOCK %d -> %s\n", m->lockid, m->resource); + cpgl_debug("LOCK %d -> %s\n", m->lockid, m->resource); m->owner_nodeid = l->l.owner_nodeid; m->owner_pid = l->l.owner_pid; m->owner_tid = l->l.owner_tid; @@ -816,29 +829,29 @@ find_lock(struct cpg_lock_msg *m) static int -process_join(struct cpg_lock_msg *m, uint32_t nodeid) +process_join(struct cpg_lock_msg *m, uint32_t nodeid, uint32_t pid) { struct member_node *n; int x; list_for(&group_members, n, x) { if (n->nodeid == nodeid) { - list_remove(&group_members, n); - list_append(&group_members, n); - printf("JOIN: moving %d to back\n", nodeid); + cpgl_debug("IGNORING JOIN from existing member %d.%d (%d.%d)\n", + nodeid, pid, nodeid, n->pid); return 0; } } n = do_alloc(sizeof(*n)); n->nodeid = nodeid; - printf("JOIN: node %d", n->nodeid); + n->pid = pid; + cpgl_debug("JOIN: node %d.%u", n->nodeid, n->pid); if (nodeid == my_node_id) { - printf(" (self)"); + cpgl_debug(" (self)"); joined = 1; } total_members++; - printf("\n"); + cpgl_debug("\n"); list_insert(&group_members, n); return 0; @@ -846,10 +859,10 @@ process_join(struct cpg_lock_msg *m, uint32_t nodeid) static int -process_request(struct cpg_lock_msg *m, uint32_t nodeid) +process_request(struct cpg_lock_msg *m, uint32_t nodeid, uint32_t pid) { if (m->request == MSG_HALT) { - printf("FAULT: Halting operations; see node %d\n", m->owner_nodeid); + cpgl_debug("FAULT: Halting operations; see node %d\n", m->owner_nodeid); while (1) sleep(30); } @@ -873,7 +886,7 @@ process_request(struct cpg_lock_msg *m, uint32_t nodeid) purge_requests(m->owner_nodeid, m->owner_pid); break; case MSG_JOIN: - process_join(m, nodeid); + process_join(m, nodeid, pid); break; } @@ -891,10 +904,10 @@ cpg_deliver_func(cpg_handle_t h, { if (msglen != sizeof(struct cpg_lock_msg)) { - printf("Invalid message size %d\n", (int)msglen); + cpgl_debug("Invalid message size %d\n", (int)msglen); } - process_request((struct cpg_lock_msg *)msg, nodeid); + process_request((struct cpg_lock_msg *)msg, nodeid, pid); } @@ -908,7 +921,7 @@ cpg_config_change(cpg_handle_t h, struct member_node *n; size_t x, y; struct cpg_lock_msg m; - + int cpglock_members_removed = 0; memset(&m, 0, sizeof(m)); strncpy(m.resource, "(none)", sizeof(m.resource)); @@ -917,8 +930,7 @@ cpg_config_change(cpg_handle_t h, old_msg(&m); if (total_members == 0) { - - printf("JOIN: Setting up initial node list\n"); + cpgl_debug("JOIN: Setting up initial node list\n"); for (x = 0; x < memberlen; x++) { for (y = 0; y < joinlen; y++) { if (join[y].nodeid == members[x].nodeid) @@ -928,16 +940,17 @@ cpg_config_change(cpg_handle_t h, n = do_alloc(sizeof(*n)); n->nodeid = members[x].nodeid; - printf("JOIN: node %d\n", n->nodeid); + n->pid = members[x].pid; + cpgl_debug("JOIN: node %d.%u\n", n->nodeid, n->pid); list_insert(&group_members, n); } } - printf("JOIN: Done\n"); + cpgl_debug("JOIN: Done\n"); total_members = memberlen; } - //printf("members %d now, %d joined, %d left\n", memberlen, joinlen, leftlen); + //cpgl_debug("members %d now, %d joined, %d left\n", memberlen, joinlen, leftlen); #if 0 /* XXX process join on receipt of JOIN message rather than here @@ -946,34 +959,39 @@ cpg_config_change(cpg_handle_t h, for (x = 0; x < joinlen; x++) { n = do_alloc(sizeof(*n)); n->nodeid = join[x].nodeid; - printf("ADD: node %d\n", n->nodeid); + cpgl_debug("ADD: node %d\n", n->nodeid); list_insert(&group_members, n); } #endif for (x = 0; x < leftlen; x++) { - list_for(&group_members, n, y) { if (n->nodeid == left[x].nodeid) { - list_remove(&group_members, n); - printf("DELETE: node %d\n", n->nodeid); - del_node(n->nodeid); - free(n); - break; + if (n->pid == left[x].pid) { + list_remove(&group_members, n); + cpgl_debug("DELETE: node %d.%u\n", n->nodeid, n->pid); + del_node(n->nodeid); + cpglock_members_removed++; + free(n); + break; + } else { + cpgl_debug("DUPE NODE %d LEFT (%u != %u)\n", + n->nodeid, n->pid, left[x].pid); + } } } - - total_members -= leftlen; - if (total_members < 0) - total_members = 0; } + total_members -= cpglock_members_removed; + if (total_members < 0) + total_members = 0; + #if 0 - printf("MEMBERS:"); + cpgl_debug("MEMBERS:"); list_for(&group_members, n, y) { - printf(" %d", n->nodeid); + cpgl_debug(" %d.%u", n->nodeid, n->pid); } - printf("\n"); + cpgl_debug("\n"); #endif return; @@ -985,9 +1003,8 @@ static cpg_callbacks_t my_callbacks = { .cpg_confchg_fn = cpg_config_change }; - static int -cpg_init(void) +cpg_fin(void) { struct cpg_name gname; @@ -1004,27 +1021,19 @@ cpg_init(void) if (gname.length <= 0) return -1; - memset(&cpg, 0, sizeof(cpg)); - if (cpg_initialize(&cpg, &my_callbacks) != CPG_OK) { - perror("cpg_initialize"); - return -1; - } - - if (cpg_join(cpg, &gname) != CPG_OK) { - perror("cpg_join"); - return -1; - } - - cpg_local_get(cpg, &my_node_id); + cpg_leave(cpg, &gname); + cpg_finalize(cpg); return 0; } - static int -cpg_fin(void) +cpg_init(void) { struct cpg_name gname; + struct cpg_address member_list[64]; + int cpg_member_list_len = 0; + int i, ret; errno = EINVAL; @@ -1039,13 +1048,38 @@ cpg_fin(void) if (gname.length <= 0) return -1; - cpg_leave(cpg, &gname); - cpg_finalize(cpg); + memset(&cpg, 0, sizeof(cpg)); + if (cpg_initialize(&cpg, &my_callbacks) != CPG_OK) { + perror("cpg_initialize"); + return -1; + } + + if (cpg_join(cpg, &gname) != CPG_OK) { + perror("cpg_join"); + return -1; + } + + cpg_local_get(cpg, &my_node_id); + + ret = cpg_membership_get(cpg, &gname, member_list, &cpg_member_list_len); + if (ret != CPG_OK) { + fprintf(stderr, "cpg_membership_get() failed: %s", strerror(errno)); + cpg_fin(); + return -1; + } + + for (i = 0 ; i < cpg_member_list_len ; i++) { + if (member_list[i].nodeid == my_node_id) { + fprintf(stderr, "nodeid %d already in group with PID %u\n", + member_list[i].nodeid, member_list[i].pid); + cpg_fin(); + return -1; + } + } return 0; } - int main(int argc, char **argv) { @@ -1054,14 +1088,43 @@ main(int argc, char **argv) int cpgfd; int afd = -1; int n,x; - + int nofork = 0; + int opt; struct cpg_lock_msg m; struct client_node *client; + while ((opt = getopt(argc, argv, "fh")) != EOF) { + switch (opt) { + case 'f': + nofork = 1; + break; + case 'h': + printf("Usage: %s [options]\n\ + -f Don't daemonize\n\ + -h Print this help message\n", + argv[0]); + return 0; + } + } + signal(SIGPIPE, SIG_IGN); + if (cpg_init() < 0) { + fprintf(stderr, "Unable to join CPG group\n"); + return -1; + } + fd = sock_listen(CPG_LOCKD_SOCK); - cpg_init(); + if (fd < 0) { + fprintf(stderr, "Error connecting to %s: %s\n", + CPG_LOCKD_SOCK, strerror(errno)); + cpg_fin(); + return -1; + } + + if (!nofork) + daemon_init((char *) "cpglockd"); + cpg_local_get(cpg, &my_node_id); cpg_fd_get(cpg, &cpgfd); if (send_join() < 0) @@ -1090,7 +1153,7 @@ main(int argc, char **argv) } if (FD_ISSET(cpgfd, &rfds)) { - cpg_dispatch(cpg, CPG_DISPATCH_ONE); + cpg_dispatch(cpg, CPG_DISPATCH_ALL); --n; } @@ -1103,7 +1166,7 @@ main(int argc, char **argv) continue; --n; if (read_retry(client->fd, &m, sizeof(m), NULL) < 0) { - printf("Closing client fd %d pid %d: %d\n", + cpgl_debug("Closing client fd %d pid %d: %d\n", client->fd, client->pid, errno); del_client(client->fd); @@ -1118,7 +1181,7 @@ main(int argc, char **argv) } if (m.request == MSG_UNLOCK) { - //printf("Unlock from fd %d\n", client->fd); + //cpgl_debug("Unlock from fd %d\n", client->fd); find_lock(&m); if (grant_next(&m) == 0) send_unlock(&m);

1 0

cluster: STABLE32 - rgmanager: Start cpglockd automatically when RRP mode is enabled
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=d9… Commit: d93069e4fc9905a9f4bd417605907d208fb91a80 Parent: d347b183dac60754351566ed82c0844d9c9425f9 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Mon Apr 23 10:49:29 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 rgmanager: Start cpglockd automatically when RRP mode is enabled Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/include/cpglock-internal.h | 4 ++ rgmanager/src/daemons/main.c | 75 +++++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 1 deletions(-) diff --git a/rgmanager/include/cpglock-internal.h b/rgmanager/include/cpglock-internal.h index d6734eb..27917c7 100644 --- a/rgmanager/include/cpglock-internal.h +++ b/rgmanager/include/cpglock-internal.h @@ -11,6 +11,10 @@ #define CPG_LOCKD_SOCK "/var/run/cpglockd.sk" #endif +#ifndef CPGLOCKD_BIN_PATH +# define CPGLOCKD_BIN_PATH "/usr/sbin/cpglockd" +#endif + #include <stdint.h> typedef enum { diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c index 40717e6..4b88df2 100644 --- a/rgmanager/src/daemons/main.c +++ b/rgmanager/src/daemons/main.c @@ -25,6 +25,8 @@ #include <groups.h> #include <rg_dbus.h> +#include <cpglock-internal.h> + #ifdef WRAP_THREADS void dump_thread_states(FILE *); #endif @@ -796,6 +798,74 @@ statedump(int __attribute__ ((unused)) sig) signalled++; } +static int +cpglockd_needed(void) +{ + char *v; + int ccsfd; + int start_cpglockd = 0; + + ccsfd = ccs_force_connect(NULL, 0); + if (ccsfd < 0) + return -1; + + if (ccs_get(ccsfd, "/cluster/cman/altmulticast/@addr", &v) == 0) { + if (v != NULL) { + int ret; + free(v); + + ret = ccs_get(ccsfd, + "/cluster/clusternodes/clusternode/altname/@name", &v); + if (ret == 0) { + if (v != NULL) { + free(v); + start_cpglockd = 1; + } + } + } + } + + if (start_cpglockd) { + if (ccs_get(ccsfd, "/cluster/totem/@rrp_mode", &v) == 0) { + if (v != NULL) { + if (!strcasecmp(v, "none")) + start_cpglockd = 0; + free(v); + } + } + } + + ccs_disconnect(ccsfd); + return start_cpglockd; +} + +static int +cpglockd_start(void) { + int pid; + int status; + + pid = fork(); + if (pid == -1) + return -1; + + if (pid) { + waitpid(pid, &status, 0); + if (!WIFEXITED(status) || WEXITSTATUS(status)) { + logt_print(LOG_NOTICE, "Unable to start cpglockd\n"); + return -1; + } + } else { + int i; + + for (i = 3 ; i < __FD_SETSIZE ; i++) + close(i); + + execl(CPGLOCKD_BIN_PATH, "cpglockd", NULL); + exit(-1); + } + + return 0; +} static int rgmanager_disabled(int ccsfd) @@ -1060,6 +1130,10 @@ main(int argc, char **argv) return -1; } + xmlInitParser(); + if (cpg_locks || cpglockd_needed() == 1) + cpglockd_start(); + if (!cpg_locks) { if (clu_lock_init(rgmanager_lsname) != 0) { printf("Locks not working!\n"); @@ -1096,7 +1170,6 @@ main(int argc, char **argv) We know we're quorate. At this point, we need to read the resource group trees from ccsd. */ - xmlInitParser(); configure_rgmanager(-1, debug, &cluster_timeout); if (shutdown_pending == 1) goto out_ls;

1 0

cluster: STABLE32 - rgmanager: Add a utility to dump the cpglockd state
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=09… Commit: 096fe8d673933d2830aa0d6f37de9f863200a823 Parent: d93069e4fc9905a9f4bd417605907d208fb91a80 Author: Ryan McCabe <rmccabe(a)redhat.com> AuthorDate: Mon Apr 23 10:55:58 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 rgmanager: Add a utility to dump the cpglockd state Signed-off-by: Ryan McCabe <rmccabe(a)redhat.com> --- rgmanager/src/daemons/Makefile | 10 ++++++++-- rgmanager/src/daemons/cpglockdump.c | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/rgmanager/src/daemons/Makefile b/rgmanager/src/daemons/Makefile index 61ae6e4..336b6cf 100644 --- a/rgmanager/src/daemons/Makefile +++ b/rgmanager/src/daemons/Makefile @@ -2,11 +2,12 @@ TARGET1= rgmanager TARGET2= rg_test TARGET3= clurgmgrd TARGET4= cpglockd +TARGET5= cpglockdump -SBINDIRT=$(TARGET1) $(TARGET2) $(TARGET4) +SBINDIRT=$(TARGET1) $(TARGET2) $(TARGET4) $(TARGET5) SBINSYMT=$(TARGET3) -all: depends ${TARGET1} ${TARGET2} ${TARGET3} $(TARGET4) +all: depends ${TARGET1} ${TARGET2} ${TARGET3} $(TARGET4) $(TARGET5) include ../../../make/defines.mk include $(OBJDIR)/make/cobj.mk @@ -44,6 +45,8 @@ OBJS2= test-noccs.o \ OBJS4= cpglockd.o +OBJS5= cpglockdump.o + CFLAGS += -DSHAREDIR=\"${sharedir}\" -D_GNU_SOURCE CFLAGS += -fPIC CFLAGS += -I${ccsincdir} -I${cmanincdir} -I${dlmincdir} -I${logtincdir} @@ -105,6 +108,9 @@ ${TARGET4}: ${OBJS4} ${LDDEPS} $(CC) -o $@ $^ $(CPG_LDFLAGS) $(CMAN_LDFLAGS) $(CCS_LDFLAGS) \ $(FENCED_LDFLAGS) $(EXTRA_LDFLAGS) $(LDFLAGS) +${TARGET5}: ${OBJS5} ${LDDEPS} + $(CC) -o $@ $^ $(CPG_LDFLAGS) + check: rg_test cd tests && ./runtests.sh diff --git a/rgmanager/src/daemons/cpglockdump.c b/rgmanager/src/daemons/cpglockdump.c new file mode 100644 index 0000000..2750e4a --- /dev/null +++ b/rgmanager/src/daemons/cpglockdump.c @@ -0,0 +1,8 @@ +#include <stdio.h> +#include <cpglock.h> + +int +main(int argc, char **argv) +{ + return cpg_lock_dump(stdout); +}

1 0

cluster: STABLE32 - rgmanager: Merge upstream cpglockd
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=00… Commit: 009c89cb56610a5ba2da310a36d7bc2822e4d919 Parent: d145234077b65725c0e2166e66d55b41f54edca1 Author: Lon Hohberger <lhh(a)redhat.com> AuthorDate: Wed Mar 21 16:35:10 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 rgmanager: Merge upstream cpglockd Signed-off-by: Lon Hohberger <lhh(a)redhat.com> --- rgmanager/src/daemons/cpglockd.c | 9 +++------ 1 files changed, 3 insertions(+), 6 deletions(-) diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c index 55d832a..3ed132b 100644 --- a/rgmanager/src/daemons/cpglockd.c +++ b/rgmanager/src/daemons/cpglockd.c @@ -176,7 +176,7 @@ old_msg(struct cpg_lock_msg *m) n = do_alloc(sizeof(*n)); memcpy(&n->m, m, sizeof(n->m)); list_append(&messages, n); - if (message_count < 10) { + if (message_count < 20) { ++message_count; } else { n = messages; @@ -781,8 +781,8 @@ process_unlock(struct cpg_lock_msg *m, uint32_t nodeid) l->l.owner_pid == m->owner_pid) { printf("UNLOCK %s: %d:%d:%d\n", m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid); l->l.state = LOCK_FREE; - if (grant_next(m) != 0) - l->l.state = LOCK_PENDING; + //if (grant_next(m) != 0) + //l->l.state = LOCK_PENDING; } } @@ -1055,12 +1055,9 @@ main(int argc, char **argv) int afd = -1; int n,x; - struct cpg_lock l; struct cpg_lock_msg m; struct client_node *client; - l.state = 0; - signal(SIGPIPE, SIG_IGN); fd = sock_listen(CPG_LOCKD_SOCK);

1 0

cluster: STABLE32 - rgmanager: Add simple locking over CPG
by Ryan McCabe 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=d1… Commit: d145234077b65725c0e2166e66d55b41f54edca1 Parent: 413f459962c9bb5a9f90fd2e09b56253d5120cc6 Author: Lon Hohberger <lhh(a)redhat.com> AuthorDate: Wed Mar 21 16:29:12 2012 -0400 Committer: Fabio M. Di Nitto <fdinitto(a)redhat.com> CommitterDate: Tue Apr 24 17:04:09 2012 +0200 rgmanager: Add simple locking over CPG Signed-off-by: Lon Hohberger <lhh(a)redhat.com> --- rgmanager/include/cpglock-internal.h | 38 ++ rgmanager/include/cpglock.h | 40 ++ rgmanager/include/list.h | 1 + rgmanager/include/lock.h | 15 +- rgmanager/include/sock.h | 18 + rgmanager/src/clulib/Makefile | 2 +- rgmanager/src/clulib/cpg_lock.c | 104 +++ rgmanager/src/clulib/dlm_lock.c | 301 +++++++++ rgmanager/src/clulib/libcpglock.c | 286 +++++++++ rgmanager/src/clulib/lock.c | 294 +--------- rgmanager/src/clulib/sock.c | 354 +++++++++++ rgmanager/src/daemons/Makefile | 12 +- rgmanager/src/daemons/cpglockd.c | 1147 ++++++++++++++++++++++++++++++++++ rgmanager/src/daemons/main.c | 24 +- 14 files changed, 2326 insertions(+), 310 deletions(-) diff --git a/rgmanager/include/cpglock-internal.h b/rgmanager/include/cpglock-internal.h new file mode 100644 index 0000000..1c322ae --- /dev/null +++ b/rgmanager/include/cpglock-internal.h @@ -0,0 +1,38 @@ +#ifndef _CPGLOCK_INT_H +#define _CPGLOCK_INT_H + +#ifndef CPG_LOCKD_SOCK +#define CPG_LOCKD_SOCK "/var/run/cpglockd.sk" +#endif + +#include <stdint.h> + +typedef enum { + MSG_LOCK = 1, + MSG_NAK = 2, + MSG_GRANT = 3, + MSG_UNLOCK = 4, + MSG_PURGE = 5, + MSG_CONFCHG= 6, + MSG_JOIN = 7, + MSG_DUMP = 998, + MSG_HALT = 999 +} cpg_lock_req_t; + +/* Mixed architecture not supported yet */ +struct __attribute__((packed)) cpg_lock_msg { + char resource[96]; + int32_t request; + uint32_t owner_nodeid; + uint32_t owner_pid; + uint32_t flags; + uint32_t lockid; + uint32_t owner_tid; + char pad[8]; +}; /* 128 */ + +pid_t _gettid (void); + +#define CPG_LOCKD_NAME "cpglockd" + +#endif diff --git a/rgmanager/include/cpglock.h b/rgmanager/include/cpglock.h new file mode 100644 index 0000000..e451d2f --- /dev/null +++ b/rgmanager/include/cpglock.h @@ -0,0 +1,40 @@ +#ifndef _CPGLOCK_H +#define _CPGLOCK_H + +typedef enum { + LOCK_FREE = 0, + LOCK_PENDING = 1, + LOCK_HELD = 2 +} lock_state_t; + +typedef enum { + FL_TRY = 0x1 +} lock_flag_t; + +struct cpg_lock { + char resource[96]; + int local_fd; + int local_id; + lock_state_t state; + int owner_nodeid; + int owner_pid; + int owner_tid; +}; + +typedef void * cpg_lock_handle_t; + +int cpg_lock_init(cpg_lock_handle_t *h); + +/* 0 if successful, -1 if error */ +int cpg_lock(cpg_lock_handle_t h, const char *resource, + lock_flag_t flags, struct cpg_lock *lock); + +int cpg_unlock(cpg_lock_handle_t h, + struct cpg_lock *lock); + +/* Warning: drops all locks with this client */ +int cpg_lock_fin(cpg_lock_handle_t h); + +int cpg_lock_dump(FILE *fp); + +#endif diff --git a/rgmanager/include/list.h b/rgmanager/include/list.h index 0dac6e8..0d1c1d5 100644 --- a/rgmanager/include/list.h +++ b/rgmanager/include/list.h @@ -36,6 +36,7 @@ do { \ *list = newnode; \ } while (0) +#define list_append list_insert #define list_remove(list, oldnode) \ do { \ diff --git a/rgmanager/include/lock.h b/rgmanager/include/lock.h index 7941304..82a1faf 100644 --- a/rgmanager/include/lock.h +++ b/rgmanager/include/lock.h @@ -6,17 +6,12 @@ #include <stdlib.h> #include <libdlm.h> -int clu_ls_lock(dlm_lshandle_t ls, int mode, struct dlm_lksb *lksb, - int options, const char *resource); -dlm_lshandle_t clu_open_lockspace(const char *lsname); -int clu_ls_unlock(dlm_lshandle_t ls, struct dlm_lksb *lksb); -int clu_close_lockspace(dlm_lshandle_t ls, const char *lsname); - /* Default lockspace wrappers */ int clu_lock_init(const char *default_lsname); -int clu_lock(int mode, struct dlm_lksb *lksb, - int options, const char *resource); -int clu_unlock(struct dlm_lksb *lksb); -void clu_lock_finished(const char *default_lsname); +int cpg_lock_initialize(void); + +extern int (*clu_lock)(int, struct dlm_lksb *, int, const char *); +extern int (*clu_unlock)(struct dlm_lksb *lksb); +extern void (*clu_lock_finished)(const char *); #endif diff --git a/rgmanager/include/sock.h b/rgmanager/include/sock.h new file mode 100644 index 0000000..6cd2331 --- /dev/null +++ b/rgmanager/include/sock.h @@ -0,0 +1,18 @@ +#ifndef _SOCK_H +#define _SOCK_H + +ssize_t read_retry(int sockfd, void *buf, size_t count, + struct timeval * timeout); +int select_retry(int fdmax, fd_set * rfds, fd_set * wfds, fd_set * xfds, + struct timeval *timeout); +ssize_t write_retry(int fd, void *buf, size_t count, + struct timeval *timeout); + +int sock_listen(const char *sockpath); +int sock_connect(const char *sockpath, int tout); +int sock_accept(int fd); +void hexdump(const void *buf, size_t len); + +void *do_alloc(size_t); + +#endif diff --git a/rgmanager/src/clulib/Makefile b/rgmanager/src/clulib/Makefile index 34f58fb..adb5578 100644 --- a/rgmanager/src/clulib/Makefile +++ b/rgmanager/src/clulib/Makefile @@ -12,7 +12,7 @@ include $(OBJDIR)/make/uninstall.mk OBJS1= logging.o daemon_init.o signals.o msgsimple.o \ gettid.o rg_strings.o message.o members.o fdops.o \ lock.o cman.o vft.o msg_cluster.o msg_socket.o \ - wrap_lock.o sets.o + wrap_lock.o sets.o sock.o libcpglock.o dlm_lock.o cpg_lock.o OBJS2= msgtest.o diff --git a/rgmanager/src/clulib/cpg_lock.c b/rgmanager/src/clulib/cpg_lock.c new file mode 100644 index 0000000..a747419 --- /dev/null +++ b/rgmanager/src/clulib/cpg_lock.c @@ -0,0 +1,104 @@ +/** @file + * Locking. + */ +#include <errno.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <stdio.h> +#include <assert.h> +#include <sys/ioctl.h> +#include <lock.h> +#include <sys/types.h> +#include <sys/select.h> +#include <pthread.h> + +#include <cpglock.h> + +/* Default lockspace stuff */ +static cpg_lock_handle_t _cpgh = NULL; +static pthread_mutex_t _default_lock = PTHREAD_MUTEX_INITIALIZER; + +static void +dlm2cpg(struct dlm_lksb *dlm, struct cpg_lock *cpg) +{ + memset(cpg, 0, sizeof(*cpg)); + cpg->local_id = dlm->sb_lkid; + cpg->state = dlm->sb_status; +} + +static void +cpg2dlm(struct cpg_lock *cpg, struct dlm_lksb *dlm) +{ + memset(dlm, 0, sizeof(*dlm)); + dlm->sb_status = cpg->state; + dlm->sb_lkid = cpg->local_id; +} + + +static int +_cpg_lock(int mode, + struct dlm_lksb *lksb, + int options, + const char *resource) +{ + int ret = 0; + + struct cpg_lock l; + + if (options == LKF_NOQUEUE) + ret = cpg_lock(_cpgh, resource, 1, &l); + else + ret = cpg_lock(_cpgh, resource, 0, &l); + + if (ret == 0) { + cpg2dlm(&l, lksb); + } + + return ret; +} + + +static int +_cpg_unlock(struct dlm_lksb *lksb) +{ + struct cpg_lock l; + + dlm2cpg(lksb, &l); + return cpg_unlock(_cpgh, &l); +} + + +static void +_cpg_lock_finished(const char *name) +{ + pthread_mutex_lock(&_default_lock); + cpg_lock_fin(_cpgh); + pthread_mutex_unlock(&_default_lock); +} + + +int +cpg_lock_initialize(void) +{ + int ret, err; + + pthread_mutex_lock(&_default_lock); + if (_cpgh) { + pthread_mutex_unlock(&_default_lock); + return 0; + } + + cpg_lock_init(&_cpgh); + ret = (_cpgh == NULL); + err = errno; + pthread_mutex_unlock(&_default_lock); + + /* Set up function pointers */ + clu_lock = _cpg_lock; + clu_unlock = _cpg_unlock; + clu_lock_finished = _cpg_lock_finished; + + errno = err; + return ret; +} diff --git a/rgmanager/src/clulib/dlm_lock.c b/rgmanager/src/clulib/dlm_lock.c new file mode 100644 index 0000000..d3f4250 --- /dev/null +++ b/rgmanager/src/clulib/dlm_lock.c @@ -0,0 +1,301 @@ +/** @file + * Locking. + */ +#include <errno.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <stdio.h> +#include <assert.h> +#include <sys/ioctl.h> +#include <lock.h> +#include <sys/types.h> +#include <sys/select.h> +#include <pthread.h> + +/* Default lockspace stuff */ +static dlm_lshandle_t _default_ls = NULL; +static pthread_mutex_t _default_lock = PTHREAD_MUTEX_INITIALIZER; + + +static void +ast_function(void * __attribute__ ((unused)) arg) +{ +} + + +static int +wait_for_dlm_event(dlm_lshandle_t ls) +{ + fd_set rfds; + int fd = dlm_ls_get_fd(ls); + + FD_ZERO(&rfds); + FD_SET(fd, &rfds); + + if (select(fd + 1, &rfds, NULL, NULL, NULL) == 1) + return dlm_dispatch(fd); + + return -1; +} + + +static int +clu_ls_lock(dlm_lshandle_t ls, + int mode, + struct dlm_lksb *lksb, + int options, + const char *resource) +{ + int ret; + + if (!ls || !lksb || !resource || !strlen(resource)) { + printf("%p %p %p %d\n", ls, lksb, resource, + (int)strlen(resource)); + printf("INVAL...\n"); + errno = EINVAL; + return -1; + } + + ret = dlm_ls_lock(ls, mode, lksb, options, resource, + strlen(resource), 0, ast_function, lksb, + NULL, NULL); + + if (ret < 0) { + if (errno == ENOENT) + assert(0); + + return -1; + } + + if ((ret = (wait_for_dlm_event(ls) < 0))) { + fprintf(stderr, "wait_for_dlm_event: %d / %d\n", + ret, errno); + return -1; + } + + if (lksb->sb_status == 0) + return 0; + + errno = lksb->sb_status; + return -1; +} + + +static dlm_lshandle_t +clu_open_lockspace(const char *lsname) +{ + dlm_lshandle_t ls = NULL; + + //printf("opening lockspace %s\n", lsname); + + while (!ls) { + ls = dlm_open_lockspace(lsname); + if (ls) + break; + + /* + printf("Failed to open: %s; trying create.\n", + strerror(errno)); + */ + + ls = dlm_create_lockspace(lsname, 0644); + if (ls) + break; + + /* Work around race: Someone was closing lockspace as + we were trying to open it. Retry. */ + if (errno == ENOENT) + continue; + + fprintf(stderr, "failed acquiring lockspace: %s\n", + strerror(errno)); + + return NULL; + } + + return ls; +} + + +static int +clu_ls_unlock(dlm_lshandle_t ls, struct dlm_lksb *lksb) +{ + int ret; + + if (!ls || !lksb) { + errno = EINVAL; + return -1; + } + + ret = dlm_ls_unlock(ls, lksb->sb_lkid, 0, lksb, NULL); + + if (ret != 0) + return ret; + + /* lksb->sb_status should be EINPROG at this point */ + + if (wait_for_dlm_event(ls) < 0) { + errno = lksb->sb_status; + return -1; + } + + return 0; +} + + +static int +clu_close_lockspace(dlm_lshandle_t ls, const char *name) +{ + return dlm_release_lockspace(name, ls, 1); +} + + +static int +_clu_lock(int mode, + struct dlm_lksb *lksb, + int options, + const char *resource) +{ + int ret = 0, block = 0, conv = 0, err; + + block = !(options & LKF_NOQUEUE); + + errno = EINVAL; + if (!lksb) + return -1; + + memset(lksb, 0, sizeof(struct dlm_lksb)); + + /* + Try to use a conversion lock mechanism when possible + If the caller calls explicitly with a NULL lock, then + assume the caller knows what it is doing. + + Only take the NULL lock if: + (a) the user isn't specifying CONVERT; if they are, they + know what they are doing. + + ...and one of... + + (b) This is a blocking call, or + (c) The user requested a NULL lock explicitly. In this case, + short-out early; there's no reason to convert a NULL lock + to a NULL lock. + */ + if (!(options & LKF_CONVERT) && + (block || (mode == LKM_NLMODE))) { + /* Acquire NULL lock */ + pthread_mutex_lock(&_default_lock); + ret = clu_ls_lock(_default_ls, LKM_NLMODE, lksb, + (options & ~LKF_NOQUEUE), + resource); + err = errno; + pthread_mutex_unlock(&_default_lock); + if (ret == 0) { + if (mode == LKM_NLMODE) { + /* User only wanted a NULL lock... */ + return 0; + } + /* + Ok, NULL lock was taken, rest of blocking + call should be done using lock conversions. + */ + options |= LKF_CONVERT; + conv = 1; + } else { + switch(err) { + case EINVAL: + /* Oops, null locks don't work on this + plugin; use normal spam mode */ + break; + default: + errno = err; + return -1; + } + } + } + + while (1) { + pthread_mutex_lock(&_default_lock); + ret = clu_ls_lock(_default_ls, mode, lksb, + (options | LKF_NOQUEUE), + resource); + err = errno; + pthread_mutex_unlock(&_default_lock); + + if ((ret != 0) && (err == EAGAIN) && block) { + usleep(random()&32767); + continue; + } + + break; + } + + if (ret != 0 && conv) { + /* If we get some other error, release the NL lock we + took so we don't leak locks*/ + pthread_mutex_lock(&_default_lock); + clu_ls_unlock(_default_ls, lksb); + pthread_mutex_unlock(&_default_lock); + errno = err; + } + + return ret; +} + + +static int +_clu_unlock(struct dlm_lksb *lksb) +{ + int ret, err; + pthread_mutex_lock(&_default_lock); + ret = clu_ls_unlock(_default_ls, lksb); + err = errno; + pthread_mutex_unlock(&_default_lock); + + usleep(random()&32767); + errno = err; + return ret; +} + + +static void +_clu_lock_finished(const char *name) +{ + pthread_mutex_lock(&_default_lock); + if (_default_ls) + clu_close_lockspace(_default_ls, name); + pthread_mutex_unlock(&_default_lock); +} + + +int +clu_lock_init(const char *dflt_lsname) +{ + int ret, err; + + pthread_mutex_lock(&_default_lock); + if (_default_ls) { + pthread_mutex_unlock(&_default_lock); + return 0; + } + + if (!dflt_lsname || !strlen(dflt_lsname)) { + pthread_mutex_unlock(&_default_lock); + errno = EINVAL; + return -1; + } + + _default_ls = clu_open_lockspace(dflt_lsname); + ret = (_default_ls == NULL); + err = errno; + pthread_mutex_unlock(&_default_lock); + + clu_lock = _clu_lock; + clu_unlock = _clu_unlock; + clu_lock_finished = _clu_lock_finished; + + errno = err; + return ret; +} diff --git a/rgmanager/src/clulib/libcpglock.c b/rgmanager/src/clulib/libcpglock.c new file mode 100644 index 0000000..8e5c7ff --- /dev/null +++ b/rgmanager/src/clulib/libcpglock.c @@ -0,0 +1,286 @@ +#include <stdlib.h> +#include <sys/socket.h> +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <pthread.h> +#include <unistd.h> + +#include "cpglock.h" +#include "cpglock-internal.h" +#include "list.h" +#include "sock.h" + + +struct pending_node { + list_head(); + struct cpg_lock_msg m; +}; + + +struct cpg_lock_handle { + pthread_mutex_t mutex; + struct pending_node *pending; + int fd; + int pid; + int seq; +}; + + +static int +check_pending(struct pending_node **l, + struct cpg_lock_msg *exp, + struct cpg_lock_msg *ret) +{ + struct pending_node *p; + int x; + + list_for(l, p, x) { + if (!strcmp(exp->resource, p->m.resource) && + exp->owner_tid == p->m.owner_tid) { + list_remove(l, p); + memcpy(ret, &p->m, sizeof(*ret)); + free(p); + return 0; + } + } + + return 1; +} + + +static void +add_pending(struct pending_node **l, + struct cpg_lock_msg *m) +{ + struct pending_node *p = do_alloc(sizeof(*p)); + + memcpy(&p->m, m, sizeof(p->m)); + list_insert(l, p); +} + + +/* Not thread safe */ +int +cpg_lock_init(void **handle) +{ + struct cpg_lock_handle *h; + int esv; + + h = do_alloc(sizeof (*h)); + if (!h) + return -1; + + h->fd = sock_connect(CPG_LOCKD_SOCK, 3); + if (h->fd < 0) { + esv = errno; + free(h); + errno = esv; + return -1; + } + + h->pid = getpid(); + pthread_mutex_init(&h->mutex, NULL); + + *handle = (void *)h; + return 0; +} + + +int +cpg_lock(void *handle, const char *resource, lock_flag_t flags, struct cpg_lock *lock) +{ + struct cpg_lock_handle *h = handle; + struct cpg_lock_msg l, r; + struct timeval tv; + int ret = -1; + + if (!h) { + errno = EINVAL; + goto out; + } + + pthread_mutex_lock(&h->mutex); + + if (h->pid != (int)getpid()) { + errno = EBADF; + goto out; + } + + if (strlen(resource) > sizeof(l.resource)-1) { + errno = ENAMETOOLONG; + goto out; + } + + memset(&l, 0, sizeof(l)); + memset(&r, 0, sizeof(r)); + strncpy(l.resource, resource, sizeof(l.resource)); + strncpy(lock->resource, resource, sizeof(lock->resource)); + l.owner_pid = h->pid; + ++h->seq; + l.owner_tid = h->seq; + l.request = MSG_LOCK; + l.flags = (uint32_t)flags; + + if (write_retry(h->fd, &l, sizeof(l), 0) < 0) + goto out; + + /* Thread concurrency: in case multiple threads wake up + from select, peek at the message to see if it's ours */ + do { + if (check_pending(&h->pending, &l, &r) == 0) + break; + + tv.tv_sec = 0; + tv.tv_usec = random() & 16383; + + if (read_retry(h->fd, &r, sizeof(r), &tv) < 0) { + if (errno == ETIMEDOUT) { + pthread_mutex_unlock(&h->mutex); + usleep(random() & 16383); + pthread_mutex_lock(&h->mutex); + continue; + } + goto out; + } + + if (strcmp(r.resource, l.resource)) { + printf("NOTE: msg for wrong lock want: %s got: %s\n", l.resource, r.resource); + add_pending(&h->pending, &r); + pthread_mutex_unlock(&h->mutex); + usleep(random() & 16383); + pthread_mutex_lock(&h->mutex); + continue; + } + if (r.owner_tid != l.owner_tid) { + printf("NOTE: msg for wrong seq want: %d got: %d\n", l.owner_tid, r.owner_tid ); + add_pending(&h->pending, &r); + pthread_mutex_unlock(&h->mutex); + usleep(random() & 16383); + pthread_mutex_lock(&h->mutex); + continue; + } + break; + } while (1); + /* locked */ + + if (r.owner_nodeid == 0) + goto out; + + if (r.request == MSG_NAK) { + errno = EAGAIN; + return -1; + } + + if (r.request != MSG_GRANT) { + //ret = -1; + goto out; + } + + lock->state = LOCK_HELD; + lock->owner_nodeid = r.owner_nodeid; + lock->owner_pid = h->pid; /* XXX */ + lock->local_id = r.lockid; + + ret = 0; + +out: + pthread_mutex_unlock(&h->mutex); + return ret; +} + + +int +cpg_unlock(void *handle, struct cpg_lock *lock) +{ + struct cpg_lock_handle *h = handle; + struct cpg_lock_msg l; + int ret = -1; + + if (!h) { + errno = EINVAL; + goto out; + } + + /* Only block on lock requests, not unlock */ + if (h->pid != (int)getpid()) { + errno = EBADF; + goto out; + } + + if (lock->state != LOCK_HELD) { + errno = EINVAL; + goto out; + } + + if (!lock->local_id) { + errno = EINVAL; + goto out; + } + + strncpy(l.resource, lock->resource, sizeof(l.resource)); + l.request = MSG_UNLOCK; + l.owner_nodeid = lock->owner_nodeid; + l.owner_pid = lock->owner_pid; + l.lockid = lock->local_id; + //l.owner_tid = _gettid(); + l.owner_tid = 0; + lock->state = LOCK_FREE; + + ret = write_retry(h->fd, &l, sizeof(l), 0); +out: + return ret; +} + + +int +cpg_lock_dump(FILE *fp) +{ + struct cpg_lock_msg l; + int fd; + char c; + + fd = sock_connect(CPG_LOCKD_SOCK, 3); + if (fd < 0) + return -1; + + memset(&l, 0, sizeof(l)); + l.request = MSG_DUMP; + + if (write_retry(fd, &l, sizeof(l), 0) < 0) + return -1; + + while (read_retry(fd, &c, 1, 0) == 1) + fprintf(fp, "%c", c); + + close(fd); + return 0; +} + + + +/* Not thread safe */ +int +cpg_lock_fin(void *handle) +{ + struct cpg_lock_handle *h = handle; + + if (!h) { + errno = EINVAL; + return -1; + } + + pthread_mutex_lock(&h->mutex); + + if (h->pid != (int)getpid()) { + errno = EBADF; + return -1; + } + + close(h->fd); + + pthread_mutex_destroy(&h->mutex); + free(h); + return 0; +} + diff --git a/rgmanager/src/clulib/lock.c b/rgmanager/src/clulib/lock.c index 078d525..74cd8b1 100644 --- a/rgmanager/src/clulib/lock.c +++ b/rgmanager/src/clulib/lock.c @@ -1,294 +1,6 @@ -/** @file - * Locking. - */ -#include <errno.h> -#include <stdlib.h> -#include <unistd.h> -#include <string.h> -#include <stdio.h> -#include <assert.h> -#include <sys/ioctl.h> #include <lock.h> -#include <sys/types.h> -#include <sys/select.h> -#include <pthread.h> -/* Default lockspace stuff */ -static dlm_lshandle_t _default_ls = NULL; -static pthread_mutex_t _default_lock = PTHREAD_MUTEX_INITIALIZER; +int (*clu_lock)(int, struct dlm_lksb *, int, const char *) = NULL; +int (*clu_unlock)(struct dlm_lksb *lksb) = NULL; +void (*clu_lock_finished)(const char *) = NULL; - -static void -ast_function(void * __attribute__ ((unused)) arg) -{ -} - - -static int -wait_for_dlm_event(dlm_lshandle_t ls) -{ - fd_set rfds; - int fd = dlm_ls_get_fd(ls); - - FD_ZERO(&rfds); - FD_SET(fd, &rfds); - - if (select(fd + 1, &rfds, NULL, NULL, NULL) == 1) - return dlm_dispatch(fd); - - return -1; -} - - -int -clu_ls_lock(dlm_lshandle_t ls, - int mode, - struct dlm_lksb *lksb, - int options, - const char *resource) -{ - int ret; - - if (!ls || !lksb || !resource || !strlen(resource)) { - errno = EINVAL; - return -1; - } - - ret = dlm_ls_lock(ls, mode, lksb, options, resource, - strlen(resource), 0, ast_function, lksb, - NULL, NULL); - - if (ret < 0) { - if (errno == ENOENT) - assert(0); - - return -1; - } - - if ((ret = (wait_for_dlm_event(ls) < 0))) { - fprintf(stderr, "wait_for_dlm_event: %d / %d\n", - ret, errno); - return -1; - } - - if (lksb->sb_status == 0) - return 0; - - errno = lksb->sb_status; - return -1; -} - - - -dlm_lshandle_t -clu_open_lockspace(const char *lsname) -{ - dlm_lshandle_t ls = NULL; - - //printf("opening lockspace %s\n", lsname); - - while (!ls) { - ls = dlm_open_lockspace(lsname); - if (ls) - break; - - /* - printf("Failed to open: %s; trying create.\n", - strerror(errno)); - */ - - ls = dlm_create_lockspace(lsname, 0644); - if (ls) - break; - - /* Work around race: Someone was closing lockspace as - we were trying to open it. Retry. */ - if (errno == ENOENT) - continue; - - fprintf(stderr, "failed acquiring lockspace: %s\n", - strerror(errno)); - - return NULL; - } - - return ls; -} - - -int -clu_ls_unlock(dlm_lshandle_t ls, struct dlm_lksb *lksb) -{ - int ret; - - if (!ls || !lksb) { - errno = EINVAL; - return -1; - } - - ret = dlm_ls_unlock(ls, lksb->sb_lkid, 0, lksb, NULL); - - if (ret != 0) - return ret; - - /* lksb->sb_status should be EINPROG at this point */ - - if (wait_for_dlm_event(ls) < 0) { - errno = lksb->sb_status; - return -1; - } - - return 0; -} - - -int -clu_close_lockspace(dlm_lshandle_t ls, const char *name) -{ - return dlm_release_lockspace(name, ls, 1); -} - - -int -clu_lock(int mode, - struct dlm_lksb *lksb, - int options, - const char *resource) -{ - int ret = 0, block = 0, conv = 0, err; - - block = !(options & LKF_NOQUEUE); - - errno = EINVAL; - if (!lksb) - return -1; - - memset(lksb, 0, sizeof(struct dlm_lksb)); - - /* - Try to use a conversion lock mechanism when possible - If the caller calls explicitly with a NULL lock, then - assume the caller knows what it is doing. - - Only take the NULL lock if: - (a) the user isn't specifying CONVERT; if they are, they - know what they are doing. - - ...and one of... - - (b) This is a blocking call, or - (c) The user requested a NULL lock explicitly. In this case, - short-out early; there's no reason to convert a NULL lock - to a NULL lock. - */ - if (!(options & LKF_CONVERT) && - (block || (mode == LKM_NLMODE))) { - /* Acquire NULL lock */ - pthread_mutex_lock(&_default_lock); - ret = clu_ls_lock(_default_ls, LKM_NLMODE, lksb, - (options & ~LKF_NOQUEUE), - resource); - err = errno; - pthread_mutex_unlock(&_default_lock); - if (ret == 0) { - if (mode == LKM_NLMODE) { - /* User only wanted a NULL lock... */ - return 0; - } - /* - Ok, NULL lock was taken, rest of blocking - call should be done using lock conversions. - */ - options |= LKF_CONVERT; - conv = 1; - } else { - switch(err) { - case EINVAL: - /* Oops, null locks don't work on this - plugin; use normal spam mode */ - break; - default: - errno = err; - return -1; - } - } - } - - while (1) { - pthread_mutex_lock(&_default_lock); - ret = clu_ls_lock(_default_ls, mode, lksb, - (options | LKF_NOQUEUE), - resource); - err = errno; - pthread_mutex_unlock(&_default_lock); - - if ((ret != 0) && (err == EAGAIN) && block) { - usleep(random()&32767); - continue; - } - - break; - } - - if (ret != 0 && conv) { - /* If we get some other error, release the NL lock we - took so we don't leak locks*/ - pthread_mutex_lock(&_default_lock); - clu_ls_unlock(_default_ls, lksb); - pthread_mutex_unlock(&_default_lock); - errno = err; - } - - return ret; -} - - -int -clu_unlock(struct dlm_lksb *lksb) -{ - int ret, err; - pthread_mutex_lock(&_default_lock); - ret = clu_ls_unlock(_default_ls, lksb); - err = errno; - pthread_mutex_unlock(&_default_lock); - - usleep(random()&32767); - errno = err; - return ret; -} - - -int -clu_lock_init(const char *dflt_lsname) -{ - int ret, err; - - pthread_mutex_lock(&_default_lock); - if (_default_ls) { - pthread_mutex_unlock(&_default_lock); - return 0; - } - - if (!dflt_lsname || !strlen(dflt_lsname)) { - pthread_mutex_unlock(&_default_lock); - errno = EINVAL; - return -1; - } - - _default_ls = clu_open_lockspace(dflt_lsname); - ret = (_default_ls == NULL); - err = errno; - pthread_mutex_unlock(&_default_lock); - - errno = err; - return ret; -} - -void -clu_lock_finished(const char *name) -{ - pthread_mutex_lock(&_default_lock); - if (_default_ls) - clu_close_lockspace(_default_ls, name); - pthread_mutex_unlock(&_default_lock); -} diff --git a/rgmanager/src/clulib/sock.c b/rgmanager/src/clulib/sock.c new file mode 100644 index 0000000..3965a38 --- /dev/null +++ b/rgmanager/src/clulib/sock.c @@ -0,0 +1,354 @@ +/* + Copyright Red Hat, Inc. 2002-2003, 2012 + Copyright Mission Critical Linux, 2000 + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; either version 2, or (at your option) any + later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 675 Mass Ave, Cambridge, + MA 02139, USA. +*/ +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/errno.h> +#include <time.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <fcntl.h> + +#include "sock.h" + +void * +do_alloc(size_t n) +{ + void *p; + + do { + p = malloc(n); + if (!p) { + usleep(10000); + } + } while(!p); + + memset(p, 0, n); + return p; +} + + +/** + * This is a wrapper around select which will retry in the case we receive + * EINTR. This is necessary for read_retry, since it wouldn't make sense + * to have read_retry terminate if and only if two EINTRs were received + * in a row - one during the read() call, one during the select call... + * + * See select(2) for description of parameters. + */ +int +select_retry(int fdmax, fd_set * rfds, fd_set * wfds, fd_set * xfds, + struct timeval *timeout) +{ + int rv; + + while (1) { + rv = select(fdmax, rfds, wfds, xfds, timeout); + if ((rv == -1) && (errno == EINTR)) + /* return on EBADF/EINVAL/ENOMEM; continue on EINTR */ + continue; + return rv; + } +} + +/** + * Retries a write in the event of a non-blocked interrupt signal. + * + * @param fd File descriptor to which we are writing. + * @param buf Data buffer to send. + * @param count Number of bytes in buf to send. + * @param timeout (struct timeval) telling us how long we should retry. + * @return The number of bytes written to the file descriptor, + * or -1 on error (with errno set appropriately). + */ +ssize_t +write_retry(int fd, void *buf, size_t count, struct timeval * timeout) +{ + int n, rv = 0; + ssize_t total = 0, remain = count; + fd_set wfds, xfds; + + while (total < count) { + + /* Create the write FD set of 1... */ + FD_ZERO(&wfds); + FD_SET(fd, &wfds); + FD_ZERO(&xfds); + FD_SET(fd, &xfds); + + /* wait for the fd to be available for writing */ + rv = select_retry(fd + 1, NULL, &wfds, &xfds, timeout); + if (rv == -1) + return -1; + else if (rv == 0) { + errno = ETIMEDOUT; + return -1; + } + + if (FD_ISSET(fd, &xfds)) { + errno = EPIPE; + return -1; + } + + /* + * Attempt to write to fd + */ + n = write(fd, (char *)buf + (off_t) total, remain); + + /* + * When we know our fd was select()ed and we receive 0 bytes + * when we write, the fd was closed. + */ + if ((n == 0) && (rv == 1)) { + errno = EPIPE; + return -1; + } + + if (n == -1) { + if ((errno == EAGAIN) || (errno == EINTR)) { + /* + * Not ready? + */ + continue; + } + + /* Other errors: EIO, EINVAL, etc */ + return -1; + } + + total += n; + remain -= n; + } + + return total; +} + +/** + * Retry reads until we (a) time out or (b) get our data. Of course, if + * timeout is NULL, it'll wait forever. + * + * @param sockfd File descriptor we want to read from. + * @param buf Preallocated buffer into which we will read data. + * @param count Number of bytes to read. + * @param timeout (struct timeval) describing how long we should retry. + * @return The number of bytes read on success, or -1 on failure. + Note that we will always return (count) or (-1). + */ +ssize_t +read_retry(int sockfd, void *buf, size_t count, struct timeval * timeout) +{ + int n, rv = 0; + ssize_t total = 0, remain = count; + fd_set rfds, xfds; + + memset(buf, 0, count); + + while (total < count) { + FD_ZERO(&rfds); + FD_SET(sockfd, &rfds); + FD_ZERO(&xfds); + FD_SET(sockfd, &xfds); + + /* + * Select on the socket, in case it closes while we're not + * looking... + */ + rv = select_retry(sockfd + 1, &rfds, NULL, &xfds, timeout); + if (rv == -1) + return -1; + else if (rv == 0) { + errno = ETIMEDOUT; + return -1; + } + + if (FD_ISSET(sockfd, &xfds)) { + errno = EPIPE; + return -1; + } + + /* + * Attempt to read off the socket + */ + n = read(sockfd, (char *)buf + (off_t) total, remain); + + /* + * When we know our socket was select()ed and we receive 0 bytes + * when we read, the socket was closed. + */ + if ((n == 0) && (rv == 1)) { + errno = EPIPE; + return -1; + } + + if (n == -1) { + if ((errno == EAGAIN) || (errno == EINTR)) { + /* + * Not ready? Wait for data to become available + */ + continue; + } + + /* Other errors: EPIPE, EINVAL, etc */ + return -1; + } + + total += n; + remain -= n; + + //printf("read-retry %d/%d remain %d \n", total, count, remain); + } + + return total; +} + + +int +sock_listen(const char *sockpath) +{ + int sock = -1; + struct sockaddr_un su; + mode_t om; + + sock = socket(PF_LOCAL, SOCK_STREAM, 0); + if (sock < 0) + goto fail; + + su.sun_family = PF_LOCAL; + snprintf(su.sun_path, (sizeof(su.sun_path)), "%s", sockpath); + + unlink(su.sun_path); + om = umask(077); + + if (bind(sock, (struct sockaddr *)&su, sizeof(su)) < 0) { + umask(om); + goto fail; + } + umask(om); + + if (listen(sock, SOMAXCONN) < 0) + goto fail; + return sock; +fail: + if (sock >= 0) + close(sock); + return -1; +} + + +int +sock_connect(const char *sockpath, int tout) +{ + struct timeval timeout = {tout, 0}; + int sock, flags, error, ret; + socklen_t len; + struct sockaddr_un sun; + fd_set rfds, wfds; + + sock = socket(PF_LOCAL, SOCK_STREAM, 0); + if (sock < 0) + return -1; + + sun.sun_family = PF_LOCAL; + snprintf(sun.sun_path, (sizeof(sun.sun_path)), "%s", sockpath); + + flags = fcntl(sock, F_GETFL, 0); + fcntl(sock, F_SETFL, flags | O_NONBLOCK); + + ret = connect(sock, (struct sockaddr *) &sun, sizeof(sun)); + + if (ret < 0 && (errno != EINPROGRESS)) { + close(sock); + return -1; + } + + if (ret == 0) + goto done; + + FD_ZERO(&rfds); + FD_SET(sock, &rfds); + wfds = rfds; + + ret = select(sock + 1, &rfds, &wfds, NULL, &timeout); + if (ret == 0) { + close(sock); + errno = ETIMEDOUT; + return -1; + } + + if (FD_ISSET(sock, &rfds) || FD_ISSET(sock, &wfds)) { + len = sizeof (error); + if (getsockopt(sock, SOL_SOCKET, SO_ERROR, &error, &len) < 0) { + close(sock); + return -1; + } + } else { + close(sock); + return -1; + } + +done: + + return sock; +} + + +int +sock_accept(int sockfd) +{ + int acceptfd; + + if (sockfd < 0) { + errno = EBADF; + return -1; + } + + while ((acceptfd = + accept(sockfd, (struct sockaddr *) NULL, NULL)) < 0) { + if (errno == EINTR) { + continue; + } + + return -1; + } + + return acceptfd; +} + + +void +hexdump(const void *buf, size_t len) +{ + size_t x; + + printf("%d bytes @ %p \n", (int)len, buf); + + for (x = 0; x < len; x++) { + printf(" %02x", (((char *)buf)[x])&0xff); + if (((x+1) % 16) == 0) + printf("\n"); + } + + printf("\n"); +} + + + diff --git a/rgmanager/src/daemons/Makefile b/rgmanager/src/daemons/Makefile index 7218c53..dbd75ea 100644 --- a/rgmanager/src/daemons/Makefile +++ b/rgmanager/src/daemons/Makefile @@ -1,11 +1,12 @@ TARGET1= rgmanager TARGET2= rg_test TARGET3= clurgmgrd +TARGET4= cpglockd -SBINDIRT=$(TARGET1) $(TARGET2) +SBINDIRT=$(TARGET1) $(TARGET2) $(TARGET4) SBINSYMT=$(TARGET3) -all: depends ${TARGET1} ${TARGET2} ${TARGET3} +all: depends ${TARGET1} ${TARGET2} ${TARGET3} $(TARGET4) include ../../../make/defines.mk include $(OBJDIR)/make/cobj.mk @@ -41,6 +42,8 @@ OBJS2= test-noccs.o \ rg_locks-noccs.o \ event_config-noccs.o +OBJS4= cpglockd.o + CFLAGS += -DSHAREDIR=\"${sharedir}\" -D_GNU_SOURCE CFLAGS += -fPIC CFLAGS += -I${ccsincdir} -I${cmanincdir} -I${dlmincdir} -I${logtincdir} @@ -57,6 +60,7 @@ CCS_LDFLAGS += -L${ccslibdir} -lccs CMAN_LDFLAGS += -L${cmanlibdir} -lcman LOGSYS_LDFLAGS += -L${logtlibdir} -llogthread DLM_LDFLAGS += -L${dlmlibdir} -ldlm +CPG_LDFLAGS += -lcpg XML2_LDFLAGS += `xml2-config --libs` SLANG_LDFLAGS += -L${slanglibdir} -lslang EXTRA_LDFLAGS += -lpthread @@ -96,6 +100,10 @@ ${TARGET2}: ${OBJS2} ${LDDEPS} ${TARGET3}: ${TARGET1} ln -sf ${TARGET1} ${TARGET3} +${TARGET4}: ${OBJS4} ${LDDEPS} + $(CC) -o $@ $^ $(CPG_LDFLAGS) $(EXTRA_LDFLAGS) $(LDFLAGS) + + check: rg_test cd tests && ./runtests.sh diff --git a/rgmanager/src/daemons/cpglockd.c b/rgmanager/src/daemons/cpglockd.c new file mode 100644 index 0000000..55d832a --- /dev/null +++ b/rgmanager/src/daemons/cpglockd.c @@ -0,0 +1,1147 @@ +#include <stdio.h> +#include <signal.h> +#include <sys/types.h> +#include <sys/time.h> +#include <sys/select.h> +#include <errno.h> +#include <unistd.h> +#include <malloc.h> +#include <string.h> +#include <time.h> +#include <sys/uio.h> +#include <corosync/cpg.h> + +#include "sock.h" +#include "cpglock.h" +#include "cpglock-internal.h" +#include "list.h" + + +struct request_node { + list_head(); + struct cpg_lock l; +}; + +struct lock_node { + list_head(); + struct cpg_lock l; +}; + +struct client_node { + list_head(); + int fd; + int pid; +}; + +struct member_node { + list_head(); + int nodeid; +}; + +struct msg_node { + list_head(); + struct cpg_lock_msg m; +}; + +/* Local vars */ +static cpg_handle_t cpg; +static uint32_t my_node_id = 0; +static struct request_node *requests = NULL; +static struct lock_node *locks = NULL; +static struct client_node *clients = NULL; +static struct member_node *group_members = NULL; +static struct msg_node *messages = NULL; +static int total_members = 0; +static int local_lockid = 0; +static int message_count = 0; +static int joined = 0; + + +static const char * +ls2str(int x) +{ + switch(x){ + case LOCK_FREE: return "FREE"; + case LOCK_HELD: return "HELD"; + case LOCK_PENDING: return "PENDING"; + } + return "unknown"; +} + + +static const char * +rq2str(int x) +{ + switch(x){ + case MSG_LOCK: return "LOCK"; + case MSG_UNLOCK: return "UNLOCK"; + case MSG_GRANT: return "GRANT"; + case MSG_NAK: return "NAK"; + case MSG_PURGE: return "PURGE"; + case MSG_CONFCHG: return "CONFCHG"; + case MSG_JOIN: return "JOIN"; + case MSG_HALT: return "HALT"; + } + return "unknown"; +} + + +static void +dump_state(FILE *fp) +{ + struct request_node *r = NULL; + struct lock_node *l = NULL; + struct client_node *c = NULL; + struct member_node *m = NULL; + struct msg_node *s = NULL; + int x; + + fprintf(fp, "cpglockd state\n"); + fprintf(fp, "======== =====\n"); + + fprintf(fp, "Node ID: %d\n", my_node_id); + + if (group_members) { + fprintf(fp, "Participants:"); + list_for(&group_members, m, x) { + fprintf(fp, " %d", m->nodeid); + } + fprintf(fp, "\n"); + } + if (clients) { + fprintf(fp, "Clients:"); + list_for(&clients, c, x) { + fprintf(fp, " %d.%d", c->pid, c->fd ); + } + fprintf(fp, "\n"); + } + fprintf(fp, "\n"); + + if (locks) { + fprintf(fp, "Locks\n"); + fprintf(fp, "=====\n"); + list_for(&locks, l, x) { + fprintf(fp, " %s: %s", l->l.resource, ls2str(l->l.state)); + if (l->l.owner_nodeid) { + fprintf(fp, ", owner %d:%d:%d", l->l.owner_nodeid,l->l.owner_pid, l->l.owner_tid); + if (l->l.owner_nodeid == my_node_id && + l->l.state == LOCK_HELD) + fprintf(fp, ", Local ID %d", l->l.local_id); + } + fprintf(fp, "\n"); + } + fprintf(fp, "\n"); + } + if (requests) { + fprintf(fp, "Requests\n"); + fprintf(fp, "========\n"); + list_for(&requests, r, x) { + fprintf(fp, " %s: %s", r->l.resource, rq2str(r->l.state)); + if (r->l.owner_nodeid) { + fprintf(fp, ", from %d:%d:%d", r->l.owner_nodeid,r->l.owner_pid, r->l.owner_tid); + } + fprintf(fp, "\n"); + } + fprintf(fp, "\n"); + } + if (messages) { + fprintf(fp, "Message History\n"); + fprintf(fp, "======= =======\n"); + list_for(&messages, s, x) { + switch(s->m.request) { + case MSG_CONFCHG: + fprintf(fp, " CONFIG CHANGE\n"); + break; + case MSG_PURGE: + fprintf(fp, " PURGE for %d:%d\n", s->m.owner_nodeid, s->m.owner_pid); + break; + case MSG_JOIN: + fprintf(fp, " JOIN %d\n", s->m.owner_nodeid); + break; + default: + fprintf(fp, " %s: %s %d:%d:%d\n", rq2str(s->m.request), s->m.resource, s->m.owner_nodeid, s->m.owner_pid, s->m.owner_tid); + break; + } + } + fprintf(fp, "\n"); + } +} + + +static void +old_msg(struct cpg_lock_msg *m) +{ + struct msg_node *n; + + n = do_alloc(sizeof(*n)); + memcpy(&n->m, m, sizeof(n->m)); + list_append(&messages, n); + if (message_count < 10) { + ++message_count; + } else { + n = messages; + list_remove(&messages, n); + free(n); + } +} + + +static void +insert_client(int fd) +{ + struct client_node *n = NULL; + + n = do_alloc(sizeof(*n)); + n->fd = fd; + list_append(&clients, n); +} + + +/* forward request from client */ +static int +send_lock_msg(struct cpg_lock_msg *m) +{ + struct iovec iov; + + iov.iov_base = m; + iov.iov_len = sizeof (*m); + + return cpg_mcast_joined(cpg, CPG_TYPE_AGREED, &iov, 1); +} + + +/* forward request from client */ +static int +send_lock(struct cpg_lock_msg *m) +{ + m->owner_nodeid = my_node_id; + + return send_lock_msg(m); +} + + +static int +send_grant(struct request_node *n) +{ + struct cpg_lock_msg m; + + printf("-> sending grant %s to %d:%d:%d\n", + n->l.resource, n->l.owner_nodeid, n->l.owner_pid, n->l.owner_tid); + + memset(&m, 0, sizeof(m)); + strncpy(m.resource, n->l.resource, sizeof(m.resource)); + m.request = MSG_GRANT; + m.owner_nodeid = n->l.owner_nodeid; + m.owner_pid = n->l.owner_pid; + m.owner_tid = n->l.owner_tid; + + return send_lock_msg(&m); +} + + +static int +send_nak(struct cpg_lock_msg *m) +{ + m->request = MSG_NAK; + + return send_lock_msg(m); +} + + +static int +send_join(void) +{ + struct cpg_lock_msg m; + + m.request = MSG_JOIN; + m.owner_nodeid = my_node_id; + return send_lock_msg(&m); +} + + + +static int +send_unlock(struct cpg_lock_msg *m) +{ + m->request = MSG_UNLOCK; + return send_lock_msg(m); +} + + +/* + * Grant the lock in this request node to the next + * waiting client. + */ +static int +grant_next(struct cpg_lock_msg *m) +{ + struct request_node *r; + int x; + + list_for(&requests, r, x) { + if (strcmp(m->resource, r->l.resource)) + continue; + + /* Send grant */ + if (r->l.state == LOCK_PENDING) { + printf("LOCK %s: grant to %d:%d:%d\n", m->resource, + r->l.owner_nodeid, r->l.owner_pid, r->l.owner_tid); + /* don't send dup grants */ + r->l.state = LOCK_HELD; + send_grant(r); + } + return 1; + } + + return 0; +} + + +static void +purge_requests(uint32_t nodeid, uint32_t pid) +{ + struct request_node *r; + int found = 0, count = 0, x = 0; + + do { + found = 0; + list_for(&requests, r, x) { + if (r->l.owner_nodeid != nodeid || + (pid && + r->l.owner_pid != pid)) + continue; + + list_remove(&requests, r); + free(r); + found = 1; + ++count; + break; + } + } while (found); + + if (count) { + if (pid) { + printf("RECOVERY: purged %d requests from %d:%d\n", count, nodeid, pid); + } else { + printf("RECOVERY: purged %d requests from node %d\n", count, nodeid); + } + } +} + + + +static void +del_client(int fd) +{ + struct cpg_lock_msg m; + struct client_node *n; + struct lock_node *l; + int x, pid = 0, recovered = 0; + + list_for(&clients, n, x) { + if (n->fd == fd) { + list_remove(&clients, n); + close(n->fd); + pid = n->pid; + free(n); + break; + } + } + + if (!pid) + return; + + printf("RECOVERY: Looking for locks held by PID %d\n", pid); + + /* This may not be needed */ + purge_requests(my_node_id, pid); + + m.request = MSG_PURGE; + m.owner_nodeid = my_node_id; + m.owner_pid = pid; + + send_lock_msg(&m); + + list_for(&locks, l, x) { + if (l->l.owner_nodeid != my_node_id || + l->l.owner_pid != pid || + l->l.state != LOCK_HELD) + continue; + + printf("RECOVERY: Releasing %s \n", l->l.resource); + + l->l.state = LOCK_FREE; + strncpy(m.resource, l->l.resource, sizeof(m.resource)); + if (grant_next(&m) == 0) + send_unlock(&m); + } + + if (recovered) { + printf("RECOVERY: %d locks from local PID %d\n", recovered, pid); + } + printf("RECOVERY: Complete\n"); +} + + +static void +del_node(uint32_t nodeid) +{ + struct cpg_lock_msg m; + struct lock_node *l; + int x, recovered = 0, granted = 0; + + if (group_members->nodeid != my_node_id) + return; + + printf("RECOVERY: I am oldest node in the group, recovering locks\n"); + + /* pass 1: purge outstanding requests from this node. */ + + /* This may not be needed */ + purge_requests(nodeid, 0); + + m.request = MSG_PURGE; + m.owner_nodeid = nodeid; + m.owner_pid = 0; + + send_lock_msg(&m); + + list_for(&locks, l, x) { + if (l->l.owner_nodeid == nodeid && + l->l.state == LOCK_HELD) { + printf("RECOVERY: Releasing %s held by dead node %d\n", l->l.resource, + nodeid); + + l->l.state = LOCK_FREE; + strncpy(m.resource, l->l.resource, sizeof(m.resource)); + if (grant_next(&m) == 0) + send_unlock(&m); + ++recovered; + } else if (l->l.state == LOCK_FREE) { + if (grant_next(&m) == 0) + send_unlock(&m); + ++granted; + } + } + + if (recovered) { + printf("RECOVERY: %d locks from node %d\n", recovered, nodeid); + } + if (granted) { + printf("RECOVERY: %d pending locks granted\n", granted); + } + + printf("RECOVERY: Complete\n"); +} + + +static int +client_fdset(fd_set *set) +{ + int max = -1, x = 0; + struct client_node *n; + + FD_ZERO(set); + + list_for(&clients, n, x) { + FD_SET(n->fd, set); + if (n->fd > max) + max = n->fd; + } + + if (!x) + return 0; + + return max; +} + + +static struct client_node * +find_client(int pid) +{ + int x; + struct client_node *n; + + list_for(&clients, n, x) { + if (n->pid == pid) + return n; + } + + return NULL; +} + + +#if 0 +static void +send_fault(const char *resource) +{ + struct cpg_lock_msg m; + + strncpy(m.resource, resource, sizeof(m.resource)); + m.request = MSG_HALT; + m.owner_pid = 0; + m.owner_nodeid = my_node_id; + + send_lock_msg(&m); +} +#endif + + +static int +grant_client(struct lock_node *l) +{ + struct client_node *c; + struct cpg_lock_msg m; + + //memset(&m, 0, sizeof(m)); + strncpy(m.resource, l->l.resource, sizeof(m.resource)); + m.request = MSG_GRANT; + m.owner_pid = l->l.owner_pid; + m.owner_tid = l->l.owner_tid; + l->l.local_id = ++local_lockid; + m.lockid = l->l.local_id; + m.owner_nodeid = my_node_id; + + c = find_client(l->l.owner_pid); + if (!c) { + printf("can't find client for pid %d\n", l->l.owner_pid); + return 1; + } + + if (c->fd < 0) { + printf(" Client has bad fd\n"); + return -1; + } + + if (write_retry(c->fd, &m, sizeof(m), NULL) < 0) { + /* no client anymore; drop and send to next guy XXX */ + /* This should be handled by our main loop */ + //printf("Failed to notify client!\n"); + } + + return 0; +} + + +static int +nak_client(struct request_node *l) +{ + struct client_node *c; + struct cpg_lock_msg m; + + strncpy(m.resource, l->l.resource, sizeof(m.resource)); + m.request = MSG_NAK; + m.owner_pid = l->l.owner_pid; + m.owner_tid = l->l.owner_tid; + m.owner_nodeid = my_node_id; + + c = find_client(l->l.owner_pid); + if (!c) { + printf("can't find client for pid %d\n", l->l.owner_pid); + return 1; + } + + if (c->fd < 0) { + printf(" Client has bad fd\n"); + return -1; + } + + if (write_retry(c->fd, &m, sizeof(m), NULL) < 0) { + /* no client anymore; drop and send to next guy XXX */ + /* This should be handled by our main loop */ + //printf("Failed to notify client!\n"); + } + + return 0; +} + +static int +queue_request(struct cpg_lock_msg *m) +{ + struct request_node *r; + + r = do_alloc(sizeof(*r)); + strncpy(r->l.resource, m->resource, sizeof(r->l.resource)); + r->l.owner_nodeid = m->owner_nodeid; + r->l.owner_pid = m->owner_pid; + r->l.owner_tid = m->owner_tid; + r->l.state = LOCK_PENDING; + + list_insert(&requests, r); + return 0; +} + + +static int +process_lock(struct cpg_lock_msg *m) +{ + struct lock_node *l; + int x; + + if (!joined) + return 0; + + printf("LOCK %s: queue for %d:%d:%d\n", m->resource, + m->owner_nodeid, m->owner_pid, m->owner_tid); + queue_request(m); + + list_for(&locks, l, x) { + if (strcmp(m->resource, l->l.resource)) + continue; + + /* if it's owned locally, we need send a + GRANT to the first node on the request queue */ + if (l->l.owner_nodeid == my_node_id) { + if (l->l.state == LOCK_FREE) { + /* Set local state to PENDING to avoid double-grants */ + l->l.state = LOCK_PENDING; + grant_next(m); + } else { + /* state is PENDING or HELD */ + if (m->flags & FL_TRY) { + /* nack to client if needed */ + send_nak(m); + } + } + } + + + return 0; + } + + l = do_alloc(sizeof(*l)); + strncpy(l->l.resource, m->resource, sizeof(l->l.resource)); + l->l.state = LOCK_FREE; + list_insert(&locks, l); + + if (group_members->nodeid == my_node_id) { + /* Allocate a lock structure and immediately grant */ + l->l.state = LOCK_PENDING; + if (grant_next(m) == 0) + l->l.state = LOCK_FREE; + } + + return 0; +} + + +static int +is_member(uint32_t nodeid) +{ + struct member_node *n; + int x; + + list_for(&group_members, n, x) { + if (n->nodeid == nodeid) + return 1; + } + + return 0; +} + + +static int +process_grant(struct cpg_lock_msg *m, uint32_t nodeid) +{ + struct lock_node *l; + struct request_node *r; + int x, y; + + if (!joined) + return 0; + + list_for(&locks, l, x) { + if (strcmp(m->resource, l->l.resource)) + continue; + + if (l->l.state == LOCK_HELD) { + if (m->owner_pid == 0 || + m->owner_nodeid == 0) { + printf("GRANT averted\n"); + return 0; + } + } else { + l->l.state = LOCK_HELD; + } + + printf("GRANT %s: to %d:%d:%d\n", + m->resource, m->owner_nodeid, + m->owner_pid, m->owner_tid); + + l->l.owner_nodeid = m->owner_nodeid; + l->l.owner_pid = m->owner_pid; + l->l.owner_tid = m->owner_tid; + + list_for(&requests, r, y) { + if (strcmp(r->l.resource, m->resource)) + continue; + + if (r->l.owner_nodeid == m->owner_nodeid && + r->l.owner_pid == m->owner_pid && + r->l.owner_tid == m->owner_tid) { + list_remove(&requests, r); + free(r); + break; + } + } + + /* granted lock */ + if (l->l.owner_nodeid == my_node_id) { + if (grant_client(l) != 0) { + /* Grant to a nonexistent PID can + happen because we may have a pending + request after a fd was closed. + since we process on delivery, we + now simply make an unlock request + and move on */ + purge_requests(my_node_id, l->l.owner_pid); + if (grant_next(m) == 0) + send_unlock(m); + return 0; + } + } + + /* What if node has died with a GRANT in flight? */ + if (group_members->nodeid == my_node_id && + !is_member(l->l.owner_nodeid)) { + + printf("GRANT to non-member %d; giving to next requestor\n", l->l.owner_nodeid); + + l->l.state = LOCK_FREE; + if (grant_next(m) == 0) + send_unlock(m); + return 0; + } + return 0; + } + + /* Record lock state since we now know it */ + /* Allocate a lock structure */ + l = do_alloc(sizeof(*m)); + strncpy(l->l.resource, m->resource, sizeof(l->l.resource)); + l->l.state = LOCK_HELD; + l->l.owner_nodeid = m->owner_nodeid; + l->l.owner_pid = m->owner_pid; + l->l.owner_tid = m->owner_tid; + list_insert(&locks, l); + + return 0; +} + + +static int +process_nak(struct cpg_lock_msg *m, uint32_t nodeid) +{ + struct request_node *r = NULL; + int y; + + if (!joined) + return 0; + + list_for(&requests, r, y) { + if (strcmp(r->l.resource, m->resource)) + continue; + + if (r->l.owner_nodeid == m->owner_nodeid && + r->l.owner_pid == m->owner_pid && + r->l.owner_tid == m->owner_tid) { + list_remove(&requests, r); + if (r->l.owner_nodeid == my_node_id) { + if (nak_client(r) != 0) { + purge_requests(my_node_id, r->l.owner_pid); + } + } + free(r); + break; + } + } + + return 0; +} + + +static int +process_unlock(struct cpg_lock_msg *m, uint32_t nodeid) +{ + struct lock_node *l; + int x; + + if (!joined) + return 0; + + list_for(&locks, l, x) { + if (l->l.state != LOCK_HELD) + continue; + if (strcmp(m->resource, l->l.resource)) + continue; + + /* Held lock... if it's local, we need send a + GRANT to the first node on the request queue */ + if (l->l.owner_nodeid == m->owner_nodeid && + l->l.owner_pid == m->owner_pid) { + printf("UNLOCK %s: %d:%d:%d\n", m->resource, m->owner_nodeid, m->owner_pid, m->owner_tid); + l->l.state = LOCK_FREE; + if (grant_next(m) != 0) + l->l.state = LOCK_PENDING; + } + } + + return 0; +} + + +static int +find_lock(struct cpg_lock_msg *m) +{ + struct lock_node *l; + int x; + + if (m->resource[0] != 0) + return 0; + + list_for(&locks, l, x) { + if (m->lockid == l->l.local_id) { + strncpy(m->resource, l->l.resource, sizeof(m->resource)); + printf("LOCK %d -> %s\n", m->lockid, m->resource); + m->owner_nodeid = l->l.owner_nodeid; + m->owner_pid = l->l.owner_pid; + m->owner_tid = l->l.owner_tid; + m->lockid = 0; + return 0; + } + } + + return 1; +} + + +static int +process_join(struct cpg_lock_msg *m, uint32_t nodeid) +{ + struct member_node *n; + int x; + + list_for(&group_members, n, x) { + if (n->nodeid == nodeid) { + list_remove(&group_members, n); + list_append(&group_members, n); + printf("JOIN: moving %d to back\n", nodeid); + return 0; + } + } + + n = do_alloc(sizeof(*n)); + n->nodeid = nodeid; + printf("JOIN: node %d", n->nodeid); + if (nodeid == my_node_id) { + printf(" (self)"); + joined = 1; + } + total_members++; + printf("\n"); + list_insert(&group_members, n); + + return 0; +} + + +static int +process_request(struct cpg_lock_msg *m, uint32_t nodeid) +{ + if (m->request == MSG_HALT) { + printf("FAULT: Halting operations; see node %d\n", m->owner_nodeid); + while (1) + sleep(30); + } + + old_msg(m); + + switch (m->request) { + case MSG_LOCK: + process_lock(m); + break; + case MSG_NAK: + process_nak(m, nodeid); + break; + case MSG_GRANT: + process_grant(m, nodeid); + break; + case MSG_UNLOCK: + process_unlock(m, nodeid); + break; + case MSG_PURGE: + purge_requests(m->owner_nodeid, m->owner_pid); + break; + case MSG_JOIN: + process_join(m, nodeid); + break; + } + + return 0; +} + + +static void +cpg_deliver_func(cpg_handle_t h, + const struct cpg_name *group_name, + uint32_t nodeid, + uint32_t pid, + void *msg, + size_t msglen) +{ + + if (msglen != sizeof(struct cpg_lock_msg)) { + printf("Invalid message size %d\n", (int)msglen); + } + + process_request((struct cpg_lock_msg *)msg, nodeid); +} + + +static void +cpg_config_change(cpg_handle_t h, + const struct cpg_name *group_name, + const struct cpg_address *members, size_t memberlen, + const struct cpg_address *left, size_t leftlen, + const struct cpg_address *join, size_t joinlen) +{ + struct member_node *n; + size_t x, y; + struct cpg_lock_msg m; + + + memset(&m, 0, sizeof(m)); + strncpy(m.resource, "(none)", sizeof(m.resource)); + m.request = MSG_CONFCHG; + + old_msg(&m); + + if (total_members == 0) { + + printf("JOIN: Setting up initial node list\n"); + for (x = 0; x < memberlen; x++) { + for (y = 0; y < joinlen; y++) { + if (join[y].nodeid == members[x].nodeid) + continue; + if (members[x].nodeid == my_node_id) + continue; + + n = do_alloc(sizeof(*n)); + n->nodeid = members[x].nodeid; + printf("JOIN: node %d\n", n->nodeid); + list_insert(&group_members, n); + } + } + printf("JOIN: Done\n"); + + total_members = memberlen; + } + + //printf("members %d now, %d joined, %d left\n", memberlen, joinlen, leftlen); +#if 0 + + /* XXX process join on receipt of JOIN message rather than here + since ordered delivery is agreed, this prevents >1 member from + believing it is the oldest host */ + for (x = 0; x < joinlen; x++) { + n = do_alloc(sizeof(*n)); + n->nodeid = join[x].nodeid; + printf("ADD: node %d\n", n->nodeid); + list_insert(&group_members, n); + } +#endif + + for (x = 0; x < leftlen; x++) { + + list_for(&group_members, n, y) { + if (n->nodeid == left[x].nodeid) { + list_remove(&group_members, n); + printf("DELETE: node %d\n", n->nodeid); + del_node(n->nodeid); + free(n); + break; + } + } + + total_members -= leftlen; + if (total_members < 0) + total_members = 0; + } + +#if 0 + printf("MEMBERS:"); + list_for(&group_members, n, y) { + printf(" %d", n->nodeid); + } + printf("\n"); +#endif + + return; +} + + +static cpg_callbacks_t my_callbacks = { + .cpg_deliver_fn = cpg_deliver_func, + .cpg_confchg_fn = cpg_config_change +}; + + +static int +cpg_init(void) +{ + struct cpg_name gname; + + errno = EINVAL; + + gname.length = snprintf(gname.value, + sizeof(gname.value), + CPG_LOCKD_NAME); + if (gname.length >= sizeof(gname.value)) { + errno = ENAMETOOLONG; + return -1; + } + + if (gname.length <= 0) + return -1; + + memset(&cpg, 0, sizeof(cpg)); + if (cpg_initialize(&cpg, &my_callbacks) != CPG_OK) { + perror("cpg_initialize"); + return -1; + } + + if (cpg_join(cpg, &gname) != CPG_OK) { + perror("cpg_join"); + return -1; + } + + cpg_local_get(cpg, &my_node_id); + + return 0; +} + + +static int +cpg_fin(void) +{ + struct cpg_name gname; + + errno = EINVAL; + + gname.length = snprintf(gname.value, + sizeof(gname.value), + CPG_LOCKD_NAME); + if (gname.length >= sizeof(gname.value)) { + errno = ENAMETOOLONG; + return -1; + } + + if (gname.length <= 0) + return -1; + + cpg_leave(cpg, &gname); + cpg_finalize(cpg); + + return 0; +} + + +int +main(int argc, char **argv) +{ + fd_set rfds; + int fd; + int cpgfd; + int afd = -1; + int n,x; + + struct cpg_lock l; + struct cpg_lock_msg m; + struct client_node *client; + + l.state = 0; + + signal(SIGPIPE, SIG_IGN); + + fd = sock_listen(CPG_LOCKD_SOCK); + cpg_init(); + cpg_local_get(cpg, &my_node_id); + cpg_fd_get(cpg, &cpgfd); + if (send_join() < 0) + return -1; + + while (1) { + FD_ZERO(&rfds); + x = client_fdset(&rfds); + FD_SET(fd, &rfds); + if (fd > x) + x = fd; + FD_SET(cpgfd, &rfds); + if (cpgfd > x) + x = cpgfd; + + n = select_retry(x+1, &rfds, NULL, NULL, NULL); + if (n < 0) { + perror("select"); + return -1; + } + + if (FD_ISSET(fd, &rfds)) { + afd = accept(fd, NULL, NULL); + insert_client(afd); + --n; + } + + if (FD_ISSET(cpgfd, &rfds)) { + cpg_dispatch(cpg, CPG_DISPATCH_ONE); + --n; + } + + if (n <= 0) + continue; + + do { + list_for(&clients, client, x) { + if (!FD_ISSET(client->fd, &rfds)) + continue; + --n; + if (read_retry(client->fd, &m, sizeof(m), NULL) < 0) { + printf("Closing client fd %d pid %d: %d\n", + client->fd, client->pid, errno); + + del_client(client->fd); + break; + } + + /* send lock request */ + /* XXX check for dup connection */ + if (m.request == MSG_LOCK) { + client->pid = m.owner_pid; + send_lock(&m); + } + + if (m.request == MSG_UNLOCK) { + //printf("Unlock from fd %d\n", client->fd); + find_lock(&m); + if (grant_next(&m) == 0) + send_unlock(&m); + } + + if (m.request == MSG_DUMP) { + FILE *fp = fdopen(client->fd, "w"); + + list_remove(&clients, client); + dump_state(fp); + fclose(fp); + close(client->fd); + free(client); + break; + } + } + } while (n); + } + + cpg_fin(); + + return 0; +} diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c index 931d95e..40717e6 100644 --- a/rgmanager/src/daemons/main.c +++ b/rgmanager/src/daemons/main.c @@ -987,14 +987,14 @@ int main(int argc, char **argv) { int rv, do_init = 1; - char foreground = 0, wd = 1; + char foreground = 0, wd = 1, cpg_locks = 0; cman_node_t me; msgctx_t *cluster_ctx; msgctx_t *local_ctx; pthread_t th; cman_handle_t clu = NULL; - while ((rv = getopt(argc, argv, "wfdNq")) != EOF) { + while ((rv = getopt(argc, argv, "wfdNqC")) != EOF) { switch (rv) { case 'w': wd = 0; @@ -1011,6 +1011,9 @@ main(int argc, char **argv) case 'q': rgm_dbus_notify = 0; break; + case 'C': + cpg_locks = 1; + break; default: return 1; break; @@ -1057,10 +1060,19 @@ main(int argc, char **argv) return -1; } - if (clu_lock_init(rgmanager_lsname) != 0) { - printf("Locks not working!\n"); - cman_finish(clu); - return -1; + if (!cpg_locks) { + if (clu_lock_init(rgmanager_lsname) != 0) { + printf("Locks not working!\n"); + cman_finish(clu); + return -1; + } + } else { + if (cpg_lock_initialize() != 0) { + printf("Locks not working!\n"); + cman_finish(clu); + return -1; + } + logt_print(LOG_INFO, "Using CPG for locking (EXPERIMENTAL)\n"); } memset(&me, 0, sizeof(me));

1 0

gfs2-utils: master - libgfs2: Clean up some warnings
by Andrew Price 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=f6… Commit: f6cc05782370c12f8178b1f07ffaac8629657f3a Parent: e639b773bc6568bf5861ca6738c6c471dafa086d Author: Andrew Price <anprice(a)redhat.com> AuthorDate: Fri Apr 27 11:15:12 2012 +0100 Committer: Andrew Price <anprice(a)redhat.com> CommitterDate: Mon Apr 30 18:29:01 2012 +0100 libgfs2: Clean up some warnings Fixes these warnings: fs_geometry.c:80:47: warning: variable 'rlast2' set but not used [-Wunused-but-set-variable] meta.c:732:3: warning: large integer implicitly truncated to unsigned type [-Woverflow] Signed-off-by: Andrew Price <anprice(a)redhat.com> --- gfs2/libgfs2/fs_geometry.c | 3 +-- gfs2/libgfs2/meta.c | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/gfs2/libgfs2/fs_geometry.c b/gfs2/libgfs2/fs_geometry.c index fb55697..e248f7c 100644 --- a/gfs2/libgfs2/fs_geometry.c +++ b/gfs2/libgfs2/fs_geometry.c @@ -77,7 +77,7 @@ void compute_rgrp_layout(struct gfs2_sbd *sdp, struct osi_root *rgtree, int rgsize_specified) { struct device *dev; - struct rgrp_tree *rl, *rlast = NULL, *rlast2 = NULL; + struct rgrp_tree *rl, *rlast = NULL; struct osi_node *n, *next = NULL; unsigned int rgrp = 0, nrgrp, rglength; uint64_t rgaddr; @@ -105,7 +105,6 @@ void compute_rgrp_layout(struct gfs2_sbd *sdp, struct osi_root *rgtree, PRIx64 "), length = %"PRIu64" (0x%" PRIx64 ")\n", rgrp + 1, rl->start, rl->start, rl->length, rl->length); - rlast2 = rlast; rlast = rl; } rlast->start = rlast->ri.ri_addr; diff --git a/gfs2/libgfs2/meta.c b/gfs2/libgfs2/meta.c index 87f8f31..1f796c8 100644 --- a/gfs2/libgfs2/meta.c +++ b/gfs2/libgfs2/meta.c @@ -729,7 +729,7 @@ const struct lgfs2_metadata lgfs2_metadata[] = { }, [LGFS2_MT_FREE] = { .gfs1 = 1, - .gfs2 = 2, + .gfs2 = 1, .name = "free", }, };

1 0

gfs2-utils: master - libgfs2: Remove gfs_get_leaf_nr
by Andrew Price 01 May '12

01 May '12

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=e6… Commit: e639b773bc6568bf5861ca6738c6c471dafa086d Parent: 63da1508c84a38b550510810cdfe1c3cb68298a4 Author: Andrew Price <anprice(a)redhat.com> AuthorDate: Fri Apr 27 11:04:35 2012 +0100 Committer: Andrew Price <anprice(a)redhat.com> CommitterDate: Mon Apr 30 18:28:48 2012 +0100 libgfs2: Remove gfs_get_leaf_nr gfs2_get_leaf_nr and gfs_get_leaf_nr are identical so we can remove gfs_get_leaf_nr. Signed-off-by: Andrew Price <anprice(a)redhat.com> --- gfs2/libgfs2/fs_ops.c | 2 -- gfs2/libgfs2/gfs1.c | 16 ---------------- gfs2/libgfs2/libgfs2.h | 2 -- 3 files changed, 0 insertions(+), 20 deletions(-) diff --git a/gfs2/libgfs2/fs_ops.c b/gfs2/libgfs2/fs_ops.c index a72394f..470916c 100644 --- a/gfs2/libgfs2/fs_ops.c +++ b/gfs2/libgfs2/fs_ops.c @@ -851,8 +851,6 @@ void gfs2_get_leaf_nr(struct gfs2_inode *dip, uint32_t lindex, uint64_t leaf_no; int count; - if (dip->i_sbd->gfs1) - return gfs_get_leaf_nr(dip, lindex, leaf_out); count = gfs2_readi(dip, (char *)&leaf_no, lindex * sizeof(uint64_t), sizeof(uint64_t)); if (count != sizeof(uint64_t)) { diff --git a/gfs2/libgfs2/gfs1.c b/gfs2/libgfs2/gfs1.c index 8e3fc93..d6f8794 100644 --- a/gfs2/libgfs2/gfs1.c +++ b/gfs2/libgfs2/gfs1.c @@ -395,22 +395,6 @@ void gfs_rgrp_out(struct gfs_rgrp *rgrp, struct gfs2_buffer_head *rbh) bmodified(rbh); } -void gfs_get_leaf_nr(struct gfs2_inode *dip, uint32_t lindex, - uint64_t *leaf_out) -{ - uint64_t leaf_no; - int count; - - count = gfs2_readi(dip, (char *)&leaf_no, lindex * sizeof(uint64_t), - sizeof(uint64_t)); - if (count != sizeof(uint64_t)) { - fprintf(stderr, "gfs_get_leaf_nr: Bad internal read.\n"); - exit(1); - } - - *leaf_out = be64_to_cpu(leaf_no); -} - void gfs_put_leaf_nr(struct gfs2_inode *dip, uint32_t inx, uint64_t leaf_out) { uint64_t leaf_no; diff --git a/gfs2/libgfs2/libgfs2.h b/gfs2/libgfs2/libgfs2.h index 4bd01a8..e2a7e45 100644 --- a/gfs2/libgfs2/libgfs2.h +++ b/gfs2/libgfs2/libgfs2.h @@ -660,8 +660,6 @@ extern struct gfs2_inode *gfs_inode_read(struct gfs2_sbd *sdp, extern void gfs_jindex_in(struct gfs_jindex *jindex, char *buf); extern void gfs_rgrp_in(struct gfs_rgrp *rg, struct gfs2_buffer_head *bh); extern void gfs_rgrp_out(struct gfs_rgrp *rg, struct gfs2_buffer_head *bh); -extern void gfs_get_leaf_nr(struct gfs2_inode *dip, uint32_t lindex, - uint64_t *leaf_out); extern void gfs_put_leaf_nr(struct gfs2_inode *dip, uint32_t inx, uint64_t leaf_out);

1 0

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

cluster-commits May 2012