cluster-commits April 2011

cluster-commits@lists.stg.fedorahosted.org

4 participants
37 discussions

cluster: RHEL57 - rgmanager: Initial commit of central proc + migration support
by Lon Hohberger 08 Apr '11

08 Apr '11

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=96… Commit: 963d04336736b0ad7b4582b100048ea89ba48fad Parent: 640f9211a325d1e093fd3c108a97fedae646e86e Author: Lon Hohberger <lhh(a)redhat.com> AuthorDate: Wed Oct 21 13:38:42 2009 -0400 Committer: Lon Hohberger <lhh(a)redhat.com> CommitterDate: Fri Apr 8 09:31:15 2011 -0400 rgmanager: Initial commit of central proc + migration support This adds preliminary migration support of virtual machines when using central processing. Resolves: rhbz#525271 Signed-off-by: Lon Hohberger <lhh(a)redhat.com> --- rgmanager/src/daemons/service_op.c | 85 ++++++++++++++++++++++- rgmanager/src/daemons/slang_event.c | 59 ++++++++++++++++ rgmanager/src/resources/default_event_script.sl | 8 +- 3 files changed, 146 insertions(+), 6 deletions(-) diff --git a/rgmanager/src/daemons/service_op.c b/rgmanager/src/daemons/service_op.c index 6b0db24..2ea59f9 100644 --- a/rgmanager/src/daemons/service_op.c +++ b/rgmanager/src/daemons/service_op.c @@ -291,7 +291,88 @@ service_op_convalesce(char *svcName) /* - TODO - service_op_migrate() + service_op_migrate() - send a virtual machine to another host + in the cluster */ +int +service_op_migrate(char *svcName, + int target_node) +{ + SmMessageSt msg; + int msg_ret; + msgctx_t ctx; + rg_state_t svcStatus; + int msgtarget = my_id(); + + /* Build the message header */ + msg.sm_hdr.gh_magic = GENERIC_HDR_MAGIC; + msg.sm_hdr.gh_command = RG_ACTION_REQUEST; + msg.sm_hdr.gh_arg1 = RG_ACTION_MASTER; + msg.sm_hdr.gh_length = sizeof (SmMessageSt); + + msg.sm_data.d_action = RG_MIGRATE; + + strncpy(msg.sm_data.d_svcName, svcName, + sizeof(msg.sm_data.d_svcName)); + + msg.sm_data.d_ret = 0; + msg.sm_data.d_svcOwner = target_node; + /* Open a connection to the local node - it will decide what to + do in this case. XXX inefficient; should queue requests + locally and immediately forward requests otherwise */ + + if (get_service_state_internal(svcName, &svcStatus) < 0) + return RG_EFAIL; + if (svcStatus.rs_owner > 0) { + if (member_online(svcStatus.rs_owner)) { + msgtarget = svcStatus.rs_owner; + } + + if (msgtarget <= 0) { + return RG_EFAIL; + } + } + + if (msg_open(MSG_CLUSTER, msgtarget, RG_PORT, &ctx, 2)< 0) { + clulog(LOG_ERR, + "#58: Failed opening connection to member #%d\n", + my_id()); + return -1; + } + + /* Encode */ + swab_SmMessageSt(&msg); + + /* Send stop message to the other node */ + if (msg_send(&ctx, &msg, sizeof (SmMessageSt)) < + (int)sizeof (SmMessageSt)) { + clulog(LOG_ERR, "Failed to send complete message\n"); + msg_close(&ctx); + return -1; + } + + /* Check the response */ + do { + msg_ret = msg_receive(&ctx, &msg, + sizeof (SmMessageSt), 10); + if ((msg_ret == -1 && errno != ETIMEDOUT) || + (msg_ret > 0)) { + break; + } + } while(1); + + if (msg_ret != sizeof (SmMessageSt)) { + clulog(LOG_WARNING, "Strange response size: %d vs %d\n", + msg_ret, (int)sizeof(SmMessageSt)); + return 0; /* XXX really UNKNOWN */ + } + + /* Got a valid response from other node. */ + msg_close(&ctx); + + /* Decode */ + swab_SmMessageSt(&msg); + + return msg.sm_data.d_ret; +} diff --git a/rgmanager/src/daemons/slang_event.c b/rgmanager/src/daemons/slang_event.c index 08974ad..468706f 100644 --- a/rgmanager/src/daemons/slang_event.c +++ b/rgmanager/src/daemons/slang_event.c @@ -619,6 +619,63 @@ out: } +static int +sl_migrate_service(void) +{ + char *svcname = NULL; + int target_node = 0; + int nargs, t, newowner = 0, ret = -1; + + nargs = SLang_Num_Function_Args; + + /* Takes one, two, or three */ + if (nargs != 2) { + SLang_verror(SL_Syntax_Error, + (char *)"%s: Wrong # of args (%d), must be 2: service_name target_node\n", + __FUNCTION__, nargs); + return -1; + } + + t = SLang_peek_at_stack(); + if (t != SLANG_INT_TYPE) { + SLang_verror(SL_Syntax_Error, + (char *)"%s: expected type %d got %d\n", + __FUNCTION__, SLANG_INT_TYPE, t); + goto out; + } + + if (SLang_pop_integer(&target_node) < 0) { + SLang_verror(SL_Syntax_Error, + (char *)"%s: Failed to pop integer from stack!\n", + __FUNCTION__); + goto out; + } + + t = SLang_peek_at_stack(); + if (t != SLANG_STRING_TYPE) { + SLang_verror(SL_Syntax_Error, + (char *)"%s: expected type %d got %d\n", + __FUNCTION__, + SLANG_STRING_TYPE, t); + goto out; + } + + if (SLpop_string(&svcname) < 0) { + goto out; + } + + ret = service_op_migrate(svcname, target_node); + + if (ret == 0) + ret = target_node; +out: + if (svcname) + free(svcname); + _user_return = ret; + return ret; +} + + /* Take an array of integers given its length and push it on to the S/Lang stack */ void @@ -1016,6 +1073,8 @@ SLang_Intrin_Fun_Type rgmanager_slang[] = MAKE_INTRINSIC_S((char *)"service_convalesce", sl_convalesce_service, SLANG_INT_TYPE), MAKE_INTRINSIC_0("service_start", sl_start_service, SLANG_INT_TYPE), + MAKE_INTRINSIC_0((char *)"service_migrate", sl_migrate_service, + SLANG_INT_TYPE), MAKE_INTRINSIC_S("service_status", sl_service_status, SLANG_VOID_TYPE), MAKE_INTRINSIC_S("service_freeze", sl_service_freeze, diff --git a/rgmanager/src/resources/default_event_script.sl b/rgmanager/src/resources/default_event_script.sl index da6bc1f..db6e789 100644 --- a/rgmanager/src/resources/default_event_script.sl +++ b/rgmanager/src/resources/default_event_script.sl @@ -597,11 +597,11 @@ define default_user_event_handler() ret = service_convalesce(service_name); - } + } else if (user_request == USER_MIGRATE) { - % - % todo - migrate - % + ret = service_migrate(service_name, user_target); + + } return ret; }

1 0

cluster: RHEL57 - rgmanager: Allow non-root clustat
by Lon Hohberger 08 Apr '11

08 Apr '11

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=64… Commit: 640f9211a325d1e093fd3c108a97fedae646e86e Parent: d4dd514faa27f4f7c91205d7525c7f3bc1885aa3 Author: Lon Hohberger <lhh(a)redhat.com> AuthorDate: Mon Mar 28 15:56:08 2011 -0400 Committer: Lon Hohberger <lhh(a)redhat.com> CommitterDate: Fri Apr 8 09:31:15 2011 -0400 rgmanager: Allow non-root clustat This allows non-root users access to the running cluster and service states. This requires you to add a user to the 'root' group, matching cman_tool's requirements. Simple backport from RHEL6 / STABLE31 Resolves: rhbz#510300 Signed-off-by: Lon Hohberger <lhh(a)redhat.com> --- rgmanager/include/message.h | 1 + rgmanager/include/resgroup.h | 3 +++ rgmanager/src/clulib/msg_socket.c | 16 +++++++++++++++- rgmanager/src/clulib/rg_strings.c | 2 ++ rgmanager/src/daemons/main.c | 33 ++++++++++++++++++++++++++++++++- 5 files changed, 53 insertions(+), 2 deletions(-) diff --git a/rgmanager/include/message.h b/rgmanager/include/message.h index f987ec6..4b9ae3a 100644 --- a/rgmanager/include/message.h +++ b/rgmanager/include/message.h @@ -89,6 +89,7 @@ typedef struct _msgctx { } cluster_info; struct { int sockfd; + struct ucred cred; int pad; } local_info; } u; diff --git a/rgmanager/include/resgroup.h b/rgmanager/include/resgroup.h index ded11f4..793ad3b 100644 --- a/rgmanager/include/resgroup.h +++ b/rgmanager/include/resgroup.h @@ -187,6 +187,9 @@ int rg_unlock(struct dlm_lksb *p); /* Return codes */ +#define RG_EPERM -18 /* Permission denied */ +#define RG_ERELO -17 /* Relocation failure; service running + on original node */ #define RG_EEXCL -16 /* Service not runnable due to the fact that it is tagged exclusive and there are no diff --git a/rgmanager/src/clulib/msg_socket.c b/rgmanager/src/clulib/msg_socket.c index c392757..4fc9f88 100644 --- a/rgmanager/src/clulib/msg_socket.c +++ b/rgmanager/src/clulib/msg_socket.c @@ -330,6 +330,8 @@ static int sock_msg_accept(msgctx_t *listenctx, msgctx_t *acceptctx) { errno = EINVAL; + struct ucred cred; + socklen_t credlen = sizeof(cred); if (!listenctx || !acceptctx) return -1; @@ -345,6 +347,18 @@ sock_msg_accept(msgctx_t *listenctx, msgctx_t *acceptctx) if (acceptctx->u.local_info.sockfd < 0) return -1; + memset(&cred, 0, sizeof(cred)); + if (getsockopt(acceptctx->u.local_info.sockfd, SOL_SOCKET, + SO_PEERCRED, (void *)&cred, &credlen) < 0) { + perror("getsockopt"); + cred.uid = (uid_t)-1; + cred.gid = (gid_t)-1; + cred.pid = (pid_t)-1; + } + + memcpy(&acceptctx->u.local_info.cred, &cred, + sizeof(cred)); + set_cloexec(acceptctx->u.local_info.sockfd); acceptctx->flags = (SKF_READ | SKF_WRITE); @@ -372,7 +386,7 @@ sock_msg_listen(int me, void *portp, msgctx_t **listen_ctx) set_cloexec(sock); unlink(RGMGR_SOCK); - om = umask(077); + om = umask(0117); su.sun_family = PF_LOCAL; snprintf(su.sun_path, sizeof(su.sun_path), "%s", path); diff --git a/rgmanager/src/clulib/rg_strings.c b/rgmanager/src/clulib/rg_strings.c index 4d1eccb..47b06e0 100644 --- a/rgmanager/src/clulib/rg_strings.c +++ b/rgmanager/src/clulib/rg_strings.c @@ -26,6 +26,8 @@ struct string_val { const struct string_val rg_error_strings[] = { + { RG_EPERM, "Permission denied" }, + { RG_ERELO, "Failed; service running on original owner" }, { RG_EEXCL, "Service not runnable: cannot run exclusive" }, { RG_EDOMAIN, "Service not runnable: restricted failover domain offline" }, { RG_ESCRIPT, "S/Lang Script Error" }, diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c index 52e38bc..aa78cef 100644 --- a/rgmanager/src/daemons/main.c +++ b/rgmanager/src/daemons/main.c @@ -32,6 +32,8 @@ #include <msgsimple.h> #include <vf.h> #include <lock.h> +#include <sys/socket.h> +#include <message.h> #include <rg_queue.h> #include <malloc.h> #include <cman-private.h> @@ -428,11 +430,17 @@ do_lockreq(msgctx_t *ctx, int req) int dispatch_msg(msgctx_t *ctx, int nodeid, int need_close) { - int ret = 0, sz = -1, nid; + int ret = 0, sz = -1, nid, read_only = 1; char msgbuf[4096]; generic_msg_hdr *msg_hdr = (generic_msg_hdr *)msgbuf; SmMessageSt *msg_sm = (SmMessageSt *)msgbuf; + if (ctx->type == MSG_CLUSTER) { + read_only = 0; + } else if (ctx->u.local_info.cred.uid == 0) { + read_only = 0; + } + memset(msgbuf, 0, sizeof(msgbuf)); /* Peek-a-boo */ @@ -486,6 +494,10 @@ dispatch_msg(msgctx_t *ctx, int nodeid, int need_close) case RG_LOCK: case RG_UNLOCK: + if (read_only) { + msg_send_simple(ctx, RG_FAIL, RG_EPERM, 0); + goto out; + } if (rg_quorate()) do_lockreq(ctx, msg_hdr->gh_command); break; @@ -498,6 +510,10 @@ dispatch_msg(msgctx_t *ctx, int nodeid, int need_close) break; case RG_ACTION_REQUEST: + if (read_only) { + msg_send_simple(ctx, RG_FAIL, RG_EPERM, 0); + goto out; + } if (sz < (int)sizeof(msg_sm)) { clulog(LOG_ERR, @@ -557,6 +573,11 @@ dispatch_msg(msgctx_t *ctx, int nodeid, int need_close) return 0; case RG_EVENT: + if (read_only) { + msg_send_simple(ctx, RG_FAIL, RG_EPERM, 0); + goto out; + } + /* Service event. Run a dependency check */ if (sz < (int)sizeof(msg_sm)) { clulog(LOG_ERR, @@ -579,6 +600,11 @@ dispatch_msg(msgctx_t *ctx, int nodeid, int need_close) break; case RG_EXITING: + if (read_only) { + msg_send_simple(ctx, RG_FAIL, RG_EPERM, 0); + goto out; + } + if (!member_online(msg_hdr->gh_arg1)) break; @@ -589,6 +615,11 @@ dispatch_msg(msgctx_t *ctx, int nodeid, int need_close) break; case VF_MESSAGE: + if (read_only) { + msg_send_simple(ctx, RG_FAIL, RG_EPERM, 0); + goto out; + } + /* Ignore; our VF thread handles these - except for VF_CURRENT XXX (bad design) */ if (msg_hdr->gh_arg1 == VF_CURRENT)

1 0

cluster: RHEL57 - resource-agents: Try force-unmount before fuser for netfs.sh
by Lon Hohberger 08 Apr '11

08 Apr '11

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=d4… Commit: d4dd514faa27f4f7c91205d7525c7f3bc1885aa3 Parent: 8d94f0871db6236ac3caa366560578976477d0eb Author: Lon Hohberger <lhh(a)redhat.com> AuthorDate: Mon Mar 21 20:58:11 2011 -0400 Committer: Lon Hohberger <lhh(a)redhat.com> CommitterDate: Fri Apr 8 09:31:15 2011 -0400 resource-agents: Try force-unmount before fuser for netfs.sh Resolves: rhbz#678494 Signed-off-by: Lon Hohberger <lhh(a)redhat.com> Reviewed-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- rgmanager/src/resources/netfs.sh | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diff --git a/rgmanager/src/resources/netfs.sh b/rgmanager/src/resources/netfs.sh index 1a669d3..837a4c4 100755 --- a/rgmanager/src/resources/netfs.sh +++ b/rgmanager/src/resources/netfs.sh @@ -486,6 +486,7 @@ stopNFSFilesystem() { typeset no_umount="" typeset force_umount="" typeset fstype="" + typeset umount_flag="" # @@ -558,7 +559,7 @@ stopNFSFilesystem() { sync; sync; sync ocf_log info "unmounting $mp" - umount $mp + umount $umount_flag $mp if [ $? -eq 0 ]; then umount_failed= done=$YES @@ -568,6 +569,10 @@ stopNFSFilesystem() { umount_failed=yes if [ "$force_umount" ]; then + if [ -z "$umount_flag" ]; then + umount_flag="-f" + continue + fi if [ $try -eq 1 ]; then fuser -TERM -kvm "$mp" else

1 0

cluster: RHEL57 - cman: Don't increment on LEAVING->MEMBER transition
by Lon Hohberger 08 Apr '11

08 Apr '11

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=8d… Commit: 8d94f0871db6236ac3caa366560578976477d0eb Parent: 1837d87a8548f2244db2634897ad831d90217a44 Author: Lon Hohberger <lhh(a)redhat.com> AuthorDate: Mon Dec 6 15:01:55 2010 -0500 Committer: Lon Hohberger <lhh(a)redhat.com> CommitterDate: Fri Apr 8 09:31:15 2011 -0400 cman: Don't increment on LEAVING->MEMBER transition If a node left the cluster prior using 'cman_tool leave', its internal state is set to NODESTATE_LEAVING. Ordinarily, during the transition from NODESTATE_LEAVING to NODESTATE_DEAD, the member count is decremented. However, it is possible to rejoin the cluster membership from the NODESTATE_LEAVING state. If this occurs, the cluster member count is incremented, but since the node never transitioned from LEAVING->DEAD, it was not previously decremented, causing an ever-increasing node count in 'cman_tool status'. This patch resolves the issue by not incrementing the member count during a LEAVING->MEMBER transition. Resolves: rhbz#649533 Signed-off-by: Lon Hohberger <lhh(a)redhat.com> Reviewed-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> Reviewed-by: Christine Caulfield <ccaulfie(a)redhat.com> --- cman/daemon/commands.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/cman/daemon/commands.c b/cman/daemon/commands.c index 6ffe9e5..6c91d7a 100644 --- a/cman/daemon/commands.c +++ b/cman/daemon/commands.c @@ -1994,9 +1994,12 @@ void add_ais_node(int nodeid, uint64_t incarnation, int total_members) if (node->state == NODESTATE_DEAD || node->state == NODESTATE_LEAVING) { gettimeofday(&node->join_time, NULL); + /* If a node rejoins before it completes a leave, + * we should not increment cluster_members */ + if (node->state != NODESTATE_LEAVING) + cluster_members++; node->incarnation = incarnation; node->state = NODESTATE_MEMBER; - cluster_members++; recalculate_quorum(0, 0); } }

1 0

cluster: RHEL57 - cman: Clarify 'expected' option
by Lon Hohberger 08 Apr '11

08 Apr '11

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=18… Commit: 1837d87a8548f2244db2634897ad831d90217a44 Parent: 8a36ef759c422205d282e2f5543e8a76b00eb847 Author: Shane Bradley <sbradley(a)redhat.com> AuthorDate: Thu Apr 7 09:59:51 2011 -0400 Committer: Lon Hohberger <lhh(a)redhat.com> CommitterDate: Fri Apr 8 09:31:15 2011 -0400 cman: Clarify 'expected' option Resolves: rhbz#688701 Signed-off-by: Lon Hohberger <lhh(a)redhat.com> Reviewed-by: Fabio M. Di Nitto <fdinitto(a)redhat.com> --- cman/man/cman_tool.8 | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/cman/man/cman_tool.8 b/cman/man/cman_tool.8 index 554a316..71e1841 100644 --- a/cman/man/cman_tool.8 +++ b/cman/man/cman_tool.8 @@ -58,6 +58,12 @@ version of "leave force" so only use if if you really know what you are doing. Tells CMAN a new value of expected votes and instructs it to recalculate quorum based on this value. .br +The recalculation takes into account the number of currently +active nodes, so this option should not be used in an attempt to +artificially lower the quorum value in advance of a planned shutdown of +cluster nodes. Instead, the 'cman_tool leave remove' command should be +used (see the 'leave' subcommand above). +.br Use this option if your cluster has lost quorum due to nodes failing and you need to get it running again in a hurry.

1 0

cluster: STABLE31 - rgmanager: Pause during exit if we stopped services
by Lon Hohberger 07 Apr '11

07 Apr '11

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=07… Commit: 079f56ecb56b0d53b44b1871fa8937c22981fc1e Parent: 1d56164f2dea0772188696a8578a4144c8379885 Author: Lon Hohberger <lhh(a)redhat.com> AuthorDate: Thu Apr 7 16:01:58 2011 -0400 Committer: Lon Hohberger <lhh(a)redhat.com> CommitterDate: Thu Apr 7 16:51:00 2011 -0400 rgmanager: Pause during exit if we stopped services A difference between rgmanager 1.9.x and later versions is that they rely on openais/corosync for messaging. This messaging is quite reliable and has proved useful. However, one drawback is that if you very quickly stop rgmanager and corosync/cman, the other nodes in the cluster can not restart services because message traffic is interrupted for the duration of the token timeout. There is no simple solution to this problem. Rgmanager could (in theory) find new placements for services prior to stopping, but this is a large amount of design work; it was never designed to run policies in the exit path. A far simpler idea is to simply give the other nodes time to restart services. NOTE: This solution does not and can not work with central processing mode. Resolves: rhbz#619468 Signed-off-by: Lon Hohberger <lhh(a)redhat.com> --- rgmanager/include/event.h | 1 + rgmanager/include/groups.h | 2 +- rgmanager/src/daemons/groups.c | 12 +++++++++++- rgmanager/src/daemons/main.c | 15 ++++++++++++--- rgmanager/src/daemons/rg_event.c | 7 +++++++ 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/rgmanager/include/event.h b/rgmanager/include/event.h index ad3fe39..9fc9521 100644 --- a/rgmanager/include/event.h +++ b/rgmanager/include/event.h @@ -119,6 +119,7 @@ int slang_process_event(event_table_t *event_table, event_t *ev); /* For distributed events. */ void set_transition_throttling(int nsecs); +int get_transition_throttling(void); /* Simplified service start. */ int service_op_start(char *svcName, int *target_list, int target_list_len, diff --git a/rgmanager/include/groups.h b/rgmanager/include/groups.h index f776a31..f02f0e2 100644 --- a/rgmanager/include/groups.h +++ b/rgmanager/include/groups.h @@ -25,7 +25,7 @@ void kill_resource_groups(void); /* do this op on all resource groups. The handler for the request will sort out whether or not it's a valid request given the state */ -void rg_doall(int request, int block, const char *debugfmt); +int rg_doall(int request, int block, const char *debugfmt); void do_status_checks(void); /* Queue status checks for locally running services */ diff --git a/rgmanager/src/daemons/groups.c b/rgmanager/src/daemons/groups.c index ec8ab63..e9413cd 100644 --- a/rgmanager/src/daemons/groups.c +++ b/rgmanager/src/daemons/groups.c @@ -1276,13 +1276,21 @@ svc_exists(const char *svcname) } -void +/* + * Perform an operation on all resources groups. + * + * Returns the number of requests queued. This value is + * only used during shutdown, where we queue RG_STOP_EXITING + * only for services we have running locally as an optimization. + */ +int rg_doall(int request, int block, const char __attribute__ ((unused)) *debugfmt) { resource_node_t *curr; rg_state_t svcblk; char rg[64]; + int queued = 0; pthread_rwlock_rdlock(&resource_lock); list_do(&_tree, curr) { @@ -1307,6 +1315,7 @@ rg_doall(int request, int block, rt_enqueue_request(rg, request, NULL, 0, 0, 0, 0); + ++queued; } while (!list_done(&_tree, curr)); pthread_rwlock_unlock(&resource_lock); @@ -1316,6 +1325,7 @@ rg_doall(int request, int block, other rgmanagers to complete. */ if (block) rg_wait_threads(); + return queued; } diff --git a/rgmanager/src/daemons/main.c b/rgmanager/src/daemons/main.c index 02cd2dc..a3ab7c4 100644 --- a/rgmanager/src/daemons/main.c +++ b/rgmanager/src/daemons/main.c @@ -29,7 +29,6 @@ void dump_thread_states(FILE *); #endif static int configure_rgmanager(int ccsfd, int debug, int *cluster_timeout); -void set_transition_throttling(int); void flag_shutdown(int sig); @@ -43,6 +42,7 @@ static int signalled = 0; static uint8_t ALIGNED port = RG_PORT; static char *rgmanager_lsname = (char *)"rgmanager"; /* XXX default */ static int status_poll_interval = DEFAULT_CHECK_INTERVAL; +static int stops_queued = 0; static void segfault(int __attribute__ ((unused)) sig) @@ -930,7 +930,7 @@ static void * shutdown_thread(void __attribute__ ((unused)) *arg) { rg_lockall(L_SYS|L_SHUTDOWN); - rg_doall(RG_STOP_EXITING, 1, NULL); + stops_queued = rg_doall(RG_STOP_EXITING, 1, NULL); running = 0; pthread_exit(NULL); @@ -1119,8 +1119,17 @@ out_ls: out: rgm_dbus_release(); - logt_print(LOG_NOTICE, "Shutdown complete, exiting\n"); + logt_print(LOG_DEBUG, "Stopped %d services\n", stops_queued); + logt_print(LOG_NOTICE, "Disconnecting from CMAN\n"); cman_finish(clu); + + if (stops_queued && !central_events_enabled()) { + logt_print(LOG_DEBUG, "Pausing to allow services to " + "start on other node(s)\n"); + sleep(get_transition_throttling() * 3); + } + + logt_print(LOG_NOTICE, "Exiting\n"); close_logging(); /*malloc_stats();*/ diff --git a/rgmanager/src/daemons/rg_event.c b/rgmanager/src/daemons/rg_event.c index d03f4e3..7048bc6 100644 --- a/rgmanager/src/daemons/rg_event.c +++ b/rgmanager/src/daemons/rg_event.c @@ -56,6 +56,13 @@ set_transition_throttling(int nsecs) } +int +get_transition_throttling(void) +{ + return transition_throttling; +} + + void set_central_events(int flag) {

1 0

cluster: STABLE31 - cman: Clarify 'expected' option
by Lon Hohberger 07 Apr '11

07 Apr '11

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=1d… Commit: 1d56164f2dea0772188696a8578a4144c8379885 Parent: 00c3dbbbde98f51ddd99994f88c7a1e339ff6130 Author: Shane Bradley <sbradley(a)redhat.com> AuthorDate: Thu Apr 7 09:59:51 2011 -0400 Committer: Lon Hohberger <lhh(a)redhat.com> CommitterDate: Thu Apr 7 10:04:13 2011 -0400 cman: Clarify 'expected' option Resolves: rhbz#688701 Signed-off-by: Lon Hohberger <lhh(a)redhat.com> --- cman/man/cman_tool.8 | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/cman/man/cman_tool.8 b/cman/man/cman_tool.8 index a5b9f23..2f402a6 100644 --- a/cman/man/cman_tool.8 +++ b/cman/man/cman_tool.8 @@ -58,6 +58,12 @@ version of "leave force" so only use if if you really know what you are doing. Tells CMAN a new value of expected votes and instructs it to recalculate quorum based on this value. .br +The recalculation takes into account the number of currently +active nodes, so this option should not be used in an attempt to +artificially lower the quorum value in advance of a planned shutdown of +cluster nodes. Instead, the 'cman_tool leave remove' command should be +used (see the 'leave' subcommand above). +.br Use this option if your cluster has lost quorum due to nodes failing and you need to get it running again in a hurry.

1 0

cluster: RHEL57 - rgmanager: Fix quotaoff handling
by Lon Hohberger 07 Apr '11

07 Apr '11

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=8a… Commit: 8a36ef759c422205d282e2f5543e8a76b00eb847 Parent: 975a371ae56179be9f0d806bad7482db8541a4fc Author: Ryan Mitchell <rmitchel(a)redhat.com> AuthorDate: Thu Apr 7 09:55:36 2011 -0400 Committer: Lon Hohberger <lhh(a)redhat.com> CommitterDate: Thu Apr 7 09:55:36 2011 -0400 rgmanager: Fix quotaoff handling This is a simple port from the RHEL4 branch performed by Ryan Mitchell. Signed-off-by: Lon Hohberger <lhh(a)redhat.com> --- rgmanager/src/resources/fs.sh | 49 ++++++++++++++++++++++++++++++----------- 1 files changed, 36 insertions(+), 13 deletions(-) diff --git a/rgmanager/src/resources/fs.sh b/rgmanager/src/resources/fs.sh index ce3e983..49912c2 100755 --- a/rgmanager/src/resources/fs.sh +++ b/rgmanager/src/resources/fs.sh @@ -699,19 +699,11 @@ isAlive() # Decide which quota options are enabled and return a string # which we can pass to quotaon # -enable_fs_quotas() +quota_opts() { - declare -i need_check=0 - declare -i rv declare quotaopts="" - declare mopt declare opts=$1 - declare mp=$2 - - if [ -z "`which quotaon`" ]; then - ocf_log err "quotaon not found in $PATH" - return $OCF_ERR_GENERIC - fi + declare mopt for mopt in `echo $opts | sed -e s/,/\ /g`; do case $mopt in @@ -730,6 +722,33 @@ enable_fs_quotas() esac done + echo $quotaopts + return 0 +} + + + +# +# Enable quotas on the mount point if the user requested them +# +enable_fs_quotas() +{ + declare -i need_check=0 + declare -i rv + declare quotaopts="" + declare mopt + declare opts=$1 + declare mp=$2 + + if [ -z "`which quotaon`" ]; then + ocf_log err "quotaon not found in $PATH" + return $OCF_ERR_GENERIC + fi + + quotaopts=$(quota_opts $opts) + + ocf_log info "quotaopts = $quotaopts" + [ -z "$quotaopts" ] && return 0 # Ok, create quota files if they don't exist @@ -991,6 +1010,7 @@ stopFilesystem() { typeset force_umount="" typeset self_fence="" typeset fstype="" + typeset quotaopts="" # @@ -1057,10 +1077,13 @@ stop: Could not match $OCF_RESKEY_device with a real device" ;; $YES) sync; sync; sync - ocf_log info "unmounting $mp" - + quotaopts=$(quota_opts $OCF_RESKEY_options) + if [ -n "$quotaopts" ]; then + ocf_log debug "Turning off quotas for $mp" + quotaoff -$quotaopts $mp &> /dev/null + fi - quotaoff -gu $mp &> /dev/null + ocf_log info "unmounting $mp" umount $mp if [ $? -eq 0 ]; then umount_failed=

1 0

cluster: STABLE31 - Improve rgmanager's exclusive prioritization handling
by Lon Hohberger 07 Apr '11

07 Apr '11

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=00… Commit: 00c3dbbbde98f51ddd99994f88c7a1e339ff6130 Parent: 62cc660f462f8672fe75ebc8e53b184075ff3d5b Author: Lon Hohberger <lhh(a)redhat.com> AuthorDate: Mon Mar 21 21:16:25 2011 -0400 Committer: Lon Hohberger <lhh(a)redhat.com> CommitterDate: Thu Apr 7 09:42:58 2011 -0400 Improve rgmanager's exclusive prioritization handling Resolves: rhbz#680256 Resolves: rhbz#690191 Signed-off-by: Lon Hohberger <lhh(a)redhat.com> --- rgmanager/src/resources/default_event_script.sl | 72 ++++++++++++++++++----- 1 files changed, 56 insertions(+), 16 deletions(-) diff --git a/rgmanager/src/resources/default_event_script.sl b/rgmanager/src/resources/default_event_script.sl index 87c8a1c..759215c 100644 --- a/rgmanager/src/resources/default_event_script.sl +++ b/rgmanager/src/resources/default_event_script.sl @@ -77,7 +77,7 @@ define separate_nodes(node_list) define exclusive_prioritize(svc, node_list) { variable services = service_list(); - variable len, x, y, owner, state, preferred_owner; + variable len, x, y, owner, nowner, state, preferred_owner; variable svc_excl, other_excl; variable nodes_x, nodes_s, nodes_e; @@ -86,7 +86,8 @@ define exclusive_prioritize(svc, node_list) % svc_excl = atoi(service_property(svc, "exclusive")); if (svc_excl == 0) { - return node_list; + notice("Starting ", svc, " on ", node_list); + return service_start(svc, node_list); } (nodes_e, nodes_s, nodes_x) = separate_nodes(node_list); @@ -96,7 +97,11 @@ define exclusive_prioritize(svc, node_list) % If we've got an exclusive service, only allow it to start on % empty nodes. % - return nodes_e; + notice("Starting ", svc, " on ", nodes_e); + nowner = service_start(svc, nodes_e); + if ((nowner > 0) or (nowner != FAIL)) { + return nowner; + } } if (length(nodes_x) == 0) { @@ -105,7 +110,7 @@ define exclusive_prioritize(svc, node_list) % and no empty nodes, the service can not be started % notice("No empty / exclusive nodes available; cannot restart ", svc); - return nodes_x; + return ERR_DOMAIN; } % @@ -149,14 +154,16 @@ define exclusive_prioritize(svc, node_list) () = service_stop(services[x]); % - % Return just the one node. + % Try just the one node. % - node_list = subtract([0], 0); - node_list = union(node_list, owner); - return node_list; + notice("Starting ", svc, " on ", owner); + nowner = service_start(svc, owner); + if ((nowner > 0) or (nowner != FAIL)) { + return nowner; + } } - return node_list; + return ERR_DOMAIN; } @@ -220,13 +227,13 @@ define move_or_start(service, node_list) return ERR_ABORT; } } else { - node_list = exclusive_prioritize(service, node_list); - notice("Starting ", service, " on ", node_list); + return exclusive_prioritize(service, node_list); } if (length(node_list) == 0) { return ERR_DOMAIN; } + notice("Starting ", service, " on ", node_list); return service_start(service, node_list); } @@ -448,7 +455,7 @@ define default_node_event_handler() define default_service_event_handler() { variable services = service_list(); - variable x; + variable x, excl, len; variable depends; variable depend_mode; variable policy; @@ -492,15 +499,16 @@ define default_service_event_handler() return; } - for (x = 0; x < length(services); x++) { + % + % Simplistic dependency handling + % + len = length(services); + for (x = 0; x < len; x++) { if (service_name == services[x]) { % don't do anything to ourself! continue; } - % - % Simplistic dependency handling - % depends = service_property(services[x], "depend"); depend_mode = service_property(services[x], "depend_mode"); @@ -524,6 +532,38 @@ define default_service_event_handler() ()=service_stop(services[x]); } } + + % + % Try to restart exclusive service which might have been recently + % stopped in order to make room for other exclusive services. + % + % Note that as a side effect, exclusive services (>=2) can't be + % stopped with clusvcadm -s; they will just pop right back - you + % must disable them if want them to stay stopped. + % + % This code has a side effect of brute-forcing the lowest-priority + % exclusive service offline in a cascaded fashion. + % + for (x = 0; x < len; x++) { + if (service_name == services[x]) { + % don't do anything to ourself! + continue; + } + + excl = atoi(service_property(services[x], "exclusive")); + % non-exclusive or highest-prio (1) shouldn't get here + if ((excl == 0) or (excl == 1)) { + continue; + } + + (,,, owner, state) = service_status(services[x]); + if (state == "stopped") { + info("Restarting stopped exclusive priority ", + excl, " service ", services[x]); + nodes = allowed_nodes(services[x]); + ()=move_or_start(services[x], nodes); + } + } } define default_config_event_handler()

1 0

cluster: RHEL6 - Fix bug 683213 - mirror dev failure in HA LVM can cause service failure
by Jonathan Brassow 06 Apr '11

06 Apr '11

Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=fe… Commit: fe9bef6e01c83e87bee845caa6d763deb33da431 Parent: e51fbc090cb3b7a343aae3ff7d1ffcd94f7fbb5b Author: Jonathan Brassow <jbrassow(a)f14.(none)> AuthorDate: Wed Apr 6 11:23:29 2011 -0500 Committer: Jonathan Brassow <jbrassow(a)redhat.com> CommitterDate: Wed Apr 6 11:36:11 2011 -0500 Fix bug 683213 - mirror dev failure in HA LVM can cause service failure A new option was added to 'vgreduce', --force. This argument is now required. Additionally, repair is attempted when deactivating a device as well as when trying to activate. This is because a shutdown of a service can race against a device failure. Adding the repair option to the shutdown process adds resilience. --- rgmanager/src/resources/lvm_by_lv.sh | 63 +++++++++++++++------------------ rgmanager/src/resources/lvm_by_vg.sh | 4 +- 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/rgmanager/src/resources/lvm_by_lv.sh b/rgmanager/src/resources/lvm_by_lv.sh index 759062a..4aead58 100644 --- a/rgmanager/src/resources/lvm_by_lv.sh +++ b/rgmanager/src/resources/lvm_by_lv.sh @@ -289,45 +289,40 @@ lv_activate() if ! lv_activate_and_tag $1 $my_name $lv_path; then ocf_log err "Failed to $1 $lv_path" - if [ "$1" == "start" ]; then - ocf_log notice "Attempting cleanup of $OCF_RESKEY_vg_name" - - if vgreduce --removemissing --config \ - "activation { volume_list = \"$OCF_RESKEY_vg_name\" }" \ - $OCF_RESKEY_vg_name; then - ocf_log notice "$OCF_RESKEY_vg_name now consistent" - owner=`lvs -o tags --noheadings $lv_path` - if [ ! -z $owner ] && [ $owner != $my_name ]; then - if is_node_member_clustat $owner ; then - ocf_log err "$owner owns $lv_path unable to $1" - return $OCF_ERR_GENERIC - fi - ocf_log notice "Owner of $lv_path is not in the cluster" - ocf_log notice "Stealing $lv_path" - - lvchange --deltag $owner $lv_path - if [ $? -ne 0 ]; then - ocf_log err "Failed to steal $lv_path from $owner" - return $OCF_ERR_GENERIC - fi - - # Warning --deltag doesn't always result in failure - if [ ! -z `lvs -o tags --noheadings $lv_path` ]; then - ocf_log err "Failed to steal $lv_path from $owner." - return $OCF_ERR_GENERIC - fi + ocf_log notice "Attempting cleanup of $OCF_RESKEY_vg_name" + + if vgreduce --removemissing --force --config \ + "activation { volume_list = \"$OCF_RESKEY_vg_name\" }" \ + $OCF_RESKEY_vg_name; then + ocf_log notice "$OCF_RESKEY_vg_name now consistent" + owner=`lvs -o tags --noheadings $lv_path` + if [ ! -z $owner ] && [ $owner != $my_name ]; then + if is_node_member_clustat $owner ; then + ocf_log err "$owner owns $lv_path unable to $1" + return $OCF_ERR_GENERIC fi + ocf_log notice "Owner of $lv_path is not in the cluster" + ocf_log notice "Stealing $lv_path" - if ! lv_activate_and_tag $1 $my_name $lv_path; then - ocf_log err "Failed second attempt to $1 $lv_path" + lvchange --deltag $owner $lv_path + if [ $? -ne 0 ]; then + ocf_log err "Failed to steal $lv_path from $owner" return $OCF_ERR_GENERIC - else - ocf_log notice "Second attempt to $1 $lv_path successful" - return $OCF_SUCCESS fi - else - ocf_log err "Failed to make $OCF_RESKEY_vg_name consistent" + + # Warning --deltag doesn't always result in failure + if [ ! -z `lvs -o tags --noheadings $lv_path` ]; then + ocf_log err "Failed to steal $lv_path from $owner." + return $OCF_ERR_GENERIC + fi + fi + + if ! lv_activate_and_tag $1 $my_name $lv_path; then + ocf_log err "Failed second attempt to $1 $lv_path" return $OCF_ERR_GENERIC + else + ocf_log notice "Second attempt to $1 $lv_path successful" + return $OCF_SUCCESS fi else ocf_log err "Failed to $1 $lv_path" diff --git a/rgmanager/src/resources/lvm_by_vg.sh b/rgmanager/src/resources/lvm_by_vg.sh index f0ca671..2cce7c9 100644 --- a/rgmanager/src/resources/lvm_by_vg.sh +++ b/rgmanager/src/resources/lvm_by_vg.sh @@ -160,7 +160,7 @@ function vg_start_clustered ocf_log err "Failed to activate volume group, $OCF_RESKEY_vg_name" ocf_log notice "Attempting cleanup of $OCF_RESKEY_vg_name" - if ! vgreduce --removemissing $OCF_RESKEY_vg_name; then + if ! vgreduce --removemissing --force $OCF_RESKEY_vg_name; then ocf_log err "Failed to make $OCF_RESKEY_vg_name consistent" return $OCF_ERR_GENERIC fi @@ -238,7 +238,7 @@ function vg_start_single ocf_log err "Failed to activate volume group, $OCF_RESKEY_vg_name" ocf_log notice "Attempting cleanup of $OCF_RESKEY_vg_name" - if ! vgreduce --removemissing --config \ + if ! vgreduce --removemissing --force --config \ "activation { volume_list = \"$OCF_RESKEY_vg_name\" }" \ $OCF_RESKEY_vg_name; then

1 0

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

cluster-commits April 2011