cluster-commits January 2016

cluster-commits@lists.stg.fedorahosted.org

3 participants
14 discussions

gfs2-utils: master - gfs2-utils: Add the glocktop utility
by Andrew Price 28 Jan '16

28 Jan '16

Gitweb: http://git.fedorahosted.org/git/?p=gfs2-utils.git;a=commitdiff;h=821f728129… Commit: 821f7281291fa272ab8293c15a8e1882f538cf08 Parent: 9f44cc369967ad17843177537710c00abf35c904 Author: Andrew Price <anprice(a)redhat.com> AuthorDate: Thu Jan 28 10:13:51 2016 +0000 Committer: Andrew Price <anprice(a)redhat.com> CommitterDate: Thu Jan 28 10:13:51 2016 +0000 gfs2-utils: Add the glocktop utility This adds Bob Peterson's glocktop program and glocktop(8) man page. This tool can be used for debugging glock-related issues and to analyze locking related performance issues. Changes from the original imported glocktop.c are: - Linked against libgfs2 and de-duplicated functions provided by libgfs2 - Fixed build warnings (mainly -Wformat stuff) - Fixed issues spotted by coverity Signed-off-by: Andrew Price <anprice(a)redhat.com> --- .gitignore | 1 + configure.ac | 1 + gfs2/Makefile.am | 1 + gfs2/glocktop/Makefile.am | 23 + gfs2/glocktop/glocktop.c | 1874 +++++++++++++++++++++++++++++++++++++++++++++ gfs2/man/Makefile.am | 3 +- gfs2/man/glocktop.8 | 281 +++++++ 7 files changed, 2183 insertions(+), 1 deletions(-) diff --git a/.gitignore b/.gitignore index 60ea971..bc8236e 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ gfs2/libgfs2/parser.h gfs2/libgfs2/lexer.c gfs2/libgfs2/lexer.h gfs2/fsck/fsck.gfs2 +gfs2/glocktop/glocktop gfs2/mkfs/mkfs.gfs2 gfs2/mkfs/gfs2_grow gfs2/mkfs/gfs2_jadd diff --git a/configure.ac b/configure.ac index a5418aa..130cb01 100644 --- a/configure.ac +++ b/configure.ac @@ -204,6 +204,7 @@ AC_CONFIG_FILES([Makefile gfs2/tune/Makefile gfs2/man/Makefile gfs2/scripts/Makefile + gfs2/glocktop/Makefile doc/Makefile tests/Makefile tests/atlocal diff --git a/gfs2/Makefile.am b/gfs2/Makefile.am index 028e840..5a2eefd 100644 --- a/gfs2/Makefile.am +++ b/gfs2/Makefile.am @@ -9,4 +9,5 @@ SUBDIRS = \ mkfs \ man \ tune \ + glocktop \ scripts diff --git a/gfs2/glocktop/Makefile.am b/gfs2/glocktop/Makefile.am new file mode 100644 index 0000000..a43c519 --- /dev/null +++ b/gfs2/glocktop/Makefile.am @@ -0,0 +1,23 @@ +MAINTAINERCLEANFILES = Makefile.in + +sbin_PROGRAMS = \ + glocktop + +glocktop_SOURCES = \ + glocktop.c + +glocktop_CFLAGS = \ + $(ncurses_CFLAGS) + +glocktop_LDFLAGS = \ + $(ncurses_LIBS) + +glocktop_CPPFLAGS = \ + -D_FILE_OFFSET_BITS=64 \ + -D_LARGEFILE64_SOURCE \ + -D_GNU_SOURCE \ + -I$(top_srcdir)/gfs2/include \ + -I$(top_srcdir)/gfs2/libgfs2 + +glocktop_LDADD = \ + $(top_builddir)/gfs2/libgfs2/libgfs2.la diff --git a/gfs2/glocktop/glocktop.c b/gfs2/glocktop/glocktop.c new file mode 100644 index 0000000..43c3c3f --- /dev/null +++ b/gfs2/glocktop/glocktop.c @@ -0,0 +1,1874 @@ +#include "clusterautoconfig.h" +/** + * glocktop.c - list/print the top GFS2 glock waiters + */ +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <time.h> +#include <limits.h> +#include <sys/wait.h> +#include <sys/mount.h> +#include <dirent.h> +#include <curses.h> +#include <term.h> +#include <sys/time.h> +#include <string.h> +#include <stdint.h> +#include <inttypes.h> +#include <ctype.h> +#include <errno.h> +#include <libgfs2.h> + +#define MAX_GLOCKS 20 +#define MAX_LINES 6000 +#define MAX_MOUNT_POINTS 100 +#define MAX_FILES 512 +#define MAX_CALLTRACE_LINES 4 +#define TITLE1 "glocktop - GFS2 glock monitor" +#define TITLE2 "Press <ctrl-c> or <escape> to exit" + +#define COLOR_TITLE 1 +#define COLOR_NORMAL 2 +#define COLOR_INVERSE 3 +#define COLOR_SPECIAL 4 +#define COLOR_HIGHLIGHT 5 +#define COLOR_OFFSETS 6 +#define COLOR_CONTENTS 7 +#define COLOR_HELD 8 + +/* init_pair(COLOR_TITLE, COLOR_BLACK, COLOR_CYAN); + init_pair(COLOR_INVERSE, COLOR_BLACK, COLOR_WHITE); + init_pair(COLOR_NORMAL, COLOR_WHITE, COLOR_BLACK); + init_pair(COLOR_SPECIAL, COLOR_MAGENTA, COLOR_WHITE); + init_pair(COLOR_HIGHLIGHT, COLOR_WHITE, COLOR_BLUE); + init_pair(COLOR_OFFSETS, COLOR_CYAN, COLOR_WHITE); + init_pair(COLOR_CONTENTS, COLOR_BLUE, COLOR_WHITE); + init_pair(COLOR_HELD, COLOR_CYAN, COLOR_BLACK); +*/ + +#define STR_BLACK "[\033[0;30m]" +#define STR_RED "[\033[0;31m]" +#define STR_GREEN "[\033[0;32m]" +#define STR_YELLOW "[\033[0;33m]" +#define STR_BLUE "[\033[0;34m]" +#define STR_MAGENTA "[\033[0;35m]" +#define STR_CYAN "[\033[0;36m]" +#define STR_WHITE "[\033[0;37m]" + +#define BOLD_WHITE "[\033[1;37m]" + +#define BKG_CYAN "[\033[46m]" +#define BKG_WHITE "[\033[47m]" +#define BKG_BLUE "[\033[44m]" + +#define REFRESH_TIME 30 +#define COLORS_TITLE \ + do { \ + if (termlines) \ + attrset(COLOR_PAIR(COLOR_TITLE)); \ + else \ + printf(BKG_CYAN); \ + } while (0) +#define COLORS_NORMAL_BOLD \ + do { \ + if (termlines) { \ + attrset(COLOR_PAIR(COLOR_NORMAL)); \ + attron(A_BOLD); \ + } else { \ + printf(BOLD_WHITE); \ + } \ + } while (0) +#define COLORS_NORMAL \ + do { \ + if (termlines) { \ + attrset(COLOR_PAIR(COLOR_NORMAL)); \ + } else { \ + printf(STR_WHITE); \ + } \ + } while (0) +#define COLORS_INVERSE_BOLD \ + do { \ + if (termlines) { \ + attrset(COLOR_PAIR(COLOR_INVERSE)); \ + attron(A_BOLD); \ + } else { \ + printf(BKG_WHITE); \ + } \ + } while (0) +#define COLORS_INVERSE \ + do { \ + if (termlines) { \ + attrset(COLOR_PAIR(COLOR_INVERSE)); \ + } else { \ + printf(BKG_WHITE); \ + } \ + } while (0) +#define COLORS_HELD \ + do { \ + if (termlines) { \ + attrset(COLOR_PAIR(COLOR_HELD)); \ + } else { \ + printf(STR_CYAN); \ + } \ + } while (0) +#define COLORS_HIGHLIGHT \ + do { \ + if (termlines) { \ + attrset(COLOR_PAIR(COLOR_HIGHLIGHT)); \ + } else { \ + printf(BKG_BLUE); \ + } \ + } while (0) +#define DLM_DIRTBL "/sys/kernel/config/dlm/cluster/dirtbl_size" +#define DLM_RSBTBL "/sys/kernel/config/dlm/cluster/rsbtbl_size" +#define DLM_LKBTBL "/sys/kernel/config/dlm/cluster/lkbtbl_size" + +#define GFS2_MAX_META_HEIGHT 10 + +#define DETAILS 0x00000001 +#define FRIENDLY 0x00000002 + +enum summary_types { + all = 0, + locked = 1, + held_ex = 2, + held_sh = 3, + held_df = 4, + has_waiter = 5, + tot_waiters = 6, + stypes = 7, +}; + +char debugfs[PATH_MAX]; +int termcols = 80, termlines = 30, done = 0; +unsigned glocks = 0; +int refresh_time = REFRESH_TIME; +const char *termtype; +WINDOW *wind; +int bufsize = 4 * 1024 * 1024; +char *glock[MAX_GLOCKS]; +int iterations = 0, show_reservations = 0, iters_done = 0; +char devices[MAX_MOUNT_POINTS][80]; +char mount_points[MAX_MOUNT_POINTS][80]; +int fs_fd[MAX_MOUNT_POINTS]; +int mounted = 0; +char dlmwlines[100][96]; /* waiters lines */ +char dlmglines[MAX_LINES][97]; /* granted lines */ +char contended_filenames[MAX_FILES][PATH_MAX]; +unsigned long long contended_blocks[MAX_FILES]; +int contended_count = 0; +int line = 0; +const char *prog_name; +char dlm_dirtbl_size[32], dlm_rsbtbl_size[32], dlm_lkbtbl_size[32]; +int bsize = 0; +struct gfs2_sb sd_sb[MAX_MOUNT_POINTS]; +int sd_diptrs = 0, sd_inptrs = 0; +uint64_t sd_heightsize[GFS2_MAX_META_HEIGHT]; +uint64_t sd_jheightsize[GFS2_MAX_META_HEIGHT]; +int sd_max_height, sd_max_jheight; +char print_dlm_grants = 1; +char *gbuf = NULL; /* glocks buffer */ +char *gpos = NULL; +char *gnextpos = NULL; +int gmaxpos = 0; + +char *dbuf = NULL; /* dlm locks buffer */ +char *dpos = NULL; +char *dnextpos = NULL; +int dmaxpos = 0; +char hostname[256]; + +/* + * init_colors + */ +static void init_colors(void) +{ + init_pair(COLOR_TITLE, COLOR_BLACK, COLOR_CYAN); + init_pair(COLOR_INVERSE, COLOR_BLACK, COLOR_WHITE); + init_pair(COLOR_NORMAL, COLOR_WHITE, COLOR_BLACK); + init_pair(COLOR_SPECIAL, COLOR_MAGENTA, COLOR_WHITE); + init_pair(COLOR_HIGHLIGHT, COLOR_WHITE, COLOR_BLUE); + init_pair(COLOR_OFFSETS, COLOR_CYAN, COLOR_WHITE); + init_pair(COLOR_CONTENTS, COLOR_BLUE, COLOR_WHITE); + init_pair(COLOR_HELD, COLOR_CYAN, COLOR_BLACK); +} + +/* + * UpdateSize - screen size changed, so update it + */ +static void UpdateSize(int sig) +{ + static char term_buffer[2048]; + int rc; + + if (termlines) { + termlines = 30; + termtype = getenv("TERM"); + if (termtype == NULL) + return; + rc=tgetent(term_buffer,termtype); + if (rc >= 0) { + termlines = tgetnum((char *)"li"); + if (termlines < 10) + termlines = 30; + termcols = tgetnum((char *)"co"); + if (termcols < 80) + termcols = 80; + } else + perror("Error: tgetent failed."); + termlines--; /* last line is number of lines -1 */ + } + signal(SIGWINCH, UpdateSize); +} + +static void read_superblock(int fd, int mntpt) +{ + struct gfs2_sbd sbd = { .device_fd = fd, .bsize = GFS2_BASIC_BLOCK }; + struct gfs2_buffer_head *bh; + int x; + uint64_t space = 0; + + ioctl(fd, BLKFLSBUF, 0); + bh = bread(&sbd, GFS2_SB_ADDR); + gfs2_sb_in(&sd_sb[mntpt], bh); /* parse it out into the sb structure */ + bsize = sd_sb[mntpt].sb_bsize; + if (!bsize) + bsize = 4096; + sd_inptrs = (bsize - sizeof(struct gfs2_meta_header)) / + sizeof(uint64_t); + sd_diptrs = (bsize - sizeof(struct gfs2_dinode)) / + sizeof(uint64_t); + sd_heightsize[0] = bsize - sizeof(struct gfs2_dinode); + sd_heightsize[1] = bsize * sd_diptrs; + for (x = 2; ; x++) { + space = sd_heightsize[x - 1] * sd_inptrs; + if (space / sd_inptrs != sd_heightsize[x - 1] || + space % sd_inptrs != 0) + break; + sd_heightsize[x] = space; + } + sd_jheightsize[0] = bsize - sizeof(struct gfs2_dinode); + sd_jheightsize[1] = (bsize - sizeof(struct gfs2_meta_header)) * + sd_diptrs; + for (x = 2; ; x++){ + space = sd_jheightsize[x - 1] * sd_inptrs; + if (space / sd_inptrs != sd_jheightsize[x - 1] || + space % sd_inptrs != 0) + break; + sd_jheightsize[x] = space; + } + sd_max_jheight = x; +} + +static int parse_mounts(void) +{ + char str[PATH_MAX], dev[PATH_MAX], mnt[PATH_MAX], mtype[PATH_MAX]; + char opts[PATH_MAX]; + FILE *fp; + + memset(debugfs, 0, sizeof(debugfs)); + memset(mount_points, 0, sizeof(mount_points)); + memset(devices, 0, sizeof(devices)); + + fp = fopen("/proc/mounts", "rt"); + if (fp == NULL) { + perror("/proc/mounts"); + return 1; + } + while (fgets(str, sizeof(str) - 1, fp)) { + sscanf(str, "%s %s %s %s", dev, mnt, mtype, opts); + if (!strcmp(mtype, "debugfs")) { + strcpy(debugfs, mnt); + continue; + } + if (strcmp(mtype, "gfs2")) /* if not gfs2 */ + continue; + + strncpy(mount_points[mounted], mnt, 79); + mount_points[mounted][79] = '\0'; + strncpy(devices[mounted], dev, 79); + devices[mounted][79] = '\0'; + + /* Now find out the mount point's file system name */ + fs_fd[mounted] = open(dev, O_RDONLY); + if (fs_fd[mounted]) + read_superblock(fs_fd[mounted], mounted); + mounted++; + } + if (debugfs[0] == '\0') { + if (mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)){ + fprintf(stderr, "Unable to mount debugfs.\n"); + fprintf(stderr, "Please mount it manually.\n"); + exit(-1); + } + strcpy(debugfs, "/sys/kernel/debug"); + } + fclose(fp); + return 0; +} + +/* + * display_title_lines + */ +static void display_title_lines(void) +{ + if (termlines) { + clear(); /* don't use Erase */ + COLORS_TITLE; + attron(A_BOLD); + move(0, 0); + printw("%-80s", TITLE1); + move(termlines, 0); + printw("%-79s", TITLE2); + COLORS_NORMAL_BOLD; + move(1, 0); + } else { + printf("\n"); + } + line = 1; +} + +/* + * bobgets - get a string + * returns: 1 if user exited by hitting enter + * 0 if user exited by hitting escape + */ +static int bobgets(char string[], int x, int y, int sz, int *ch) +{ + int finished,runningy,rc; + + if (!termlines) + return 0; + move(x,y); + finished=FALSE; + COLORS_INVERSE_BOLD; + move(x,y); + addstr(string); + move(x,y); + curs_set(2); + refresh(); + runningy=y; + rc=0; + while (!finished) { + *ch = getch(); + + if(*ch < 0x0100 && isprint(*ch)) { + char *p=string+strlen(string); // end of the string + + *(p+1)='\0'; + string[runningy-y]=*ch; + runningy++; + move(x,y); + addstr(string); + if (runningy-y >= sz) { + rc=1; + *ch = KEY_RIGHT; + finished = TRUE; + } + } + else { + // special character, is it one we recognize? + switch(*ch) + { + case(KEY_ENTER): + case('\n'): + case('\r'): + rc=1; + finished=TRUE; + string[runningy-y] = '\0'; + break; + case(KEY_CANCEL): + case(0x01B): + rc=0; + finished=TRUE; + break; + case(KEY_DC): + case(0x07F): + if (runningy>=y) { + char *p; + p = &string[runningy - y]; + while (*p) { + *p = *(p + 1); + p++; + } + *p = '\0'; + runningy--; + // remove the character from the string + move(x,y); + addstr(string); + COLORS_NORMAL_BOLD; + addstr(" "); + COLORS_INVERSE_BOLD; + runningy++; + } + break; + case(KEY_BACKSPACE): + if (runningy>y) { + char *p; + + p = &string[runningy - y - 1]; + while (*p) { + *p = *(p + 1); + p++; + } + *p='\0'; + runningy--; + // remove the character from the string + move(x,y); + addstr(string); + COLORS_NORMAL_BOLD; + addstr(" "); + COLORS_INVERSE_BOLD; + } + break; + default: + move(0,70); + printw("%08x", *ch); + // ignore all other characters + break; + } // end switch on non-printable character + } // end non-printable character + move(line, runningy); + refresh(); + } // while !finished + if (sz>0) + string[sz]='\0'; + COLORS_NORMAL_BOLD; + return rc; +}/* bobgets */ + +static char *bufgets(int fd, char *bigbuf, char **nextpos, char **pos, + int *maxpos) +{ + if (*nextpos == NULL) { + *maxpos = read(fd, bigbuf, bufsize - 1); + bigbuf[bufsize - 1] = '\0'; + if (*maxpos == 0) + return NULL; + *pos = bigbuf; + } else + *pos = *nextpos; + + *nextpos = memchr(*pos, '\n', (bigbuf + *maxpos) - *pos); + while (*nextpos && (**nextpos == '\n' || **nextpos == '\r') && + *nextpos < bigbuf + (bufsize - 1)) { + **nextpos = '\0'; + (*nextpos)++; + } + if (*nextpos >= bigbuf + *maxpos) + *nextpos = NULL; + return *pos; +} + +static char *glock_number(const char *str) +{ + const char *glockid; + char *p; + static char id[32]; + + glockid = strchr(str, '/'); + if (glockid == NULL) + return NULL; + glockid++; + strncpy(id, glockid, sizeof(id)); + id[31] = '\0'; + p = strchr(id, ' '); + if (p) + *p = '\0'; + return id; +} + +static int this_glock_requested(const char *str) +{ + const char *glockid; + int i; + + if (!glocks) + return 0; + + glockid = glock_number(str); + if (glockid == NULL) + return 0; + for (i = 0; i < glocks; i++) + if (!strcmp(glockid, glock[i])) + return 1; + return 0; +} + +static int is_iopen(const char *str) +{ + char *p; + + p = strchr(str, '/'); + if (p == NULL) + return 0; + p--; + if (*p == '5') + return 1; + return 0; +} + +static int this_lkb_requested(const char *str) +{ + int i; + + if (!glocks) + return 1; + + for (i = 0; i < glocks; i++) { + if (strstr(str, glock[i])) + return 1; + } + return 0; +} + +static void eol(int col) /* end of line */ +{ + if (termlines) { + line++; + move(line, col); + } else { + printf("\n"); + for (; col > 0; col--) + printf(" "); + } +} + +void print_it(const char *label, const char *fmt, const char *fmt2, ...) +{ + va_list args; + char tmp_string[128]; + + if (!termlines || line < termlines) { + va_start(args, fmt2); + vsnprintf(tmp_string, 127, fmt, args); + tmp_string[127] = '\0'; + + if (termlines) { + printw("%s", tmp_string); + refresh(); + } else { + printf("%s", tmp_string); + fflush(stdout); + } + } + va_end(args); +} + +static void display_filename(int fd, unsigned long long block, + unsigned long long dirarray[256], int subdepth) +{ + int i, subs; + char *mntpt = NULL; + char blk[32]; + DIR *dir = NULL; + struct dirent *dent; + + for (i = 0; i < mounted; i++) { + if (fd == fs_fd[i]) { + mntpt = mount_points[i]; + break; + } + } + if (i == mounted) + return; + for (i = 0; i < contended_count; i++) { + if (contended_blocks[i] == block) { + break; + } + } + sprintf(blk, "%lld", block); + if (i >= contended_count) { + memset(contended_filenames[i], 0, PATH_MAX); + strcat(contended_filenames[i], mntpt); + for (subs = subdepth - 2; subs >= 0; subs--) { + dir = opendir(contended_filenames[i]); + while ((dent = readdir(dir))) { + if (dent->d_ino == dirarray[subs]) { + strcat(contended_filenames[i], "/"); + strcat(contended_filenames[i], + dent->d_name); + break; + } + } + closedir(dir); + } + } + + print_it(NULL, "%s", NULL, contended_filenames[i]); + eol(0); +} + +static const char *show_inode(const char *id, int fd, unsigned long long block) +{ + struct gfs2_inode *ip; + const char *inode_type = NULL; + struct gfs2_sbd sbd = { .device_fd = fd, .bsize = bsize }; + + ip = lgfs2_inode_read(&sbd, block); + if (S_ISDIR(ip->i_di.di_mode)) { + struct gfs2_inode *parent; + unsigned long long dirarray[256]; + int subdepth = 0, error; + + inode_type = "directory "; + dirarray[0] = block; + subdepth++; + /* Backtrack the directory to its source */ + while (1) { + error = gfs2_lookupi(ip, "..", 2, &parent); + if (error) + break; + /* Stop at the root inode */ + if (ip->i_di.di_num.no_addr == + parent->i_di.di_num.no_addr) + break; + inode_put(&ip); + ip = parent; + dirarray[subdepth++] = parent->i_di.di_num.no_addr; + } + display_filename(fd, block, dirarray, subdepth); + } else if (S_ISREG(ip->i_di.di_mode)) { + inode_type = "file "; + } else if (S_ISLNK(ip->i_di.di_mode)) { + inode_type = "link "; + } else if (S_ISCHR(ip->i_di.di_mode)) { + inode_type = "char device "; + } else if (S_ISBLK(ip->i_di.di_mode)) { + inode_type = "block device "; + } else if (S_ISFIFO(ip->i_di.di_mode)) { + inode_type = "fifo "; + } else if (S_ISSOCK(ip->i_di.di_mode)) { + inode_type = "socket "; + } else + inode_type = "file? "; + inode_put(&ip); + return inode_type; +} + +static const char *show_details(const char *id, const char *fsname, int btype, + int trace_dir_path) +{ + int mnt_num; + unsigned long long block = 0; + const char *blk_type = NULL; + FILE *dlmf; + + /* Figure out which mount point corresponds to this debugfs id */ + for (mnt_num = 0; mnt_num < mounted; mnt_num++) { + char *p; + + p = strchr(sd_sb[mnt_num].sb_locktable, ':'); + if (!p) + continue; + p++; + if (!strcmp(p, fsname)) + break; + } + memset(dlm_dirtbl_size, 0, sizeof(dlm_dirtbl_size)); + memset(dlm_rsbtbl_size, 0, sizeof(dlm_rsbtbl_size)); + memset(dlm_lkbtbl_size, 0, sizeof(dlm_lkbtbl_size)); + if (!strcmp(sd_sb[mnt_num].sb_lockproto, "lock_dlm")) { + char *p; + + dlmf = fopen(DLM_DIRTBL, "rt"); + if (dlmf) { + fgets(dlm_dirtbl_size, sizeof(dlm_dirtbl_size), dlmf); + p = strchr(dlm_dirtbl_size, '\n'); + if (p) + *p = '\0'; + fclose(dlmf); + } else { + strcpy(dlm_dirtbl_size, " "); + } + dlmf = fopen(DLM_RSBTBL, "rt"); + if (dlmf) { + fgets(dlm_rsbtbl_size, sizeof(dlm_rsbtbl_size), dlmf); + p = strchr(dlm_rsbtbl_size, '\n'); + if (p) + *p = '\0'; + fclose(dlmf); + } else { + strcpy(dlm_rsbtbl_size, " "); + } + dlmf = fopen(DLM_LKBTBL, "rt"); + if (dlmf) { + fgets(dlm_lkbtbl_size, sizeof(dlm_lkbtbl_size), dlmf); + p = strchr(dlm_lkbtbl_size, '\n'); + if (p) + *p = '\0'; + fclose(dlmf); + } else { + strcpy(dlm_lkbtbl_size, " "); + } + } else { + strcpy(dlm_dirtbl_size, "nolock"); + strcpy(dlm_lkbtbl_size, "nolock"); + strcpy(dlm_lkbtbl_size, "nolock"); + } + + if (mnt_num >= mounted) /* can't find the right superblock */ + return "unknown"; + + /* Read the inode in so we can see its type. */ + sscanf(id, "%llx", &block); + if (block) { + if (btype == 2) + if (trace_dir_path) + blk_type = show_inode(id, fs_fd[mnt_num], + block); + else + blk_type = ""; + else + blk_type = ""; + } + return blk_type; +} + +static int is_dlm_waiting(int dlmwaiters, int locktype, char *id) +{ + int i; + int dlmid, wait_type, nodeid, type; + char locknum[32]; + + for (i = 0; i < dlmwaiters && i < 100; i++) { + sscanf(dlmwlines[i], "%x %d %d %d %s", + &dlmid, &wait_type, &nodeid, &type, locknum); + if ((type == locktype) && (!strcmp(locknum, id))) + return 1; + } + return 0; +} + +static const char *friendly_state(const char *glock_line, const char *search) +{ + const char *p; + + p = strstr(glock_line, search); + + if (p == NULL) + return "Dazed"; + + p += 2; + if (*p == 'E') + return "Exclusive"; + else if (*p == 'S') + return "Shared"; + else if (*p == 'U') + return "Unlocked"; + else if (*p == 'D') + return "Deferred"; + else + return "Confused"; +} + +static const char *friendly_gflags(const char *glock_line) +{ + static char flagout[PATH_MAX]; + const char *p; + + memset(flagout, 0, sizeof(flagout)); + + p = strstr(glock_line, "f:"); + if (!p) + return " "; + p += 2; + strcpy(flagout, "["); + while (*p != ' ') { + switch (*p) { + case 'l': + /*strcat(flagout, "Locked");*/ + break; + case 'D': + strcat(flagout, "Demoting"); + break; + case 'd': + strcat(flagout, "Demote pending"); + break; + case 'p': + strcat(flagout, "Demote in progress"); + break; + case 'y': + strcat(flagout, "Dirty"); + break; + case 'f': + strcat(flagout, "Flush"); + break; + case 'i': + strcat(flagout, "Invalidating"); + break; + case 'r': + strcat(flagout, "Reply pending"); + break; + case 'I': + /*strcat(flagout, "Initial");*/ + break; + case 'F': + strcat(flagout, "Frozen"); + break; + case 'q': + strcat(flagout, "Queued"); + break; + case 'L': + strcat(flagout, "LRU"); + break; + case 'o': + /*strcat(flagout, "Object present");*/ + break; + case 'b': + strcat(flagout, "Blocking"); + break; + default: + strcat(flagout, "Unknown"); + break; + } + if ((strlen(flagout)) > 1 && (!strchr(" lIo", *(p + 1)))) + strcat(flagout, ", "); + p++; + } + strcat(flagout, "]"); + return flagout; +} + +static const char *friendly_glock(const char *glock_line, char prefix) +{ + static char gline[PATH_MAX]; + + if (prefix == 'W') + sprintf(gline, "Is:%s, Want:%s %s", + friendly_state(glock_line, "s:"), + friendly_state(glock_line, "t:"), + friendly_gflags(glock_line)); + else + sprintf(gline, "Held:%s %s", + friendly_state(glock_line, "s:"), + friendly_gflags(glock_line)); + return gline; +} + +static const char *dlm_grtype(int grmode) +{ + const char *dlm_types[8] = {"NL", "CR", "CW", "PR", "PW", "EX", + "NA", "NA"}; + + if (grmode < 0) + return "-1"; + return dlm_types[grmode & 0x07]; +} + +static const char *dlm_status(int status) +{ + const char *dlm_statuses[4] = {"Unknown", "Waiting", "Granted", + "Converting"}; + if (status < 0) + return "unknown"; + return dlm_statuses[status & 0x03]; +} + +static const char *dlm_nodeid(int lkbnodeid) +{ + static char nodeid[16]; + + if (lkbnodeid == 0) + return "this node"; + sprintf(nodeid, "node %d", lkbnodeid); + return nodeid; +} + +static const char *getprocname(int ownpid) +{ + char fn[1024]; + static char str[80]; + const char *procname; + FILE *fp; + + sprintf(fn, "/proc/%d/status", ownpid); + fp = fopen(fn, "r"); + if (fp == NULL) + return "ended"; + + if (fgets(str, 80, fp) != NULL) { + char *p; + + procname = str + 6; + p = strchr(procname, '\n'); + if (p) + *p = '\0'; + } else + procname = "unknown"; + + fclose(fp); + return procname; +} + +static void show_dlm_grants(int locktype, const char *g_line, int dlmgrants, + int summary) +{ + int i; + char dlm_resid[64]; + unsigned int lkb_id, lkbnodeid, remid, ownpid, exflags, flags, status; + unsigned int grmode, rqmode, nodeid, length; + unsigned long long xid, us; + char trgt_res_name[64], res_name[64], *p1, *p2; + const char *procname; + + p1 = strchr(g_line, '/'); + if (!p1) + return; + p1++; + p2 = strchr(p1, ' '); + if (!p2) + return; + memset(trgt_res_name, 0, sizeof(trgt_res_name)); + memcpy(trgt_res_name, p1, p2 - p1); + sprintf(dlm_resid, "%8d%16s", locktype, trgt_res_name); + for (i = 0; i < dlmgrants; i++) { +/* +lkb_id n remid pid x e f s g rq u n ln res_name 1234567890123456 +1100003 1 2ae0006 8954 0 0 0 2 5 -1 0 1 24 " 2 102ab" +2a20001 1 30d0001 8934 0 0 0 2 3 -1 0 1 24 " 5 102ab" + b0001 2 860001 8868 0 0 10000 2 3 -1 0 0 24 " 1 2" +2450001 2 1be0002 8962 0 0 10000 1 -1 5 12214 0 24 " 2 102ab" +*/ + p1 = strchr(dlmglines[i], '\"'); + if (!p1) + continue; + p1++; + if (strncmp(dlm_resid, p1, 24)) + continue; + + sscanf(dlmglines[i], "%x %d %x %u %llu %x %x %d %d %d %llu " + "%u %d \"%24s\"\n", + &lkb_id, &lkbnodeid, &remid, &ownpid, &xid, &exflags, + &flags, &status, &grmode, &rqmode, &us, &nodeid, + &length, res_name); + if (status == 1) { /* Waiting */ + if (!lkbnodeid) + procname = getprocname(ownpid); + else + procname = ""; + if (summary) + print_it(NULL, " (", NULL); + else + print_it(NULL, " D: ", NULL); + print_it(NULL, "%s for %s, pid %d %s", NULL, + dlm_status(status), dlm_nodeid(lkbnodeid), + ownpid, procname); + if (summary) + print_it(NULL, ")", NULL); + } else if (grmode == 0) { + continue; /* ignore "D: Granted NL on node X" */ + } else { + procname = getprocname(ownpid); + if (summary) + print_it(NULL, " (", NULL); + else + print_it(NULL, " D: ", NULL); + print_it(NULL, "%s %s on %s to pid %d %s", NULL, + dlm_status(status), dlm_grtype(grmode), + dlm_nodeid(lkbnodeid), ownpid, procname); + if (summary) + print_it(NULL, ")", NULL); + } + if (!summary) + eol(0); + } +} + +static void print_call_trace(const char *hline) +{ + char *p, *pid, tmp[32], stackfn[64], str[96]; + FILE *fp; + int i; + + p = strchr(hline, 'p'); + if (!p) + return; + pid = p + 2; + p = strchr(pid, ' '); + if (!p) + return; + memset(tmp, 0, sizeof(tmp)); + memcpy(tmp, pid, p - pid); + sprintf(stackfn, "/proc/%s/stack", tmp); + fp = fopen(stackfn, "rt"); + if (fp == NULL) + return; + for (i = 0; i < MAX_CALLTRACE_LINES; i++) { + if (fgets(str, sizeof(str) - 1, fp) == NULL) + break; + if (strstr(str, "gfs2_glock_")) { /* skip lines we don't + care about*/ + i--; + continue; + } + p = strchr(str, '\n'); + if (p) + *p = '\0'; + p = strchr(str, ']'); + if (p) + p += 2; + else + p = str; + print_it(NULL, " C: %s ", NULL, p); + eol(0); + } + fclose(fp); +} + +static int is_ex(const char *hline) +{ + if (strncmp(hline, " H: s:EX ", 9) == 0) + return 1; + return 0; +} + +static int has_holder_flag(const char *hline, char flag) +{ + const char *p; + + p = strchr(hline, 'f'); + if (p == NULL) + return 0; + p++; + if (*p != ':') + return 0; + p++; + while (*p != '\0') { + if (*p == ' ') + return 0; + if (*p == flag) + return 1; + p++; + } + return 0; +} + +static int is_holder(const char *hline) +{ + return has_holder_flag(hline, 'H'); +} + +static int is_waiter(const char *hline) +{ + return has_holder_flag(hline, 'W'); +} + +static int get_lock_type(const char *str) +{ + const char *p; + + p = strchr(str, '/'); + return (p ? (*(p - 1)) - '0' : 0); +} + +static long long get_demote_time(const char *str) +{ + char *p; + char tmp[80]; + + p = strchr(str, '/'); + if (p == NULL) + return 0; + p++; + p = strchr(p, '/'); + if (p == NULL) + return 0; + p++; + strncpy(tmp, p, 79); + tmp[79] = '\0'; + p = strchr(tmp, ' '); + if (p == NULL) + return 0; + *p = '\0'; + return atoll(tmp); +} + +static const char *pid_string(const char *str) +{ + char *p; + static char pidstr[80]; + + memset(pidstr, 0, sizeof(pidstr)); + p = strchr(str, 'p'); + if (p) { + strncpy(pidstr, p + 2, sizeof(pidstr)); + pidstr[79] = '\0'; + p = strchr(pidstr, ']'); + if (p) { + p++; + *p = '\0'; + } + } + return pidstr; +} + +/* If this glock is relevant, return 0, else the reason it's irrelevant */ +static int irrelevant(const char *holder, const char *glockstr) +{ + int lock_type = get_lock_type(glockstr); + + /* Exclude shared and locks */ + if (!is_ex(holder)) + return 1; + /* Exclude locks held at mount time: statfs*/ + if (strstr(holder, "init_per_node")) + return 2; + if (strstr(holder, "init_journal")) + return 3; + if (strstr(holder, "init_inodes")) + return 4; + if (strstr(holder, "fill_super")) + return 5; + if (lock_type == 9) /* Exclude journal locks */ + return 6; + return 0; +} + +static const char *reason(int why) +{ + const char *reasons[] = {"(N/A:------)", /* 0 */ + "(N/A:Not EX)", /* 1 */ + "(N/A:System)", /* 2 */ + "(N/A:journl)", /* 3 */ + "(N/A:System)", /* 4 */ + "(N/A:System)", /* 5 */ + "(N/A:Journl)"}; /* 6 */ + + return reasons[why]; +} + +static void print_friendly_prefix(char one_glocks_lines[MAX_LINES][97]) +{ + int why = irrelevant(one_glocks_lines[1], one_glocks_lines[0]); + + if (why) + print_it(NULL, " U: %s ", NULL, reason(why)); + else + print_it(NULL, " U: ", NULL); +} + +static void show_glock(char one_glocks_lines[MAX_LINES][97], int gline, + const char *fsname, int dlmwaiters, int dlmgrants, + int trace_dir_path, int prev_had_waiter, int flags, + int summary) +{ + int i, locktype = 0; + char id[33], *p; + char extras[80], prefix = '\0'; + long long demote_time = 0; + const char *ltype[] = {"N/A", "non-disk", "inode", "rgrp", "meta", + "i_open", "flock", "posix lock", "quota", + "journal"}; + + if (termlines) { + if (irrelevant(one_glocks_lines[1], one_glocks_lines[0])) + COLORS_HELD; + else + COLORS_NORMAL; + } + if (!gline) + return; + + memset(extras, 0, sizeof(extras)); + p = strchr(one_glocks_lines[0], '/'); + memset(id, 0, sizeof(id)); + + if (p) { + locktype = get_lock_type(one_glocks_lines[0]); + demote_time = get_demote_time(one_glocks_lines[0]); + p++; + strncpy(id, p, sizeof(id) - 1); + id[sizeof(id) - 1] = '\0'; + p = strchr(id, ' '); + if (p) + *p = '\0'; + + if (locktype != 2) { + strncpy(extras, ltype[locktype], 79); + extras[79] = '\0'; + } else { + const char *i_type = show_details(id, fsname, 2, + trace_dir_path); + sprintf(extras, "%sinode", i_type); + } + } + if (flags & DETAILS) { + print_it(NULL, " %s ", NULL, one_glocks_lines[0]); + print_it(NULL, "(%s)", NULL, extras); + if (demote_time) + print_it(NULL, " ** demote time is greater than 0 **", + NULL); + eol(0); + if (dlmgrants) + show_dlm_grants(locktype, one_glocks_lines[0], + dlmgrants, 0); + } + if (flags & FRIENDLY) { + print_friendly_prefix(one_glocks_lines); + for (i = 1; i < gline; i++) { + if (one_glocks_lines[i][0] == ' ' && + one_glocks_lines[i][1] == 'H' && + prefix != 'W') + prefix = (is_holder(one_glocks_lines[i]) ? + 'H' : 'W'); + } + print_it(NULL, " %c %-10.10s %-9.9s %s", NULL, prefix, + extras, id, friendly_glock(one_glocks_lines[0], + prefix)); + eol(0); + } + for (i = 1; i < gline; i++) { + if (!show_reservations && + one_glocks_lines[i][0] == ' ' && + one_glocks_lines[i][2] == 'B' && + one_glocks_lines[i][3] == ':') + continue; + + if (flags & DETAILS) { + print_it(NULL, " %-80.80s", NULL, one_glocks_lines[i]); + eol(0); + continue; + } + if ((flags & FRIENDLY) && + one_glocks_lines[i][1] == 'H') + print_friendly_prefix(one_glocks_lines); + + if (one_glocks_lines[i][0] == ' ' && + one_glocks_lines[i][1] == 'H') { + print_it(NULL, " %c ---> %s pid %s ", NULL, + prefix, (is_holder(one_glocks_lines[i]) ? + "held by" : "waiting"), + pid_string(one_glocks_lines[i])); + if (demote_time) + print_it(NULL, "** demote time is non-" + "zero ** ", NULL); + if (is_dlm_waiting(dlmwaiters, locktype, id)) { + print_it(NULL, "***** DLM is in a " + "comm wait for this lock " + "***** ", NULL); + } + show_dlm_grants(locktype, one_glocks_lines[0], + dlmgrants, 1); + eol(0); + print_call_trace(one_glocks_lines[i]); + } + } +} + +static int parse_dlm_waiters(FILE *dlm, const char *fsname) +{ + int dlml = 0; + + memset(dlmwlines, 0, sizeof(dlmwlines)); + while (fgets(dlmwlines[dlml], 80, dlm)) + dlml++; + + return dlml; +} + +static int parse_dlm_grants(int dlmfd, const char *fsname) +{ + int dlml = 0; + char *dlmline; + + memset(dlmglines, 0, sizeof(dlmglines)); + dnextpos = NULL; + while ((dlmline = bufgets(dlmfd, dbuf, &dnextpos, &dpos, &dmaxpos))) { + if (!this_lkb_requested(dlmline)) + continue; + strncpy(dlmglines[dlml], dlmline, 96); + dlmglines[dlml][96] = '\0'; + dlml++; + if (dlml >= MAX_LINES) + break; + } + return dlml; +} + +static void print_summary(int total_glocks[11][stypes], int dlmwaiters) +{ + int i; + int total_unlocked = 0; + const struct { + const char *name; + const int width; + } column[] = { + { "unknown", 7 }, { "nondisk", 7}, { "inode", 8 }, { "rgrp", 7 }, + { "meta", 4 }, { "iopen", 7 }, { "flock", 7 }, { "p", 1 }, + { "quota", 5 }, { "jrnl", 4 }, { "Total", 8 } + }; + const int ncols = sizeof(column) / sizeof(column[0]); + + /* Print column headers */ + print_it(NULL, "S glocks ", NULL); + for (i = 1; i < ncols; i++) + if (i != 7 && i != 4) /* Ignore plock and meta */ + print_it(NULL, "%*s ", NULL, column[i].width, column[i].name); + eol(0); + print_it(NULL, "S --------- ", NULL); + for (i = 1; i < ncols; i++) + if (i != 7 && i != 4) /* Ignore plock and meta */ + print_it(NULL, "%*s ", NULL, column[i].width, "--------"); + eol(0); + + /* Print rows */ + print_it(NULL, "S Unlocked: ", NULL); + for (i = 1; i < (ncols - 1); i++) { + if (i != 7 && i != 4) /* Ignore plock and meta */ + print_it(NULL, "%*d ", NULL, column[i].width, + total_glocks[i][all] - total_glocks[i][locked]); + total_unlocked += total_glocks[i][all] - + total_glocks[i][locked]; + } + print_it(NULL, "%*d ", NULL, column[i].width, total_unlocked); + eol(0); + print_it(NULL, "S Locked: ", NULL); + for (i = 1; i < ncols; i++) { + if (i != 7 && i != 4) /* Ignore plock and meta */ + print_it(NULL, "%*d ", NULL, column[i].width, + total_glocks[i][locked]); + total_glocks[10][locked] += total_glocks[i][locked]; + } + eol(0); + print_it(NULL, "S Total: ", NULL); + for (i = 1; i < ncols; i++) { + if (i != 7 && i != 4) /* Ignore plock and meta */ + print_it(NULL, "%*d ", NULL, column[i].width, + total_glocks[i][all]); + total_glocks[10][all] += total_glocks[i][all]; + } + eol(0); + print_it(NULL, "S", NULL); + eol(0); + print_it(NULL, "S Held EX: ", NULL); + for (i = 1; i < ncols; i++) { + if (i != 7 && i != 4) /* Ignore plock and meta */ + print_it(NULL, "%*d ", NULL, column[i].width, + total_glocks[i][held_ex]); + total_glocks[10][held_ex] += total_glocks[i][held_ex]; + } + eol(0); + print_it(NULL, "S Held SH: ", NULL); + for (i = 1; i < ncols; i++) { + if (i != 7 && i != 4) /* Ignore plock and meta */ + print_it(NULL, "%*d ", NULL, column[i].width, + total_glocks[i][held_sh]); + total_glocks[10][held_sh] += total_glocks[i][held_sh]; + } + eol(0); + print_it(NULL, "S Held DF: ", NULL); + for (i = 1; i < ncols; i++) { + if (i != 7 && i != 4) /* Ignore plock and meta */ + print_it(NULL, "%*d ", NULL, column[i].width, + total_glocks[i][held_df]); + total_glocks[10][held_df] += total_glocks[i][held_df]; + } + eol(0); + print_it(NULL, "S G Waiting: ", NULL); + for (i = 1; i < ncols; i++) { + if (i != 7 && i != 4) /* Ignore plock and meta */ + print_it(NULL, "%*d ", NULL, column[i].width, + total_glocks[i][has_waiter]); + total_glocks[10][has_waiter] += total_glocks[i][has_waiter]; + } + eol(0); + print_it(NULL, "S P Waiting: ", NULL); + for (i = 1; i < ncols; i++) { + if (i != 7 && i != 4) /* Ignore plock and meta */ + print_it(NULL, "%*d ", NULL, column[i].width, + total_glocks[i][tot_waiters]); + total_glocks[10][tot_waiters] += total_glocks[i][tot_waiters]; + } + eol(0); + print_it(NULL, "S DLM wait: %7d", NULL, dlmwaiters); + eol(0); + eol(0); +} + +/* flags = DETAILS || FRIENDLY or both */ +static void glock_details(int fd, const char *fsname, int dlmwaiters, + int dlmgrants, int trace_dir_path, int show_held, + int summary) +{ + char *ln, *p; + char one_glocks_lines[MAX_LINES][97]; + int gline = 0; + int show_prev_glock = 0, prev_had_waiter = 0; + int total_glocks[11][stypes], locktype = 0; + int holders_this_glock_ex = 0; + int holders_this_glock_sh = 0; + int holders_this_glock_df = 0; + int waiters_this_glock = 0; + + memset(total_glocks, 0, sizeof(total_glocks)); + gnextpos = NULL; + while ((ln = bufgets(fd, gbuf, &gnextpos, &gpos, &gmaxpos))) { + if (ln[0] == ' ' && ln[1] == ' ' && ln[2] == ' ') + continue; + if (ln[0] == 'G') { + /* Summary stuff------------------------------------ */ + if (waiters_this_glock) { + total_glocks[locktype][tot_waiters] += + waiters_this_glock; + total_glocks[locktype][has_waiter]++; + } + if (holders_this_glock_ex) + total_glocks[locktype][held_ex]++; + if (holders_this_glock_sh) + total_glocks[locktype][held_sh]++; + if (holders_this_glock_df) + total_glocks[locktype][held_df]++; + locktype = get_lock_type(ln); + p = ln + 6; + if (*p != 'U' || *(p + 1) != 'N') + total_glocks[locktype][locked]++; + total_glocks[locktype][all]++; + holders_this_glock_ex = 0; + holders_this_glock_sh = 0; + holders_this_glock_df = 0; + waiters_this_glock = 0; + /* Detail stuff------------------------------------- */ + if (show_prev_glock) { + show_glock(one_glocks_lines, gline, fsname, + dlmwaiters, dlmgrants, + trace_dir_path, prev_had_waiter, + DETAILS, summary); + show_glock(one_glocks_lines, gline, fsname, + dlmwaiters, dlmgrants, + trace_dir_path, prev_had_waiter, + FRIENDLY, summary); + memset(one_glocks_lines, 0, + sizeof(one_glocks_lines)); + show_prev_glock = 0; + } + prev_had_waiter = 0; + gline = 0; + if (this_glock_requested(ln)) + show_prev_glock = 1; + } else if (ln[0] == ' ' && ln[1] == 'H') { + char *flag = strchr(ln, 'f'); + char *mode = strchr(ln, 's'); + + /* Summary stuff------------------------------------ */ + while (flag) { + flag++; + switch (*flag) { + case ':': + break; + case 'W': + waiters_this_glock++; + flag = NULL; + break; + case 'H': + flag = NULL; + if (mode == NULL) + holders_this_glock_df++; + else if (*(mode + 1) == ':' && + *(mode + 2) == 'E' && + *(mode + 3) == 'X') + holders_this_glock_ex++; + else if (*(mode + 1) == ':' && + *(mode + 2) == 'S' && + *(mode + 3) == 'H') + holders_this_glock_sh++; + else + holders_this_glock_df++; + break; + case ' ': + flag = NULL; + break; + default: + break; + }; + } + /* Detail stuff------------------------------------- */ + if (!glocks) { + int haswaiter = is_waiter(ln); + + if (haswaiter) { + show_prev_glock = 1; + prev_had_waiter = 1; + } else if (show_held && is_holder(ln) && + !is_iopen(one_glocks_lines[0])) { + show_prev_glock = 1; + } else if (!irrelevant(ln, one_glocks_lines[0])) { + show_prev_glock = 1; + } + } + } + /* Detail stuff--------------------------------------------- */ + strncpy(one_glocks_lines[gline], ln, 96); + one_glocks_lines[gline][96] = '\0'; + gline++; + if (gline >= MAX_LINES) + break; + if (termlines && line >= termlines) + break; + } + /* Detail stuff----------------------------------------------------- */ + if (show_prev_glock && gline < MAX_LINES && + (!termlines || line < termlines)) { + show_glock(one_glocks_lines, gline, fsname, dlmwaiters, + dlmgrants, trace_dir_path, prev_had_waiter, + DETAILS, summary); + show_glock(one_glocks_lines, gline, fsname, dlmwaiters, + dlmgrants, trace_dir_path, prev_had_waiter, + FRIENDLY, summary); + } + if (!summary || ((iters_done % summary) != 0)) + return; + + print_summary(total_glocks, dlmwaiters); +} + +static void show_help(int help) +{ + if (help == 1) { + COLORS_NORMAL; + eol(0); + print_it(NULL, " Glock flags: ", NULL); + eol(0); + print_it(NULL, " l - Locked ", NULL); + print_it(NULL, " r - Reply pending ", NULL); + eol(0); + print_it(NULL, " d - Demote pending ", NULL); + print_it(NULL, " I - Initial ", NULL); + eol(0); + print_it(NULL, " D - Demote requested ", NULL); + print_it(NULL, " F - Frozen ", NULL); + eol(0); + print_it(NULL, " p - Demote in progress ", NULL); + print_it(NULL, " q - Queued holder ", NULL); + eol(0); + print_it(NULL, " y - Dirty data ", NULL); + print_it(NULL, " L - LRU ", NULL); + eol(0); + print_it(NULL, " f - Flush ", NULL); + print_it(NULL, " o - Object present ", NULL); + eol(0); + print_it(NULL, " i - Invalidating ", NULL); + print_it(NULL, " b - Blocking request ", NULL); + eol(0); + } else if (help == 2) { + COLORS_NORMAL; + eol(0); + print_it(NULL, " Holder flags: ", NULL); + eol(0); + print_it(NULL, " t - Try (non-blocking) ", NULL); + print_it(NULL, " E - Exact lock ", NULL); + eol(0); + print_it(NULL, " T - Try with callback ", NULL); + print_it(NULL, " c - No Cache lock ", NULL); + eol(0); + print_it(NULL, " e - No exp ", NULL); + print_it(NULL, " H - Held (locked) ", NULL); + eol(0); + print_it(NULL, " A - Any lock ", NULL); + print_it(NULL, " W - Waiting for lock ", NULL); + eol(0); + print_it(NULL, " p - Priority lock ", NULL); + print_it(NULL, " a - Asynchronous lock ", NULL); + eol(0); + print_it(NULL, " F - First ", NULL); + eol(0); + } +} + +/* flags = DETAILS || FRIENDLY or both */ +static void parse_glocks_file(int fd, const char *fsname, int dlmwaiters, + int dlmgrants, int trace_dir_path, + int show_held, int help, int summary) +{ + char fstitle[96], fsdlm[96]; + char ctimestr[64]; + time_t t; + int i; + + tzset(); + t = time(NULL); + strftime(ctimestr, 64, "%a %b %d %T %Y", localtime(&t)); + ctimestr[63] = '\0'; + memset(fstitle, 0, sizeof(fstitle)); + memset(fsdlm, 0, sizeof(fsdlm)); + sprintf(fstitle, "@ %s %s ", fsname, ctimestr); + if (dlmwaiters) { + sprintf(fsdlm, "dlm: %s/%s/%s [", dlm_dirtbl_size, + dlm_rsbtbl_size, dlm_lkbtbl_size); + for (i = 0; i < dlmwaiters; i++) + strcat(fsdlm, "*"); + for (; i < 10; i++) + strcat(fsdlm, " "); + strcat(fsdlm, "]"); + } + attron(A_BOLD); + print_it(NULL, "%s @%s %s", NULL, fstitle, hostname, fsdlm); + eol(0); + attroff(A_BOLD); + glock_details(fd, fsname, dlmwaiters, dlmgrants, trace_dir_path, + show_held, summary); + + show_help(help); + if (termlines) + refresh(); +} + +static void usage(void) +{ + printf("Usage:\n"); + printf("glocktop [-i] [-d <delay sec>] [-f] [-n <iter>] [-sX] " + "[-c] [-D] [-H] [-r] [-t]\n"); + printf("\n"); + printf("-i : Runs glocktop in interactive mode.\n"); + printf("-d : delay between refreshes, in seconds (default: 3).\n"); + printf("-n : stop after <iter> refreshes.\n"); + printf("-H : don't show Held glocks, even if not waited on, excluding " + "iopen\n"); + printf("-r : show reservations when rgrp glocks are displayed\n"); + printf("-s : show glock summary information every X iterations\n"); + printf("-t : trace directory glocks back\n"); + printf("-D : don't show DLM lock status\n"); + printf("\n"); + fflush(stdout); + exit(0); +} + +int main(int argc, char **argv) +{ + int fd; + DIR *dir = NULL; + char fn[PATH_MAX]; + struct dirent *dent; + int retval; + struct timeval tv; + fd_set readfds; + char string[96]; + int ch, i, dlmwaiters = 0, dlmgrants = 0; + int cont = TRUE, optchar; + int trace_dir_path = 0; + int show_held = 1, help = 0; + int interactive = 0; + int summary = 10; + + prog_name = argv[0]; + memset(glock, 0, sizeof(glock)); + memset(contended_filenames, 0, sizeof(contended_filenames)); + memset(contended_blocks, 0, sizeof(contended_blocks)); + UpdateSize(0); + /* decode command line arguments */ + while (cont) { + optchar = getopt(argc, argv, "-d:Dn:rs:thHi"); + + switch (optchar) { + case 'd': + refresh_time = atoi(optarg); + if (refresh_time < 1) { + fprintf(stderr, "Error: delay %d too small; " + "must be > 1\n", refresh_time); + exit(-1); + } + break; + case 'D': + print_dlm_grants = 0; + break; + case 'n': + iterations = atoi(optarg); + break; + case 'r': + show_reservations = 1; + break; + case 's': + summary = atoi(optarg); + break; + case 't': + trace_dir_path = 1; + break; + case 'h': + usage(); + break; + case 'H': + show_held = 0; /* held, but not iopen held */ + break; + case 'i': + interactive = 1; + break; + case EOF: + cont = FALSE; + break; + case 1: + if (optarg && glocks < MAX_GLOCKS) + glock[glocks++] = optarg; + break; + + default: + fprintf(stderr, "unknown option: %c\n", optchar); + exit(-1); + }; + } + + if (interactive) { + printf("Initializing. Please wait..."); + fflush(stdout); + } + if (gethostname(hostname, sizeof(hostname))) { + fprintf(stderr, "Error: unable to determine host name.\n"); + exit(-1); + } + if (parse_mounts()) + exit(-1); + + if (interactive && (wind = initscr()) == NULL) { + fprintf(stderr, "Error: unable to initialize screen.\n"); + exit(-1); + } + + if (interactive) { + /* Do our initial screen stuff: */ + signal(SIGWINCH, UpdateSize); /* handle term resize signal */ + UpdateSize(0); /* update screen size based on term settings */ + clear(); /* don't use Erase */ + start_color(); + noecho(); + keypad(stdscr, TRUE); + raw(); + curs_set(0); + init_colors(); + } else { + termlines = 0; + } + while (!gbuf) { + gbuf = malloc(bufsize); + if (gbuf) { + /*printf("bufsize=%dK\n", bufsize / 1024);*/ + break; + } + bufsize /= 2; + } + while (!dbuf) { + dbuf = malloc(bufsize); + if (dbuf) { + /*printf("bufsize=%dK\n", bufsize / 1024);*/ + break; + } + bufsize /= 2; + } + + while (!done) { + sprintf(fn, "%s/gfs2/", debugfs); + dir = opendir(fn); + + if (!dir) { + if (interactive) { + refresh(); + endwin(); + } + fprintf(stderr, "Unable to open gfs2 debugfs directory.\n"); + fprintf(stderr, "Check if debugfs and gfs2 are mounted.\n"); + exit(-1); + } + display_title_lines(); + while ((dent = readdir(dir))) { + const char *fsname; + char dlm_fn[PATH_MAX]; + FILE *dlmf; + int dlmfd; + + if (!strcmp(dent->d_name, ".")) + continue; + if (!strcmp(dent->d_name, "..")) + continue; + + fsname = strchr(dent->d_name, ':'); + if (fsname) + fsname++; + else + fsname = dent->d_name; + + memset(dlm_fn, 0, sizeof(dlm_fn)); + sprintf(dlm_fn, "%s/dlm/%s_waiters", debugfs, fsname); + dlmf = fopen(dlm_fn, "rt"); + if (dlmf) { + dlmwaiters = parse_dlm_waiters(dlmf, fsname); + fclose(dlmf); + } + + if (print_dlm_grants) { + memset(dlm_fn, 0, sizeof(dlm_fn)); + sprintf(dlm_fn, "%s/dlm/%s_locks", debugfs, + fsname); + dlmfd = open(dlm_fn, O_RDONLY); + if (dlmfd > 0) { + dlmgrants = parse_dlm_grants(dlmfd, + fsname); + close(dlmfd); + } + } + + sprintf(fn, "%s/gfs2/%s/glocks", debugfs, + dent->d_name); + fd = open(fn, O_RDONLY); + if (fd < 0) { + if (interactive) { + refresh(); + endwin(); + } + perror(fn); + exit(-1); + } + parse_glocks_file(fd, fsname, dlmwaiters, dlmgrants, + trace_dir_path, show_held, help, + summary); + close(fd); + } + closedir(dir); + tv.tv_sec = refresh_time; + tv.tv_usec = 0; + FD_ZERO(&readfds); + FD_SET(0, &readfds); + retval = select(1, &readfds, NULL, NULL, &tv); + if (retval) { + if (interactive) + ch = getch(); + else + ch = getchar(); + switch (ch) { + case 0x1b: /* mount wheel? */ + case 0x03: + case 'q': + done = 1; + break; + case 'h': + help = (help + 1) % 3; + break; + case 's': + if (!interactive) + break; + move(1, 0); + printw("Change delay from %d to: ", + refresh_time); + if (bobgets(string, 1, 25, 5, &ch) == 1) + refresh_time = atoi(string); + if (refresh_time < 1) + refresh_time = 1; + break; + } + } + iters_done++; + if (iterations && iters_done >= iterations) + break; + } + for (i = 0; i < mounted; i++) + close(fs_fd[i]); + free(gbuf); + free(dbuf); + if (interactive) { + refresh(); + endwin(); + } + exit(0); +} diff --git a/gfs2/man/Makefile.am b/gfs2/man/Makefile.am index d73f1ed..b6cc120 100644 --- a/gfs2/man/Makefile.am +++ b/gfs2/man/Makefile.am @@ -10,4 +10,5 @@ dist_man_MANS = \ mkfs.gfs2.8 \ tunegfs2.8 \ gfs2_lockcapture.8 \ - gfs2_trace.8 + gfs2_trace.8 \ + glocktop.8 diff --git a/gfs2/man/glocktop.8 b/gfs2/man/glocktop.8 new file mode 100644 index 0000000..19379c3 --- /dev/null +++ b/gfs2/man/glocktop.8 @@ -0,0 +1,281 @@ +.TH glocktop 8 + +.SH NAME +glocktop - Display or print active GFS2 locks. + +.SH SYNOPSIS +.B glocktop +[\fIOPTIONS\fR] + +.SH DESCRIPTION +The glocktop tool is used to display active GFS2 inter-node locks, +also known as glocks. Simply put, it's a tool to filter and interpret the +contents of the glocks debugfs file. The glocks debugfs file shows +all glocks known to GFS2, their holders, and technical data such as flags. +The glocktop tool will only show the glocks that are important: glocks that +are being held or for which there are waiters. It also interprets the debugfs +file of DLM (Distributed Lock Manager). + +.SH OPTIONS +.TP +\fB-d\fP \fI<delay>\fP +Specify a time delay (in seconds) between reports. (Default is 30 seconds) +.TP +\fB-h\fP +Print help information. +.TP +\fB-i\fP +Interactive mode. In this mode, glocktop acts more like the top command. +It shows the pertinent glocks on the terminal session (as many as it can +fit). The advantage is that it uses different colors to draw attention to +what's important. The disadvantage is that it's limited by the size of +your display, so you may not see all the glocks. +.TP +\fB-D\fP +Omit DLM status. This may be used to reduce the amount of output for +interactive mode. +.TP +\fB-n\fP \fI<iterations>\fP +End the program after the specified number of iterations (reports). The +default is to keep running until interrupted. +.TP +\fB-r\fP +Show resource group reservation information. Normally, glocktop omits +resource group reservation information to condense the output. This +information is only important when debugging information related to the +GFS2 block allocator and file system fragmentation. +.TP +\fB-s\fP \fI<freq>\fR +Print glock summary information every \fI<freq>\fR reports. +The glock summary information is bulky and often not needed, so it's +only printed once every 10 reports. You can eliminate it entirely from +the output by specifying a value of 0. If you want the statistics to +print after every report, specify freq as 1. +.TP +\fB-t\fP +Trace directory path. A lot of GFS2 glock performance problems are caused +by an application's contention for one or two directories. These show up +as regular inodes in the output, but there's no good way to tell from the +output which directory is contended. Ordinarily, glocktop won't try to +look up the full pathname of a contended directory because it's slow, +especially if there are millions of glocks. This option instructs glocktop +to try to determine the full directory path names when it can, so you can +tell the full path (within the mount point) of contended directories. +.TP +\fB-H\fP +Don't show Held glocks, unless there are also waiters for the lock. +Ordinarily, glocktop will show glocks that are held (but not iopen +glocks which are almost always held by the thousands) as well as glocks +for which there are waiters. If it only showed glocks with waiters, you +could see, for example, that a glock is being blocked on one node, +but you couldn't see the information for a different node currently +holding the lock and thus, blocking the waiter. This option forces glocktop to +stop printing information for glocks with no waiters (on that node). +The advantage is that the output is smaller and easier to look at. +The disadvantage is that you can't see information from the node that's +blocking the waiter, unless both waiter and holder are on the same node. +.SH OUTPUT LINES +.TP +\fB@ name\fP +This is the GFS2 file system name for which the information is printed. It +also gives the time stamp of the report, and the cluster node name. +.TP +\fBG:\fP +This line represents a glock (internode GFS2 lock). + G: s:UN n:2/609b4 f:lIqob t:EX d:EX/0 a:0 v:0 r:3 m:200 (inode) +.TP +\fBD:\fP +This line gives you glocktop's interpretation of the glock's state as +far as DLM (distributed lock manager) is concerned. + D: Granted PR on node 2 to pid 17511 [python] +.TP +\fBH:\fP +This line represents a glock holder: a process that's either holding the +glock, or is waiting to hold it. The value after S: represents how this +holder needs the lock: EX (Exclusive), SH (Shared), PR (Protected Read), +or UN (Unlocked). The value after F: indicates the holder flags: a W +indicates the holder is Waiting for the lock to be granted. An H indicates +the holder is currently holding the lock. + H: s:EX f:W e:0 p:17511 [python] gfs2_unlink+0x7e/0x250 [gfs2] +.TP +\fBU:\fP +These lines represent glocktop's user interpretation of the data, both glock +and holder. Lines that begin with (N/A:...) can probably be ignored because +they ought to be unimportant: system files such as journals, etc. + U: W inode 183f5 Is:Shared, Want:Exclusive [Demote pending, Reply pending, Queued, Blocking] + U: W ---> waiting pid 17511 [python] (Granted PR on node 2 to pid 17511 [python]) +.TP +\fBC:\fP +These lines give you the call trace (call stack) of the process that's +either holding or waiting to hold the glock. +.TP +\fBS\fP +These lines give you the summary of all glocks for this file system: How many of +each category are unlocked, locked, how many are held in EX, SH, and DF, and how +many are waiting. G Waiting is how many glocks have waiters. P Waiting is +how many processes are waiting. Thus, you could have one glock that's got +ten processes waiting, or ten glocks that have ten processes waiting. +.SH EXAMPLE OUTPUT +.nf +.RS +# glocktop +.PP +@ nate_bob1 Wed Jan 27 07:24:14 2016 @host-050 + G: s:EX n:9/1 f:Iqb t:EX d:EX/0 a:0 v:0 r:2 m:200 (journal) + D: Granted EX on node 2 to pid 17468 [ended] + H: s:EX f:eH e:0 p:17468 [(ended)] gfs2_glock_nq_num+0x5b/0xa0 [gfs2] + U: (N/A:Journl) H journal 1 Held:Exclusive [Queued, Blocking] + U: (N/A:Journl) H ---> held by pid 17468 [(ended)] (Granted EX on node 2 to pid 17468 [ended]) + G: s:SH n:1/1 f:Iqb t:SH d:EX/0 a:0 v:0 r:2 m:200 (non-disk) + D: Granted PR on node 2 to pid 17468 [ended] + H: s:SH f:eEH e:0 p:17468 [(ended)] gfs2_glock_nq_num+0x5b/0xa0 [gfs2] + U: (N/A:Not EX) H non-disk 1 Held:Shared [Queued, Blocking] + U: (N/A:Not EX) H ---> held by pid 17468 [(ended)] (Granted PR on node 2 to pid 17468 [ended]) + G: s:EX n:2/181ec f:yIqob t:EX d:EX/0 a:1 v:0 r:3 m:200 (inode) + D: Granted EX on this node to pid 17468 [ended] + H: s:EX f:H e:0 p:17468 [(ended)] init_per_node+0x17d/0x280 [gfs2] + I: n:12/98796 t:8 f:0x00 d:0x00000201 s:24 + U: (N/A:System) H inode 181ec Held:Exclusive [Dirty, Queued, Blocking] + U: (N/A:System) H ---> held by pid 17468 [(ended)] (Granted EX on this node to pid 17468 [ended]) + G: s:EX n:2/181ed f:Iqob t:EX d:EX/0 a:0 v:0 r:3 m:200 (inode) + D: Granted EX on this node to pid 17468 [ended] + H: s:EX f:H e:0 p:17468 [(ended)] init_per_node+0x1b0/0x280 [gfs2] + I: n:13/98797 t:8 f:0x00 d:0x00000200 s:1048576 + U: (N/A:System) H inode 181ed Held:Exclusive [Queued, Blocking] + U: (N/A:System) H ---> held by pid 17468 [(ended)] (Granted EX on this node to pid 17468 [ended]) + G: s:SH n:2/183f5 f:ldrIqob t:EX d:UN/0 a:0 v:0 r:5 m:10 (inode) + D: Granted PR on node 2 to pid 17511 [python] + H: s:EX f:W e:0 p:17511 [python] gfs2_unlink+0x7e/0x250 [gfs2] + I: n:1/99317 t:4 f:0x00 d:0x00000003 s:2048 + U: W inode 183f5 Is:Shared, Want:Exclusive [Demote pending, Reply pending, Queued, Blocking] + U: W ---> waiting pid 17511 [python] (Granted PR on node 2 to pid 17511 [python]) + C: gfs2_unlink+0xdc/0x250 [gfs2] + C: vfs_unlink+0xa0/0xf0 + C: do_unlinkat+0x163/0x260 + C: sys_unlink+0x16/0x20 + G: s:SH n:2/805b f:Iqob t:SH d:EX/0 a:0 v:0 r:3 m:200 (inode) + D: Granted PR on node 2 to pid 17468 [ended] + H: s:SH f:eEcH e:0 p:17468 [(ended)] init_journal+0x185/0x500 [gfs2] + I: n:5/32859 t:8 f:0x01 d:0x00000200 s:134217728 + U: (N/A:Not EX) H inode 805b Held:Shared [Queued, Blocking] + U: (N/A:Not EX) H ---> held by pid 17468 [(ended)] (Granted PR on node 2 to pid 17468 [ended]) +S glocks nondisk inode rgrp iopen flock quota jrnl Total +S --------- ------- -------- ------- ------- ------- ----- ---- -------- +S Unlocked: 1 5 4 0 0 0 0 10 +S Locked: 2 245 6 58 0 0 1 313 +S Total: 3 250 10 58 0 0 1 323 +S +S Held EX: 0 2 0 0 0 0 1 3 +S Held SH: 1 1 0 57 0 0 0 59 +S Held DF: 0 0 0 0 0 0 0 0 +S G Waiting: 0 1 0 0 0 0 0 1 +S P Waiting: 0 1 0 0 0 0 0 1 +S DLM wait: 0 + +@ nate_bob0 Wed Jan 27 07:24:14 2016 @host-050 + G: s:EX n:2/180e9 f:yIqob t:EX d:EX/0 a:1 v:0 r:3 m:200 (inode) + D: Granted EX on this node to pid 17465 [ended] + H: s:EX f:H e:0 p:17465 [(ended)] init_per_node+0x17d/0x280 [gfs2] + I: n:9/98537 t:8 f:0x00 d:0x00000201 s:24 + U: (N/A:System) H inode 180e9 Held:Exclusive [Dirty, Queued, Blocking] + U: (N/A:System) H ---> held by pid 17465 [(ended)] (Granted EX on this node to pid 17465 [ended]) + G: s:UN n:2/609b4 f:lIqob t:EX d:EX/0 a:0 v:0 r:3 m:200 (inode) + D: Granted EX on node 2 to pid 14367 [ended] + H: s:EX f:W e:0 p:16297 [delete_workqueu] gfs2_delete_inode+0x9d/0x450 [gfs2] + U: W inode 609b4 Is:Unlocked, Want:Exclusive [Queued, Blocking] + U: W ---> waiting pid 16297 [delete_workqueu] (Granted EX on node 2 to pid 14367 [ended]) + C: gfs2_delete_inode+0xa5/0x450 [gfs2] + C: generic_delete_inode+0xde/0x1d0 + C: generic_drop_inode+0x65/0x80 + C: gfs2_drop_inode+0x37/0x40 [gfs2] + G: s:SH n:2/19 f:Iqob t:SH d:EX/0 a:0 v:0 r:3 m:200 (inode) + D: Granted PR on this node to pid 17465 [ended] + H: s:SH f:eEcH e:0 p:17465 [(ended)] init_journal+0x185/0x500 [gfs2] + I: n:4/25 t:8 f:0x01 d:0x00000200 s:134217728 + U: (N/A:Not EX) H inode 19 Held:Shared [Queued, Blocking] + U: (N/A:Not EX) H ---> held by pid 17465 [(ended)] (Granted PR on this node to pid 17465 [ended]) + G: s:EX n:2/180ea f:Iqob t:EX d:EX/0 a:0 v:0 r:3 m:200 (inode) + D: Granted EX on this node to pid 17465 [ended] + H: s:EX f:H e:0 p:17465 [(ended)] init_per_node+0x1b0/0x280 [gfs2] + I: n:10/98538 t:8 f:0x00 d:0x00000200 s:1048576 + U: (N/A:System) H inode 180ea Held:Exclusive [Queued, Blocking] + U: (N/A:System) H ---> held by pid 17465 [(ended)] (Granted EX on this node to pid 17465 [ended]) + G: s:EX n:9/0 f:Iqb t:EX d:EX/0 a:0 v:0 r:2 m:200 (journal) + D: Granted EX on this node to pid 17465 [ended] + H: s:EX f:eH e:0 p:17465 [(ended)] gfs2_glock_nq_num+0x5b/0xa0 [gfs2] + U: (N/A:Journl) H journal 0 Held:Exclusive [Queued, Blocking] + U: (N/A:Journl) H ---> held by pid 17465 [(ended)] (Granted EX on this node to pid 17465 [ended]) + G: s:UN n:2/4fe12 f:ldIqob t:EX d:UN/0 a:0 v:0 r:4 m:10 (inode) + H: s:EX f:W e:0 p:17523 [python] gfs2_rename+0x344/0x8b0 [gfs2] + H: s:SH f:AW e:0 p:17527 [python] gfs2_permission+0x176/0x210 [gfs2] + U: W inode 4fe12 Is:Unlocked, Want:Exclusive [Demote pending, Queued, Blocking] + U: W ---> waiting pid 17523 [python] + C: gfs2_permission+0x17f/0x210 [gfs2] + C: __link_path_walk+0xb3/0x1000 + C: path_walk+0x6a/0xe0 + C: filename_lookup+0x6b/0xc0 + U: W ---> waiting pid 17527 [python] + C: do_unlinkat+0x107/0x260 + C: sys_unlink+0x16/0x20 + C: system_call_fastpath+0x16/0x1b + C: 0xffffffffffffffff + G: s:SH n:1/1 f:Iqb t:SH d:EX/0 a:0 v:0 r:2 m:200 (non-disk) + D: Granted PR on node 2 to pid 14285 [ended] + D: Granted PR on this node to pid 17465 [ended] + H: s:SH f:eEH e:0 p:17465 [(ended)] gfs2_glock_nq_num+0x5b/0xa0 [gfs2] + U: (N/A:Not EX) H non-disk 1 Held:Shared [Queued, Blocking] + U: (N/A:Not EX) H ---> held by pid 17465 [(ended)] (Granted PR on node 2 to pid 14285 [ended]) (Granted PR on this node to pid 17465 [ended]) +S glocks nondisk inode rgrp iopen flock quota jrnl Total +S --------- ------- -------- ------- ------- ------- ----- ---- -------- +S Unlocked: 1 8 7 0 0 0 0 16 +S Locked: 2 208 3 41 0 0 1 256 +S Total: 3 216 10 41 0 0 1 272 +S +S Held EX: 0 2 0 0 0 0 1 3 +S Held SH: 1 1 0 41 0 0 0 43 +S Held DF: 0 0 0 0 0 0 0 0 +S G Waiting: 0 2 0 0 0 0 0 2 +S P Waiting: 0 3 0 0 0 0 0 3 +S DLM wait: 0 +.RE +.fi +.PP +From this example output, we can see there are two GFS2 file systems +mounted on system host-050: nate_bob1 and nate_bob0. In nate_bob1, we can +see six glocks, but we can ignore all of them marked (N/A:...) because they +are system files or held in SHared mode, and therefore other nodes should +be able to hold the lock in SHared as well. +.PP +There is one glock, for inode 183f5, which is has a process waiting to +hold it. The lock is currently in SHared mode (s:SH on the G: line) but +process 17511 (python) wants to hold the lock in EXclusive mode (S:EX +on the H: line). That process has a call stack that indicates it is trying +to hold the glock from gfs2_unlink. The DLM says the lock is currently +granted on node 2 in PR (Protected Read) mode. +.PP +For file system nate_bob0, there are 7 glocks listed. All but two are +uninteresting. Locks 2/609b4 and 2/4fe12 have processes waiting to +hold them. +.PP +In the summary data for nate_bob0, you can see there are 3 processes waiting +for 2 inode glocks (so one of those glocks has multiple processes waiting). +.PP +Since DLM wait is 0 in the summary data for both GFS2 mount points, +nobody is waiting for DLM to grant the lock. + +.SH KNOWN BUGS AND LIMITATIONS +.PP +Since the GFS2 debugfs files are completely separate from the DLM debugfs +files, and locks can change status in a few nanoseconds time, there will +always be a lag between the GFS2 view of a lock and the DLM view of a lock. +If there is some kind of long-term hang, they are more likely to match. +However, under ordinary conditions, by the time glocktop gets around to +fetching the DLM status of a lock, the information has changed. Therefore, +don't be surprised if the DLM's view of a lock is at odds with its glock. +.PP +Since iopen glocks are held by the thousands, glocktop skips most of the +information related to them unless there's a waiter. For that reason, +iopen lock problems may be difficult to debug with glocktop. +.PP +It doesn't handle very large numbers (millions) of glocks. +

1 0

cluster: RHEL6 - cman: Properly check for votes when node names aren't specified
by Christine Caulfield 14 Jan '16

14 Jan '16

Gitweb: http://git.fedorahosted.org/git/?p=cluster.git;a=commitdiff;h=4769ced12fb3a… Commit: 4769ced12fb3a834f801b26d975f023600662245 Parent: d4ec75242b7f7e7463113eb36cfba07112630125 Author: Christine Caulfield <ccaulfie(a)redhat.com> AuthorDate: Thu Jan 14 09:53:44 2016 +0000 Committer: Christine Caulfield <ccaulfie(a)redhat.com> CommitterDate: Thu Jan 14 09:55:35 2016 +0000 cman: Properly check for votes when node names aren't specified Signed-off-by: Ken Gaillot <kgaillot(a)redhat.com> Reviewed-by: Christine Caulfield <ccaulfie(a)redhat.com> --- cman/daemon/cmanconfig.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/cman/daemon/cmanconfig.c b/cman/daemon/cmanconfig.c index 7d35bea..c50767c 100644 --- a/cman/daemon/cmanconfig.c +++ b/cman/daemon/cmanconfig.c @@ -221,7 +221,7 @@ static int get_cman_join_info(struct corosync_api_v1 *corosync) } node_object = nodelist_byname(corosync, cluster_parent_handle, our_nodename); - if (!node_object) { + if (!votes && !node_object) { log_printf(LOG_ERR, "unable to find votes for %s", our_nodename); write_cman_pipe("Unable to find votes for node in CCS"); return -E2BIG;

1 0

cluster: STABLE32 - cman: Properly check for votes when node names aren't specified
by Christine Caulfield 14 Jan '16

14 Jan '16

Gitweb: http://git.fedorahosted.org/git/?p=cluster.git;a=commitdiff;h=9b8e381474c4e… Commit: 9b8e381474c4e7fcbdb192b516cb5a0dd41b1730 Parent: be46b108112a4a5027bba42a4b54d619743c7b82 Author: Christine Caulfield <ccaulfie(a)redhat.com> AuthorDate: Thu Jan 14 09:53:44 2016 +0000 Committer: Christine Caulfield <ccaulfie(a)redhat.com> CommitterDate: Thu Jan 14 09:53:44 2016 +0000 cman: Properly check for votes when node names aren't specified Signed-off-by: Ken Gaillot <kgaillot(a)redhat.com> Reviewed-by: Christine Caulfield <ccaulfie(a)redhat.com> --- cman/daemon/cmanconfig.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/cman/daemon/cmanconfig.c b/cman/daemon/cmanconfig.c index d6758e6..ca95609 100644 --- a/cman/daemon/cmanconfig.c +++ b/cman/daemon/cmanconfig.c @@ -229,7 +229,7 @@ static int get_cman_join_info(struct corosync_api_v1 *corosync) } node_object = nodelist_byname(corosync, cluster_parent_handle, our_nodename); - if (!node_object) { + if (!votes && !node_object) { log_printf(LOG_ERR, "unable to find votes for %s", our_nodename); write_cman_pipe("Unable to find votes for node in CCS"); return -E2BIG;

1 0

cluster: STABLE32 - cman: fix misleading message if cluster has only one node
by Christine Caulfield 14 Jan '16

14 Jan '16

Gitweb: http://git.fedorahosted.org/git/?p=cluster.git;a=commitdiff;h=be46b108112a4… Commit: be46b108112a4a5027bba42a4b54d619743c7b82 Parent: 9d6a1843fefe0799065afbe5d0a7637fed6887fc Author: Christine Caulfield <ccaulfie(a)redhat.com> AuthorDate: Thu Oct 30 09:48:15 2014 +0000 Committer: Christine Caulfield <ccaulfie(a)redhat.com> CommitterDate: Thu Oct 30 09:48:15 2014 +0000 cman: fix misleading message if cluster has only one node If a cluster is created with one node but two_node=1 is set then cman exits with the message: two_node set but there are more than 2 nodes This patch changes the message slightly so it doesn't imply that 1 is greater than 2! Signed-off-by: Christine Caulfield <ccaulfie(a)redhat.com> --- cman/daemon/cmanconfig.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/cman/daemon/cmanconfig.c b/cman/daemon/cmanconfig.c index 7eb52e4..d6758e6 100644 --- a/cman/daemon/cmanconfig.c +++ b/cman/daemon/cmanconfig.c @@ -270,7 +270,7 @@ static int get_cman_join_info(struct corosync_api_v1 *corosync) "nodes with one vote each and expected " "votes of 1 (node_count=%d vote_sum=%d)", node_count, vote_sum); - write_cman_pipe("two_node set but there are more than 2 nodes"); + write_cman_pipe("two_node is set but there are not 2 nodes in cluster.conf"); error = -EINVAL; goto out; }

1 0

cluster: STABLE32 - liblogthread: Avoid potetntial race when changing log files
by Christine Caulfield 14 Jan '16

14 Jan '16

Gitweb: http://git.fedorahosted.org/git/?p=cluster.git;a=commitdiff;h=9d6a1843fefe0… Commit: 9d6a1843fefe0799065afbe5d0a7637fed6887fc Parent: 2f05327bd8e7a072418dca45988ab5a814109258 Author: Christine Caulfield <ccaulfie(a)redhat.com> AuthorDate: Thu Oct 30 09:43:35 2014 +0000 Committer: Christine Caulfield <ccaulfie(a)redhat.com> CommitterDate: Thu Oct 30 09:43:35 2014 +0000 liblogthread: Avoid potetntial race when changing log files There is a potential for a crash when a log file is changed or re-opened as the fclose can happen in the main thread while the writing thread is about to output data. This patch avoids that by allocating a new fp for the new file and only closing the old fp* later one when the thread holds the lock. Signed-off-by: Christine Caulfield <ccaulfie(a)redhat.com> --- common/liblogthread/liblogthread.c | 36 +++++++++++++++++++++++++++--------- 1 files changed, 27 insertions(+), 9 deletions(-) diff --git a/common/liblogthread/liblogthread.c b/common/liblogthread/liblogthread.c index 088d171..b44f01a 100644 --- a/common/liblogthread/liblogthread.c +++ b/common/liblogthread/liblogthread.c @@ -38,6 +38,7 @@ static int logt_logfile_priority; static char logt_name[PATH_MAX]; static char logt_logfile[PATH_MAX]; static FILE *logt_logfile_fp; +static FILE *new_logt_logfile_fp; static char *_time(time_t *t) { @@ -100,6 +101,13 @@ static void *thread_fn(void *arg) prev_dropped = dropped; dropped = 0; + + if (new_logt_logfile_fp) { + fclose(logt_logfile_fp); + logt_logfile_fp = new_logt_logfile_fp; + new_logt_logfile_fp = NULL; + } + pthread_mutex_unlock(&mutex); if (prev_dropped) { @@ -176,20 +184,30 @@ static void _conf(const char *name, int mode, int syslog_facility, strncpy(logt_logfile, logfile, PATH_MAX - 1); if (logt_mode & LOG_MODE_OUTPUT_FILE && logt_logfile[0]) { - if (logt_logfile_fp) { - fclose(logt_logfile_fp); - logt_logfile_fp = NULL; - } - logt_logfile_fp = fopen(logt_logfile, "a+"); - if (logt_logfile_fp != NULL) { - fd = fileno(logt_logfile_fp); - fcntl(fd, F_SETFD, fcntl(fd, F_GETFD, 0) | FD_CLOEXEC); - } + /* Don't close the existing fp in this thread, let the main logthread do it later */ + if (!logt_logfile_fp) { + logt_logfile_fp = fopen(logt_logfile, "a+"); + if (logt_logfile_fp != NULL) { + fd = fileno(logt_logfile_fp); + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD, 0) | FD_CLOEXEC); + } + } + else { + new_logt_logfile_fp = fopen(logt_logfile, "a+"); + if (new_logt_logfile_fp != NULL) { + fd = fileno(new_logt_logfile_fp); + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD, 0) | FD_CLOEXEC); + } + } } else { if (logt_logfile_fp) { fclose(logt_logfile_fp); logt_logfile_fp = NULL; } + if (new_logt_logfile_fp) { + fclose(new_logt_logfile_fp); + new_logt_logfile_fp = NULL; + } } if (logt_mode & LOG_MODE_OUTPUT_SYSLOG) {

1 0

dlm: dlm-4.0.4 tag has been created
by David Teigland 05 Jan '16

05 Jan '16

1 0

dlm: master - release 4.0.4
by David Teigland 05 Jan '16

05 Jan '16

Gitweb: http://git.fedorahosted.org/git/?p=dlm.git;a=commitdiff;h=78979b03eb3d6f16a… Commit: 78979b03eb3d6f16adfdd816d8d4d4ca70ac3cd8 Parent: f9646b9265380ceeb65e9dbdf9773cb30a61ab71 Author: David Teigland <teigland(a)redhat.com> AuthorDate: Tue Jan 5 11:15:44 2016 -0600 Committer: David Teigland <teigland(a)redhat.com> CommitterDate: Tue Jan 5 11:15:44 2016 -0600 release 4.0.4 Signed-off-by: David Teigland <teigland(a)redhat.com> --- include/version.cf | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/include/version.cf b/include/version.cf index 27b4f4d..f128f14 100644 --- a/include/version.cf +++ b/include/version.cf @@ -1,6 +1,6 @@ #ifndef _RELEASE_VERSION_CF_ #define _RELEASE_VERSION_CF_ -#define RELEASE_VERSION "4.0.3" +#define RELEASE_VERSION "4.0.4" #endif

1 0

dlm: master - Don't SIGKILL dlm_controld
by David Teigland 05 Jan '16

05 Jan '16

Gitweb: http://git.fedorahosted.org/git/?p=dlm.git;a=commitdiff;h=f9646b9265380ceeb… Commit: f9646b9265380ceeb65e9dbdf9773cb30a61ab71 Parent: a51b2bbe413222829778698e62af88a73ebec233 Author: Ferenc W��gner <wferi(a)niif.hu> AuthorDate: Wed Sep 30 11:54:58 2015 +0200 Committer: David Teigland <teigland(a)redhat.com> CommitterDate: Tue Jan 5 11:11:20 2016 -0600 Don't SIGKILL dlm_controld Signed-off-by: Ferenc W��gner <wferi(a)niif.hu> --- init/dlm.service | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/init/dlm.service b/init/dlm.service index 598a8ae..1db5925 100644 --- a/init/dlm.service +++ b/init/dlm.service @@ -12,6 +12,10 @@ ExecStartPre=/sbin/modprobe dlm ExecStart=/usr/sbin/dlm_controld --foreground $DLM_CONTROLD_OPTS #ExecStopPost=/sbin/modprobe -r dlm +# If dlm_controld doesn't stop, there are active lockspaces. +# Killing it will just get the node fenced. +SendSIGKILL=no + [Install] WantedBy=multi-user.target

1 0

dlm: master - If an error occurs unlink the lock file and exit with status 1
by David Teigland 05 Jan '16

05 Jan '16

Gitweb: http://git.fedorahosted.org/git/?p=dlm.git;a=commitdiff;h=a51b2bbe413222829… Commit: a51b2bbe413222829778698e62af88a73ebec233 Parent: 79e87eb5913fcf93addc81c044ef050977cc4b37 Author: Ferenc W��gner <wferi(a)niif.hu> AuthorDate: Wed Sep 30 11:44:54 2015 +0200 Committer: David Teigland <teigland(a)redhat.com> CommitterDate: Tue Jan 5 11:11:20 2016 -0600 If an error occurs unlink the lock file and exit with status 1 Signed-off-by: Ferenc W��gner <wferi(a)niif.hu> --- dlm_controld/main.c | 19 +++++++++++-------- 1 files changed, 11 insertions(+), 8 deletions(-) diff --git a/dlm_controld/main.c b/dlm_controld/main.c index 287b82d..4f1399f 100644 --- a/dlm_controld/main.c +++ b/dlm_controld/main.c @@ -925,7 +925,7 @@ void cluster_dead(int ci) cluster_down = 1; } -static void loop(void) +static int loop(void) { struct lockspace *ls; int poll_timeout = -1; @@ -1029,6 +1029,7 @@ static void loop(void) rv = poll(pollfd, client_maxi + 1, poll_timeout); if (rv == -1 && errno == EINTR) { if (daemon_quit && list_empty(&lockspaces)) + rv = 0; goto out; if (daemon_quit) { log_error("shutdown ignored, active lockspaces"); @@ -1101,6 +1102,7 @@ static void loop(void) list_for_each_entry(ls, &lockspaces, list) log_error("abandoned lockspace %s", ls->name); + return rv; } static int lockfile(const char *dir, const char *name) @@ -1594,7 +1596,7 @@ int main(int argc, char **argv) fd = lockfile(RUNDIR, RUN_FILE_NAME); if (fd < 0) - return fd; + return 1; log_level(NULL, LOG_INFO, "dlm_controld %s started", RELEASE_VERSION); @@ -1602,29 +1604,30 @@ int main(int argc, char **argv) act.sa_handler = sigterm_handler; rv = sigaction(SIGTERM, &act, NULL); if (rv < 0) - return -rv; + goto out; rv = sigaction(SIGINT, &act, NULL); if (rv < 0) - return -rv; + goto out; memset(&act, 0, sizeof(act)); act.sa_handler = SIG_IGN; rv = sigaction(SIGHUP, &act, NULL); if (rv < 0) - return -rv; + goto out; memset(&act, 0, sizeof(act)); act.sa_handler = sigchld_handler; act.sa_flags = SA_NOCLDSTOP; rv = sigaction(SIGCHLD, &act, NULL); if (rv < 0) - return -rv; + goto out; /* set_scheduler(); */ - loop(); + rv = loop(); + out: unlink_lockfile(fd, RUNDIR, RUN_FILE_NAME); - return 0; + return rv < 0 ? 1 : 0; }

1 0

dlm: master - Make systemd stop dlm on corosync restart
by David Teigland 05 Jan '16

05 Jan '16

Gitweb: http://git.fedorahosted.org/git/?p=dlm.git;a=commitdiff;h=79e87eb5913fcf93a… Commit: 79e87eb5913fcf93addc81c044ef050977cc4b37 Parent: bddc503b1e1e56afcbcc02edded74f56b5157bb4 Author: Ferenc W��gner <wferi(a)niif.hu> AuthorDate: Mon Sep 28 13:41:26 2015 +0200 Committer: David Teigland <teigland(a)redhat.com> CommitterDate: Tue Jan 5 11:11:19 2016 -0600 Make systemd stop dlm on corosync restart If corosync stops, dlm_controld exits anyway. This dependency makes this explicit. Also, syslog is socket-activated nowadays, and corosync already depends on the network, so those are superfluous. Signed-off-by: Ferenc W��gner <wferi(a)niif.hu> --- init/dlm.service | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/init/dlm.service b/init/dlm.service index 198c8ce..598a8ae 100644 --- a/init/dlm.service +++ b/init/dlm.service @@ -1,6 +1,7 @@ [Unit] Description=dlm control daemon -After=syslog.target network.target corosync.service sys-kernel-config.mount +Requires=corosync.service sys-kernel-config.mount +After=corosync.service sys-kernel-config.mount [Service] OOMScoreAdjust=-1000

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

cluster-commits January 2016