As I'm sure you've notice the 15sec delay on all NFS mounts. This is because the kernel now tries to see if it should use Kerberos in the mount setup. If Kerberos is not set up (aka rpc.gssd is not running) it takes the kernel 15sec to figure that out (aka the upcall times out).
The current workaround is to start rpc.gssd, but again, if Kerberos is not set up, rpc.gssd spews tons of message in /var/log/messages for every mount. Plus putting a user level daemon in the mount path is and never was a good idea... IMHO...
Jeff Layton (jlayton@redhat.com) has come up with some kernel patches what will detect whether the rpc.gssd daemon is or is not running. When rpc.gssd is not running the mount continues with no delay. I've ported them to both F19 and F20.
Now, These patches are working their way upstream atm and the final version could be slightly different than these... I doubt it... But until then... The 15 sec delay will be dead! Thank very much Jeff Layton!!!
steved.
rpc.gssd will naturally hold open any pipe named */clnt*/gssd that shows up under rpc_pipefs. That behavior gives us a reliable mechanism to tell whether it's actually running or not.
Create a new toplevel "gssd" directory in rpc_pipefs when it's mounted. Under that directory create another directory called "clntXX", and then within that a pipe called "gssd".
We'll never send an upcall along that pipe, and any downcall written to it will just return -EINVAL.
Signed-off-by: Jeff Layton jlayton@redhat.com Signed-off-by: Trond Myklebust Trond.Myklebust@netapp.com --- include/linux/sunrpc/rpc_pipe_fs.h | 3 +- net/sunrpc/netns.h | 1 + net/sunrpc/rpc_pipe.c | 93 ++++++++++++++++++++++++++++++++++- net/sunrpc/sunrpc_syms.c | 8 +++- 4 files changed, 100 insertions(+), 5 deletions(-)
diff -up linux-3.11.9-200.fc19.x86_64/include/linux/sunrpc/rpc_pipe_fs.h.orig linux-3.11.9-200.fc19.x86_64/include/linux/sunrpc/rpc_pipe_fs.h --- linux-3.11.9-200.fc19.x86_64/include/linux/sunrpc/rpc_pipe_fs.h.orig 2013-09-02 16:46:10.000000000 -0400 +++ linux-3.11.9-200.fc19.x86_64/include/linux/sunrpc/rpc_pipe_fs.h 2013-11-21 10:11:17.893026000 -0500 @@ -64,7 +64,8 @@ enum {
extern struct dentry *rpc_d_lookup_sb(const struct super_block *sb, const unsigned char *dir_name); -extern void rpc_pipefs_init_net(struct net *net); +extern int rpc_pipefs_init_net(struct net *net); +extern void rpc_pipefs_exit_net(struct net *net); extern struct super_block *rpc_get_sb_net(const struct net *net); extern void rpc_put_sb_net(const struct net *net);
diff -up linux-3.11.9-200.fc19.x86_64/net/sunrpc/netns.h.orig linux-3.11.9-200.fc19.x86_64/net/sunrpc/netns.h --- linux-3.11.9-200.fc19.x86_64/net/sunrpc/netns.h.orig 2013-09-02 16:46:10.000000000 -0400 +++ linux-3.11.9-200.fc19.x86_64/net/sunrpc/netns.h 2013-11-21 10:11:17.897029000 -0500 @@ -14,6 +14,7 @@ struct sunrpc_net { struct cache_detail *rsi_cache;
struct super_block *pipefs_sb; + struct rpc_pipe *gssd_dummy; struct mutex pipefs_sb_lock;
struct list_head all_clients; diff -up linux-3.11.9-200.fc19.x86_64/net/sunrpc/rpc_pipe.c.orig linux-3.11.9-200.fc19.x86_64/net/sunrpc/rpc_pipe.c --- linux-3.11.9-200.fc19.x86_64/net/sunrpc/rpc_pipe.c.orig 2013-09-02 16:46:10.000000000 -0400 +++ linux-3.11.9-200.fc19.x86_64/net/sunrpc/rpc_pipe.c 2013-11-21 10:11:17.903026000 -0500 @@ -38,7 +38,7 @@ #define NET_NAME(net) ((net == &init_net) ? " (init_net)" : "")
static struct file_system_type rpc_pipe_fs_type; - +static const struct rpc_pipe_ops gssd_dummy_pipe_ops;
static struct kmem_cache *rpc_inode_cachep __read_mostly;
@@ -1019,6 +1019,7 @@ enum { RPCAUTH_nfsd4_cb, RPCAUTH_cache, RPCAUTH_nfsd, + RPCAUTH_gssd, RPCAUTH_RootEOF };
@@ -1055,6 +1056,10 @@ static const struct rpc_filelist files[] .name = "nfsd", .mode = S_IFDIR | S_IRUGO | S_IXUGO, }, + [RPCAUTH_gssd] = { + .name = "gssd", + .mode = S_IFDIR | S_IRUGO | S_IXUGO, + }, };
/* @@ -1068,13 +1073,25 @@ struct dentry *rpc_d_lookup_sb(const str } EXPORT_SYMBOL_GPL(rpc_d_lookup_sb);
-void rpc_pipefs_init_net(struct net *net) +int rpc_pipefs_init_net(struct net *net) { struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
+ sn->gssd_dummy = rpc_mkpipe_data(&gssd_dummy_pipe_ops, 0); + if (IS_ERR(sn->gssd_dummy)) + return PTR_ERR(sn->gssd_dummy); + mutex_init(&sn->pipefs_sb_lock); sn->gssd_running = 1; sn->pipe_version = -1; + return 0; +} + +void rpc_pipefs_exit_net(struct net *net) +{ + struct sunrpc_net *sn = net_generic(net, sunrpc_net_id); + + rpc_destroy_pipe_data(sn->gssd_dummy); }
/* @@ -1104,11 +1121,73 @@ void rpc_put_sb_net(const struct net *ne } EXPORT_SYMBOL_GPL(rpc_put_sb_net);
+static const struct rpc_filelist gssd_dummy_clnt_dir[] = { + [0] = { + .name = "clntXX", + .mode = S_IFDIR | S_IRUGO | S_IXUGO, + }, +}; + +static ssize_t +dummy_downcall(struct file *filp, const char __user *src, size_t len) +{ + return -EINVAL; +} + +static const struct rpc_pipe_ops gssd_dummy_pipe_ops = { + .upcall = rpc_pipe_generic_upcall, + .downcall = dummy_downcall, +}; + +/** + * rpc_gssd_dummy_populate - create a dummy gssd pipe + * @root: root of the rpc_pipefs filesystem + * @pipe_data: pipe data created when netns is initialized + * + * Create a dummy set of directories and a pipe that gssd can hold open to + * indicate that it is up and running. + */ +static struct dentry * +rpc_gssd_dummy_populate(struct dentry *root, struct rpc_pipe *pipe_data) +{ + int ret = 0; + struct dentry *gssd_dentry; + struct dentry *clnt_dentry = NULL; + struct dentry *pipe_dentry = NULL; + struct qstr q = QSTR_INIT(files[RPCAUTH_gssd].name, + strlen(files[RPCAUTH_gssd].name)); + + /* We should never get this far if "gssd" doesn't exist */ + gssd_dentry = d_hash_and_lookup(root, &q); + if (!gssd_dentry) + return ERR_PTR(-ENOENT); + + ret = rpc_populate(gssd_dentry, gssd_dummy_clnt_dir, 0, 1, NULL); + if (ret) { + pipe_dentry = ERR_PTR(ret); + goto out; + } + + q.name = gssd_dummy_clnt_dir[0].name; + q.len = strlen(gssd_dummy_clnt_dir[0].name); + clnt_dentry = d_hash_and_lookup(gssd_dentry, &q); + if (!clnt_dentry) { + pipe_dentry = ERR_PTR(-ENOENT); + goto out; + } + + pipe_dentry = rpc_mkpipe_dentry(clnt_dentry, "gssd", NULL, pipe_data); +out: + dput(clnt_dentry); + dput(gssd_dentry); + return pipe_dentry; +} + static int rpc_fill_super(struct super_block *sb, void *data, int silent) { struct inode *inode; - struct dentry *root; + struct dentry *root, *gssd_dentry; struct net *net = data; struct sunrpc_net *sn = net_generic(net, sunrpc_net_id); int err; @@ -1126,6 +1205,13 @@ rpc_fill_super(struct super_block *sb, v return -ENOMEM; if (rpc_populate(root, files, RPCAUTH_lockd, RPCAUTH_RootEOF, NULL)) return -ENOMEM; + + gssd_dentry = rpc_gssd_dummy_populate(root, sn->gssd_dummy); + if (IS_ERR(gssd_dentry)) { + __rpc_depopulate(root, files, RPCAUTH_lockd, RPCAUTH_RootEOF); + return PTR_ERR(gssd_dentry); + } + dprintk("RPC: sending pipefs MOUNT notification for net %p%s\n", net, NET_NAME(net)); mutex_lock(&sn->pipefs_sb_lock); @@ -1140,6 +1226,7 @@ rpc_fill_super(struct super_block *sb, v return 0;
err_depopulate: + dput(gssd_dentry); blocking_notifier_call_chain(&rpc_pipefs_notifier_list, RPC_PIPEFS_UMOUNT, sb); diff -up linux-3.11.9-200.fc19.x86_64/net/sunrpc/sunrpc_syms.c.orig linux-3.11.9-200.fc19.x86_64/net/sunrpc/sunrpc_syms.c --- linux-3.11.9-200.fc19.x86_64/net/sunrpc/sunrpc_syms.c.orig 2013-09-02 16:46:10.000000000 -0400 +++ linux-3.11.9-200.fc19.x86_64/net/sunrpc/sunrpc_syms.c 2013-11-21 10:11:17.908026000 -0500 @@ -44,12 +44,17 @@ static __net_init int sunrpc_init_net(st if (err) goto err_unixgid;
- rpc_pipefs_init_net(net); + err = rpc_pipefs_init_net(net); + if (err) + goto err_pipefs; + INIT_LIST_HEAD(&sn->all_clients); spin_lock_init(&sn->rpc_client_lock); spin_lock_init(&sn->rpcb_clnt_lock); return 0;
+err_pipefs: + unix_gid_cache_destroy(net); err_unixgid: ip_map_cache_destroy(net); err_ipmap: @@ -60,6 +65,7 @@ err_proc:
static __net_exit void sunrpc_exit_net(struct net *net) { + rpc_pipefs_exit_net(net); unix_gid_cache_destroy(net); ip_map_cache_destroy(net); rpc_proc_exit(net);
Now that we have a more reliable method to tell if gssd is running, we can replace the sn->gssd_running flag with a function that will query to see if it's up and running.
There's also no need to attempt an upcall that we know will fail, so just return -EACCES if gssd isn't running. Finally, fix the warn_gss() message not to claim that that the upcall timed out since we don't necesarily perform one now when gssd isn't running, and remove the extraneous newline from the message.
Signed-off-by: Jeff Layton jlayton@redhat.com Signed-off-by: Trond Myklebust Trond.Myklebust@netapp.com --- include/linux/sunrpc/rpc_pipe_fs.h | 2 ++ net/sunrpc/auth_gss/auth_gss.c | 17 +++++++---------- net/sunrpc/netns.h | 2 -- net/sunrpc/rpc_pipe.c | 14 ++++++++++---- 4 files changed, 19 insertions(+), 16 deletions(-)
diff -up linux-3.11.9-200.fc19.x86_64/include/linux/sunrpc/rpc_pipe_fs.h.orig linux-3.11.9-200.fc19.x86_64/include/linux/sunrpc/rpc_pipe_fs.h --- linux-3.11.9-200.fc19.x86_64/include/linux/sunrpc/rpc_pipe_fs.h.orig 2013-11-21 10:11:17.893026000 -0500 +++ linux-3.11.9-200.fc19.x86_64/include/linux/sunrpc/rpc_pipe_fs.h 2013-11-21 10:14:17.709348000 -0500 @@ -94,5 +94,7 @@ extern int rpc_unlink(struct dentry *); extern int register_rpc_pipefs(void); extern void unregister_rpc_pipefs(void);
+extern bool gssd_running(struct net *net); + #endif #endif diff -up linux-3.11.9-200.fc19.x86_64/net/sunrpc/auth_gss/auth_gss.c.orig linux-3.11.9-200.fc19.x86_64/net/sunrpc/auth_gss/auth_gss.c --- linux-3.11.9-200.fc19.x86_64/net/sunrpc/auth_gss/auth_gss.c.orig 2013-09-02 16:46:10.000000000 -0400 +++ linux-3.11.9-200.fc19.x86_64/net/sunrpc/auth_gss/auth_gss.c 2013-11-21 10:18:33.681923000 -0500 @@ -507,8 +507,7 @@ static void warn_gssd(void) unsigned long now = jiffies;
if (time_after(now, ratelimit)) { - printk(KERN_WARNING "RPC: AUTH_GSS upcall timed out.\n" - "Please check user daemon is running.\n"); + pr_warn("RPC: AUTH_GSS upcall failed. Please check user daemon is running.\n"); ratelimit = now + 15*HZ; } } @@ -571,7 +570,6 @@ gss_create_upcall(struct gss_auth *gss_a struct rpc_pipe *pipe; struct rpc_cred *cred = &gss_cred->gc_base; struct gss_upcall_msg *gss_msg; - unsigned long timeout; DEFINE_WAIT(wait); int err;
@@ -579,17 +577,16 @@ gss_create_upcall(struct gss_auth *gss_a __func__, from_kuid(&init_user_ns, cred->cr_uid)); retry: err = 0; - /* Default timeout is 15s unless we know that gssd is not running */ - timeout = 15 * HZ; - if (!sn->gssd_running) - timeout = HZ >> 2; + /* if gssd is down, just skip upcalling altogether */ + if (!gssd_running(net)) { + warn_gssd(); + return -EACCES; + } gss_msg = gss_setup_upcall(gss_auth->client, gss_auth, cred); if (PTR_ERR(gss_msg) == -EAGAIN) { err = wait_event_interruptible_timeout(pipe_version_waitqueue, - sn->pipe_version >= 0, timeout); + sn->pipe_version >= 0, 15 * HZ); if (sn->pipe_version < 0) { - if (err == 0) - sn->gssd_running = 0; warn_gssd(); err = -EACCES; } diff -up linux-3.11.9-200.fc19.x86_64/net/sunrpc/netns.h.orig linux-3.11.9-200.fc19.x86_64/net/sunrpc/netns.h --- linux-3.11.9-200.fc19.x86_64/net/sunrpc/netns.h.orig 2013-11-21 10:11:17.897029000 -0500 +++ linux-3.11.9-200.fc19.x86_64/net/sunrpc/netns.h 2013-11-21 10:14:17.722351000 -0500 @@ -33,8 +33,6 @@ struct sunrpc_net { int pipe_version; atomic_t pipe_users; struct proc_dir_entry *use_gssp_proc; - - unsigned int gssd_running; };
extern int sunrpc_net_id; diff -up linux-3.11.9-200.fc19.x86_64/net/sunrpc/rpc_pipe.c.orig linux-3.11.9-200.fc19.x86_64/net/sunrpc/rpc_pipe.c --- linux-3.11.9-200.fc19.x86_64/net/sunrpc/rpc_pipe.c.orig 2013-11-21 10:11:17.903026000 -0500 +++ linux-3.11.9-200.fc19.x86_64/net/sunrpc/rpc_pipe.c 2013-11-21 10:14:17.727348000 -0500 @@ -216,14 +216,11 @@ rpc_destroy_inode(struct inode *inode) static int rpc_pipe_open(struct inode *inode, struct file *filp) { - struct net *net = inode->i_sb->s_fs_info; - struct sunrpc_net *sn = net_generic(net, sunrpc_net_id); struct rpc_pipe *pipe; int first_open; int res = -ENXIO;
mutex_lock(&inode->i_mutex); - sn->gssd_running = 1; pipe = RPC_I(inode)->pipe; if (pipe == NULL) goto out; @@ -1082,7 +1079,6 @@ int rpc_pipefs_init_net(struct net *net) return PTR_ERR(sn->gssd_dummy);
mutex_init(&sn->pipefs_sb_lock); - sn->gssd_running = 1; sn->pipe_version = -1; return 0; } @@ -1236,6 +1232,16 @@ err_depopulate: return err; }
+bool +gssd_running(struct net *net) +{ + struct sunrpc_net *sn = net_generic(net, sunrpc_net_id); + struct rpc_pipe *pipe = sn->gssd_dummy; + + return pipe->nreaders || pipe->nwriters; +} +EXPORT_SYMBOL_GPL(gssd_running); + static struct dentry * rpc_mount(struct file_system_type *fs_type, int flags, const char *dev_name, void *data)
Currently, the client will attempt to use krb5i in the SETCLIENTID call even if rpc.gssd isn't running. When that fails, it'll then fall back to RPC_AUTH_UNIX. This introduced a delay when mounting if rpc.gssd isn't running, and causes warning messages to pop up in the ring buffer.
Check to see if rpc.gssd is running before even attempting to use krb5i auth, and just silently skip trying to do so if it isn't. In the event that the admin is actually trying to mount with krb5*, it will still fail at a later stage of the mount attempt.
Signed-off-by: Jeff Layton jlayton@redhat.com Signed-off-by: Trond Myklebust Trond.Myklebust@netapp.com --- fs/nfs/nfs4client.c | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-)
diff -up linux-3.11.9-200.fc19.x86_64/fs/nfs/nfs4client.c.orig linux-3.11.9-200.fc19.x86_64/fs/nfs/nfs4client.c --- linux-3.11.9-200.fc19.x86_64/fs/nfs/nfs4client.c.orig 2013-09-02 16:46:10.000000000 -0400 +++ linux-3.11.9-200.fc19.x86_64/fs/nfs/nfs4client.c 2013-11-21 10:20:27.288286000 -0500 @@ -10,6 +10,7 @@ #include <linux/sunrpc/auth.h> #include <linux/sunrpc/xprt.h> #include <linux/sunrpc/bc_xprt.h> +#include <linux/sunrpc/rpc_pipe_fs.h> #include "internal.h" #include "callback.h" #include "delegation.h" @@ -206,7 +207,11 @@ struct nfs_client *nfs4_init_client(stru if (clp->cl_minorversion != 0) __set_bit(NFS_CS_INFINITE_SLOTS, &clp->cl_flags); __set_bit(NFS_CS_DISCRTRY, &clp->cl_flags); - error = nfs_create_rpc_client(clp, timeparms, RPC_AUTH_GSS_KRB5I); + + error = -EINVAL; + if (gssd_running(clp->cl_net)) + error = nfs_create_rpc_client(clp, timeparms, + RPC_AUTH_GSS_KRB5I); if (error == -EINVAL) error = nfs_create_rpc_client(clp, timeparms, RPC_AUTH_UNIX); if (error < 0)
On Thu, Nov 21, 2013 at 11:57 AM, Steve Dickson SteveD@redhat.com wrote:
As I'm sure you've notice the 15sec delay on all NFS mounts. This is because the kernel now tries to see if it should use Kerberos in the mount setup. If Kerberos is not set up (aka rpc.gssd is not running) it takes the kernel 15sec to figure that out (aka the upcall times out).
The current workaround is to start rpc.gssd, but again, if Kerberos is not set up, rpc.gssd spews tons of message in /var/log/messages for every mount. Plus putting a user level daemon in the mount path is and never was a good idea... IMHO...
Jeff Layton (jlayton@redhat.com) has come up with some kernel patches what will detect whether the rpc.gssd daemon is or is not running. When rpc.gssd is not running the mount continues with no delay. I've ported them to both F19 and F20.
Thanks Jeff! Those are both at 3.11.y right now, but will be moving to 3.12 soonish. The patches should mostly apply to 3.12 as well, correct?
Now, These patches are working their way upstream atm and the final version could be slightly different than these... I doubt it... But until then... The 15 sec delay will be dead! Thank very much Jeff Layton!!!
Given they all have Trond's signoff, I'm guessing they're pretty much set. We'll look at getting them in shortly.
josh
On 21/11/13 13:06, Josh Boyer wrote:
On Thu, Nov 21, 2013 at 11:57 AM, Steve Dickson SteveD@redhat.com wrote:
As I'm sure you've notice the 15sec delay on all NFS mounts. This is because the kernel now tries to see if it should use Kerberos in the mount setup. If Kerberos is not set up (aka rpc.gssd is not running) it takes the kernel 15sec to figure that out (aka the upcall times out).
The current workaround is to start rpc.gssd, but again, if Kerberos is not set up, rpc.gssd spews tons of message in /var/log/messages for every mount. Plus putting a user level daemon in the mount path is and never was a good idea... IMHO...
Jeff Layton (jlayton@redhat.com) has come up with some kernel patches what will detect whether the rpc.gssd daemon is or is not running. When rpc.gssd is not running the mount continues with no delay. I've ported them to both F19 and F20.
Thanks Jeff! Those are both at 3.11.y right now, but will be moving to 3.12 soonish. The patches should mostly apply to 3.12 as well, correct?
In theory... ;-) Let me know if they don't...
Now, These patches are working their way upstream atm and the final version could be slightly different than these... I doubt it... But until then... The 15 sec delay will be dead! Thank very much Jeff Layton!!!
Given they all have Trond's signoff, I'm guessing they're pretty much set. We'll look at getting them in shortly.
You never know... ;-) I'll keep an eye out...
steved.
josh
kernel@lists.fedoraproject.org