From af9b6d7570ca9afbbc6076ab7920d8f00f7e55c1 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 29 Jun 2018 12:45:53 -0400 Subject: pNFS: Parse the results of layoutget on open even if permissions checks fail Even if the results of the permissions checks failed, we should parse the results of the layout on open call so that we can return the layout if required. Note that we also want to ignore the sequence counter for whether or not a layout recall occurred. If the recall pertained to our OPEN, then the callback will know, and will attempt to wait for us to finih processing anyway. Signed-off-by: Trond Myklebust --- include/linux/nfs_fs_sb.h | 1 - include/linux/nfs_xdr.h | 1 - 2 files changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 74ae3e1d19a0..2c18d618604e 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -28,7 +28,6 @@ struct nfs41_impl_id; struct nfs_client { refcount_t cl_count; atomic_t cl_mds_count; - seqcount_t cl_callback_count; int cl_cons_state; /* current construction state (-ve: init error) */ #define NFS_CS_READY 0 /* ready to be used */ #define NFS_CS_INITING 1 /* busy initialising */ diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 712eed156d09..3b7325cfb291 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -271,7 +271,6 @@ struct nfs4_layoutget { struct nfs4_layoutget_args args; struct nfs4_layoutget_res res; struct rpc_cred *cred; - unsigned callback_count; gfp_t gfp_flags; }; -- cgit v1.2.3 From 82b98ca566ca2af170eb0ab50cef09dd7335fa55 Mon Sep 17 00:00:00 2001 From: Sargun Dhillon Date: Thu, 5 Jul 2018 16:48:50 +0000 Subject: net/sunrpc: Make rpc_auth_create_args a const This turns rpc_auth_create_args into a const as it gets passed through the auth stack. Signed-off-by: Sargun Dhillon Signed-off-by: Anna Schumaker --- include/linux/sunrpc/auth.h | 5 +++-- net/sunrpc/auth.c | 2 +- net/sunrpc/auth_gss/auth_gss.c | 9 +++++---- net/sunrpc/auth_null.c | 2 +- net/sunrpc/auth_unix.c | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/auth.h b/include/linux/sunrpc/auth.h index d9af474a857d..58a6765c1c5e 100644 --- a/include/linux/sunrpc/auth.h +++ b/include/linux/sunrpc/auth.h @@ -125,7 +125,8 @@ struct rpc_authops { struct module *owner; rpc_authflavor_t au_flavor; /* flavor (RPC_AUTH_*) */ char * au_name; - struct rpc_auth * (*create)(struct rpc_auth_create_args *, struct rpc_clnt *); + struct rpc_auth * (*create)(const struct rpc_auth_create_args *, + struct rpc_clnt *); void (*destroy)(struct rpc_auth *); int (*hash_cred)(struct auth_cred *, unsigned int); @@ -174,7 +175,7 @@ struct rpc_cred * rpc_lookup_generic_cred(struct auth_cred *, int, gfp_t); struct rpc_cred * rpc_lookup_machine_cred(const char *service_name); int rpcauth_register(const struct rpc_authops *); int rpcauth_unregister(const struct rpc_authops *); -struct rpc_auth * rpcauth_create(struct rpc_auth_create_args *, +struct rpc_auth * rpcauth_create(const struct rpc_auth_create_args *, struct rpc_clnt *); void rpcauth_release(struct rpc_auth *); rpc_authflavor_t rpcauth_get_pseudoflavor(rpc_authflavor_t, diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index d2623b9f23d6..661e2277f468 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -253,7 +253,7 @@ rpcauth_list_flavors(rpc_authflavor_t *array, int size) EXPORT_SYMBOL_GPL(rpcauth_list_flavors); struct rpc_auth * -rpcauth_create(struct rpc_auth_create_args *args, struct rpc_clnt *clnt) +rpcauth_create(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt) { struct rpc_auth *auth; const struct rpc_authops *ops; diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index be8f103d22fd..21a19a9f0e33 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -985,7 +985,7 @@ static void gss_pipe_free(struct gss_pipe *p) * parameters based on the input flavor (which must be a pseudoflavor) */ static struct gss_auth * -gss_create_new(struct rpc_auth_create_args *args, struct rpc_clnt *clnt) +gss_create_new(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt) { rpc_authflavor_t flavor = args->pseudoflavor; struct gss_auth *gss_auth; @@ -1132,7 +1132,7 @@ gss_destroy(struct rpc_auth *auth) * (which is guaranteed to last as long as any of its descendants). */ static struct gss_auth * -gss_auth_find_or_add_hashed(struct rpc_auth_create_args *args, +gss_auth_find_or_add_hashed(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt, struct gss_auth *new) { @@ -1169,7 +1169,8 @@ out: } static struct gss_auth * -gss_create_hashed(struct rpc_auth_create_args *args, struct rpc_clnt *clnt) +gss_create_hashed(const struct rpc_auth_create_args *args, + struct rpc_clnt *clnt) { struct gss_auth *gss_auth; struct gss_auth *new; @@ -1188,7 +1189,7 @@ out: } static struct rpc_auth * -gss_create(struct rpc_auth_create_args *args, struct rpc_clnt *clnt) +gss_create(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt) { struct gss_auth *gss_auth; struct rpc_xprt_switch *xps = rcu_access_pointer(clnt->cl_xpi.xpi_xpswitch); diff --git a/net/sunrpc/auth_null.c b/net/sunrpc/auth_null.c index 75d72e109a04..4b48228ee8c7 100644 --- a/net/sunrpc/auth_null.c +++ b/net/sunrpc/auth_null.c @@ -19,7 +19,7 @@ static struct rpc_auth null_auth; static struct rpc_cred null_cred; static struct rpc_auth * -nul_create(struct rpc_auth_create_args *args, struct rpc_clnt *clnt) +nul_create(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt) { atomic_inc(&null_auth.au_count); return &null_auth; diff --git a/net/sunrpc/auth_unix.c b/net/sunrpc/auth_unix.c index dafd6b870ba3..185e56d4f9ae 100644 --- a/net/sunrpc/auth_unix.c +++ b/net/sunrpc/auth_unix.c @@ -30,7 +30,7 @@ static struct rpc_auth unix_auth; static const struct rpc_credops unix_credops; static struct rpc_auth * -unx_create(struct rpc_auth_create_args *args, struct rpc_clnt *clnt) +unx_create(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt) { dprintk("RPC: creating UNIX authenticator for client %p\n", clnt); -- cgit v1.2.3 From 016583d7030cec9b69e0d55269a5967f4ee871d2 Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Tue, 31 Jul 2018 10:10:51 -0400 Subject: sunrpc: Change rpc_print_iostats to rpc_clnt_show_stats and handle rpc_clnt clones The existing rpc_print_iostats has a few shortcomings. First, the naming is not consistent with other functions in the kernel that display stats. Second, it is really displaying stats for an rpc_clnt structure as it displays both xprt stats and per-op stats. Third, it does not handle rpc_clnt clones, which is important for the one in-kernel tree caller of this function, the NFS client's nfs_show_stats function. Fix all of the above by renaming the rpc_print_iostats to rpc_clnt_show_stats and looping through any rpc_clnt clones via cl_parent. Once this interface is fixed, this addresses a problem with NFSv4. Before this patch, the /proc/self/mountstats always showed incorrect counts for NFSv4 lease and session related opcodes such as SEQUENCE, RENEW, SETCLIENTID, CREATE_SESSION, etc. These counts were always 0 even though many ops would go over the wire. The reason for this is there are multiple rpc_clnt structures allocated for any given NFSv4 mount, and inside nfs_show_stats() we callled into rpc_print_iostats() which only handled one of them, nfs_server->client. Fix these counts by calling sunrpc's new rpc_clnt_show_stats() function, which handles cloned rpc_clnt structs and prints the stats together. Note that one side-effect of the above is that multiple mounts from the same NFS server will show identical counts in the above ops due to the fact the one rpc_clnt (representing the NFSv4 client state) is shared across mounts. Signed-off-by: Dave Wysochanski Signed-off-by: Anna Schumaker --- fs/nfs/super.c | 2 +- include/linux/sunrpc/metrics.h | 4 ++-- net/sunrpc/stats.c | 17 ++++++++++++----- 3 files changed, 15 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 5e470e233c83..bdf39fa1bfbc 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -884,7 +884,7 @@ int nfs_show_stats(struct seq_file *m, struct dentry *root) #endif seq_printf(m, "\n"); - rpc_print_iostats(m, nfss->client); + rpc_clnt_show_stats(m, nfss->client); return 0; } diff --git a/include/linux/sunrpc/metrics.h b/include/linux/sunrpc/metrics.h index 9baed7b355b2..1b3751327575 100644 --- a/include/linux/sunrpc/metrics.h +++ b/include/linux/sunrpc/metrics.h @@ -82,7 +82,7 @@ void rpc_count_iostats(const struct rpc_task *, struct rpc_iostats *); void rpc_count_iostats_metrics(const struct rpc_task *, struct rpc_iostats *); -void rpc_print_iostats(struct seq_file *, struct rpc_clnt *); +void rpc_clnt_show_stats(struct seq_file *, struct rpc_clnt *); void rpc_free_iostats(struct rpc_iostats *); #else /* CONFIG_PROC_FS */ @@ -95,7 +95,7 @@ static inline void rpc_count_iostats_metrics(const struct rpc_task *task, { } -static inline void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt) {} +static inline void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt) {} static inline void rpc_free_iostats(struct rpc_iostats *stats) {} #endif /* CONFIG_PROC_FS */ diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c index 32adddd7fb78..ffae7c2245b1 100644 --- a/net/sunrpc/stats.c +++ b/net/sunrpc/stats.c @@ -235,13 +235,12 @@ static void _print_rpc_iostats(struct seq_file *seq, struct rpc_iostats *stats, ktime_to_ms(stats->om_execute)); } -void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt) +void rpc_clnt_show_stats(struct seq_file *seq, struct rpc_clnt *clnt) { - struct rpc_iostats *stats = clnt->cl_metrics; struct rpc_xprt *xprt; unsigned int op, maxproc = clnt->cl_maxproc; - if (!stats) + if (!clnt->cl_metrics) return; seq_printf(seq, "\tRPC iostats version: %s ", RPC_IOSTATS_VERS); @@ -256,10 +255,18 @@ void rpc_print_iostats(struct seq_file *seq, struct rpc_clnt *clnt) seq_printf(seq, "\tper-op statistics\n"); for (op = 0; op < maxproc; op++) { - _print_rpc_iostats(seq, &stats[op], op, clnt->cl_procinfo); + struct rpc_iostats stats = {}; + struct rpc_clnt *next = clnt; + do { + _add_rpc_iostats(&stats, &next->cl_metrics[op]); + if (next == next->cl_parent) + break; + next = next->cl_parent; + } while (next); + _print_rpc_iostats(seq, &stats, op, clnt->cl_procinfo); } } -EXPORT_SYMBOL_GPL(rpc_print_iostats); +EXPORT_SYMBOL_GPL(rpc_clnt_show_stats); /* * Register/unregister RPC proc files -- cgit v1.2.3 From 0f90be132cbf1537d87a6a8b9e80867adac892f6 Mon Sep 17 00:00:00 2001 From: Bill Baker Date: Tue, 19 Jun 2018 16:24:58 -0500 Subject: NFSv4 client live hangs after live data migration recovery After a live data migration event at the NFS server, the client may send I/O requests to the wrong server, causing a live hang due to repeated recovery events. On the wire, this will appear as an I/O request failing with NFS4ERR_BADSESSION, followed by successful CREATE_SESSION, repeatedly. NFS4ERR_BADSSESSION is returned because the session ID being used was issued by the other server and is not valid at the old server. The failure is caused by async worker threads having cached the transport (xprt) in the rpc_task structure. After the migration recovery completes, the task is redispatched and the task resends the request to the wrong server based on the old value still present in tk_xprt. The solution is to recompute the tk_xprt field of the rpc_task structure so that the request goes to the correct server. Signed-off-by: Bill Baker Reviewed-by: Chuck Lever Tested-by: Helen Chao Fixes: fb43d17210ba ("SUNRPC: Use the multipath iterator to assign a ...") Cc: stable@vger.kernel.org # v4.9+ Signed-off-by: Anna Schumaker --- fs/nfs/nfs4proc.c | 9 ++++++++- include/linux/sunrpc/clnt.h | 1 + net/sunrpc/clnt.c | 28 ++++++++++++++++++++-------- 3 files changed, 29 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 3b28c0ac10bc..bddba460643a 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -581,8 +581,15 @@ nfs4_async_handle_exception(struct rpc_task *task, struct nfs_server *server, ret = -EIO; return ret; out_retry: - if (ret == 0) + if (ret == 0) { exception->retry = 1; + /* + * For NFS4ERR_MOVED, the client transport will need to + * be recomputed after migration recovery has completed. + */ + if (errorcode == -NFS4ERR_MOVED) + rpc_task_release_transport(task); + } return ret; } diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index 9b11b6a0978c..73d5c4a870fa 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -156,6 +156,7 @@ int rpc_switch_client_transport(struct rpc_clnt *, void rpc_shutdown_client(struct rpc_clnt *); void rpc_release_client(struct rpc_clnt *); +void rpc_task_release_transport(struct rpc_task *); void rpc_task_release_client(struct rpc_task *); int rpcb_create_local(struct net *); diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index d839c33ae7d9..0d85425b1e07 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -965,10 +965,20 @@ out: } EXPORT_SYMBOL_GPL(rpc_bind_new_program); +void rpc_task_release_transport(struct rpc_task *task) +{ + struct rpc_xprt *xprt = task->tk_xprt; + + if (xprt) { + task->tk_xprt = NULL; + xprt_put(xprt); + } +} +EXPORT_SYMBOL_GPL(rpc_task_release_transport); + void rpc_task_release_client(struct rpc_task *task) { struct rpc_clnt *clnt = task->tk_client; - struct rpc_xprt *xprt = task->tk_xprt; if (clnt != NULL) { /* Remove from client task list */ @@ -979,12 +989,14 @@ void rpc_task_release_client(struct rpc_task *task) rpc_release_client(clnt); } + rpc_task_release_transport(task); +} - if (xprt != NULL) { - task->tk_xprt = NULL; - - xprt_put(xprt); - } +static +void rpc_task_set_transport(struct rpc_task *task, struct rpc_clnt *clnt) +{ + if (!task->tk_xprt) + task->tk_xprt = xprt_iter_get_next(&clnt->cl_xpi); } static @@ -992,8 +1004,7 @@ void rpc_task_set_client(struct rpc_task *task, struct rpc_clnt *clnt) { if (clnt != NULL) { - if (task->tk_xprt == NULL) - task->tk_xprt = xprt_iter_get_next(&clnt->cl_xpi); + rpc_task_set_transport(task, clnt); task->tk_client = clnt; atomic_inc(&clnt->cl_count); if (clnt->cl_softrtry) @@ -1512,6 +1523,7 @@ call_start(struct rpc_task *task) clnt->cl_program->version[clnt->cl_vers]->counts[idx]++; clnt->cl_stats->rpccnt++; task->tk_action = call_reserve; + rpc_task_set_transport(task, clnt); } /* -- cgit v1.2.3 From cb95deea0b4aa5c7c7423f4e075a3ddcd59e710b Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 9 Jul 2018 15:13:29 -0400 Subject: NFS OFFLOAD_CANCEL xdr Signed-off-by: Olga Kornievskaia Signed-off-by: Anna Schumaker --- fs/nfs/nfs42xdr.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++ fs/nfs/nfs4proc.c | 1 + fs/nfs/nfs4xdr.c | 1 + include/linux/nfs4.h | 1 + include/linux/nfs_fs_sb.h | 1 + include/linux/nfs_xdr.h | 12 +++++++++ 6 files changed, 84 insertions(+) (limited to 'include/linux') diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c index 5966e1e7b1f5..09a540d035b8 100644 --- a/fs/nfs/nfs42xdr.c +++ b/fs/nfs/nfs42xdr.c @@ -26,6 +26,9 @@ NFS42_WRITE_RES_SIZE + \ 1 /* cr_consecutive */ + \ 1 /* cr_synchronous */) +#define encode_offload_cancel_maxsz (op_encode_hdr_maxsz + \ + XDR_QUADLEN(NFS4_STATEID_SIZE)) +#define decode_offload_cancel_maxsz (op_decode_hdr_maxsz) #define encode_deallocate_maxsz (op_encode_hdr_maxsz + \ encode_fallocate_maxsz) #define decode_deallocate_maxsz (op_decode_hdr_maxsz) @@ -75,6 +78,12 @@ decode_putfh_maxsz + \ decode_copy_maxsz + \ decode_commit_maxsz) +#define NFS4_enc_offload_cancel_sz (compound_encode_hdr_maxsz + \ + encode_putfh_maxsz + \ + encode_offload_cancel_maxsz) +#define NFS4_dec_offload_cancel_sz (compound_decode_hdr_maxsz + \ + decode_putfh_maxsz + \ + decode_offload_cancel_maxsz) #define NFS4_enc_deallocate_sz (compound_encode_hdr_maxsz + \ encode_putfh_maxsz + \ encode_deallocate_maxsz + \ @@ -145,6 +154,14 @@ static void encode_copy(struct xdr_stream *xdr, encode_uint32(xdr, 0); /* src server list */ } +static void encode_offload_cancel(struct xdr_stream *xdr, + const struct nfs42_offload_status_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_OFFLOAD_CANCEL, decode_offload_cancel_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->osa_stateid); +} + static void encode_deallocate(struct xdr_stream *xdr, const struct nfs42_falloc_args *args, struct compound_hdr *hdr) @@ -260,6 +277,25 @@ static void nfs4_xdr_enc_copy(struct rpc_rqst *req, encode_nops(&hdr); } +/* + * Encode OFFLOAD_CANEL request + */ +static void nfs4_xdr_enc_offload_cancel(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_offload_status_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->osa_seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->osa_seq_args, &hdr); + encode_putfh(xdr, args->osa_src_fh, &hdr); + encode_offload_cancel(xdr, args, &hdr); + encode_nops(&hdr); +} + /* * Encode DEALLOCATE request */ @@ -413,6 +449,12 @@ static int decode_copy(struct xdr_stream *xdr, struct nfs42_copy_res *res) return decode_copy_requirements(xdr, res); } +static int decode_offload_cancel(struct xdr_stream *xdr, + struct nfs42_offload_status_res *res) +{ + return decode_op_hdr(xdr, OP_OFFLOAD_CANCEL); +} + static int decode_deallocate(struct xdr_stream *xdr, struct nfs42_falloc_res *res) { return decode_op_hdr(xdr, OP_DEALLOCATE); @@ -512,6 +554,32 @@ out: return status; } +/* + * Decode OFFLOAD_CANCEL response + */ +static int nfs4_xdr_dec_offload_cancel(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_offload_status_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->osr_seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_offload_cancel(xdr, res); + +out: + return status; +} + /* * Decode DEALLOCATE request */ diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 124a9f97c366..b22b41eff9c0 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -9620,6 +9620,7 @@ static const struct nfs4_minor_version_ops nfs_v4_2_minor_ops = { | NFS_CAP_LGOPEN | NFS_CAP_ALLOCATE | NFS_CAP_COPY + | NFS_CAP_OFFLOAD_CANCEL | NFS_CAP_DEALLOCATE | NFS_CAP_SEEK | NFS_CAP_LAYOUTSTATS diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c index cd41d2577a04..b7bde12d8cd5 100644 --- a/fs/nfs/nfs4xdr.c +++ b/fs/nfs/nfs4xdr.c @@ -7789,6 +7789,7 @@ const struct rpc_procinfo nfs4_procedures[] = { PROC42(LAYOUTSTATS, enc_layoutstats, dec_layoutstats), PROC42(CLONE, enc_clone, dec_clone), PROC42(COPY, enc_copy, dec_copy), + PROC42(OFFLOAD_CANCEL, enc_offload_cancel, dec_offload_cancel), PROC(LOOKUPP, enc_lookupp, dec_lookupp), }; diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h index 57ffaa20d564..c44b87293229 100644 --- a/include/linux/nfs4.h +++ b/include/linux/nfs4.h @@ -527,6 +527,7 @@ enum { NFSPROC4_CLNT_LAYOUTSTATS, NFSPROC4_CLNT_CLONE, NFSPROC4_CLNT_COPY, + NFSPROC4_CLNT_OFFLOAD_CANCEL, NFSPROC4_CLNT_LOOKUPP, }; diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 2c18d618604e..fbc735f08d7e 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -255,5 +255,6 @@ struct nfs_server { #define NFS_CAP_LAYOUTSTATS (1U << 22) #define NFS_CAP_CLONE (1U << 23) #define NFS_CAP_COPY (1U << 24) +#define NFS_CAP_OFFLOAD_CANCEL (1U << 25) #endif diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 3b7325cfb291..85e928a56cef 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1403,6 +1403,18 @@ struct nfs42_copy_res { struct nfs_commitres commit_res; }; +struct nfs42_offload_status_args { + struct nfs4_sequence_args osa_seq_args; + struct nfs_fh *osa_src_fh; + nfs4_stateid osa_stateid; +}; + +struct nfs42_offload_status_res { + struct nfs4_sequence_res osr_seq_res; + uint64_t osr_count; + int osr_status; +}; + struct nfs42_seek_args { struct nfs4_sequence_args seq_args; -- cgit v1.2.3 From 67aa7444c4beb40aafedd8d2c60bbcc54987adda Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 9 Jul 2018 15:13:30 -0400 Subject: NFS COPY xdr handle async reply If server returns async reply, it must include a callback stateid, wr_callback_id in the write_response4. Signed-off-by: Olga Kornievskaia Signed-off-by: Anna Schumaker --- fs/nfs/nfs42xdr.c | 22 ++++++++++++---------- include/linux/nfs_xdr.h | 1 + 2 files changed, 13 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c index 09a540d035b8..205c3567ea08 100644 --- a/fs/nfs/nfs42xdr.c +++ b/fs/nfs/nfs42xdr.c @@ -389,21 +389,23 @@ static int decode_write_response(struct xdr_stream *xdr, struct nfs42_write_res *res) { __be32 *p; + int status, count; - p = xdr_inline_decode(xdr, 4 + 8 + 4); + p = xdr_inline_decode(xdr, 4); if (unlikely(!p)) goto out_overflow; - - /* - * We never use asynchronous mode, so warn if a server returns - * a stateid. - */ - if (unlikely(*p != 0)) { - pr_err_once("%s: server has set unrequested " - "asynchronous mode\n", __func__); + count = be32_to_cpup(p); + if (count > 1) return -EREMOTEIO; + else if (count == 1) { + status = decode_opaque_fixed(xdr, &res->stateid, + NFS4_STATEID_SIZE); + if (unlikely(status)) + goto out_overflow; } - p++; + p = xdr_inline_decode(xdr, 8 + 4); + if (unlikely(!p)) + goto out_overflow; p = xdr_decode_hyper(p, &res->count); res->verifier.committed = be32_to_cpup(p); return decode_verifier(xdr, &res->verifier.verifier); diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 85e928a56cef..06ddfa31cbef 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1391,6 +1391,7 @@ struct nfs42_copy_args { }; struct nfs42_write_res { + nfs4_stateid stateid; u64 count; struct nfs_writeverf verifier; }; -- cgit v1.2.3 From 62164f317972fcd36590578888f33a1994dda519 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 9 Jul 2018 15:13:31 -0400 Subject: NFS add support for asynchronous COPY Change xdr to always send COPY asynchronously. Keep the list copies send in a list under a server structure. Once copy is sent, it waits on a completion structure that will be signalled by the callback thread that receives CB_OFFLOAD. If CB_OFFLOAD returned an error and even if it returned partial bytes, ignore them (as we can't commit without a verifier to match) and return an error. Signed-off-by: Olga Kornievskaia Signed-off-by: Anna Schumaker --- fs/nfs/callback_proc.c | 38 +++++++++++++++++++++++++++++++- fs/nfs/client.c | 1 + fs/nfs/nfs42proc.c | 55 ++++++++++++++++++++++++++++++++++++++++++----- fs/nfs/nfs42xdr.c | 8 ++++--- include/linux/nfs_fs.h | 9 ++++++++ include/linux/nfs_fs_sb.h | 1 + include/linux/nfs_xdr.h | 1 + 7 files changed, 104 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c index d6f45bd176a9..acdda259912e 100644 --- a/fs/nfs/callback_proc.c +++ b/fs/nfs/callback_proc.c @@ -662,9 +662,45 @@ __be32 nfs4_callback_notify_lock(void *argp, void *resp, } #endif /* CONFIG_NFS_V4_1 */ #ifdef CONFIG_NFS_V4_2 -__be32 nfs4_callback_offload(void *args, void *dummy, +static void nfs4_copy_cb_args(struct nfs4_copy_state *cp_state, + struct cb_offloadargs *args) +{ + cp_state->count = args->wr_count; + cp_state->error = args->error; + if (!args->error) { + cp_state->verf.committed = args->wr_writeverf.committed; + memcpy(&cp_state->verf.verifier.data[0], + &args->wr_writeverf.verifier.data[0], + NFS4_VERIFIER_SIZE); + } +} + +__be32 nfs4_callback_offload(void *data, void *dummy, struct cb_process_state *cps) { + struct cb_offloadargs *args = data; + struct nfs_server *server; + struct nfs4_copy_state *copy; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &cps->clp->cl_superblocks, + client_link) { + spin_lock(&server->nfs_client->cl_lock); + list_for_each_entry(copy, &server->ss_copies, copies) { + if (memcmp(args->coa_stateid.other, + copy->stateid.other, + sizeof(args->coa_stateid.other))) + continue; + nfs4_copy_cb_args(copy, args); + complete(©->completion); + spin_unlock(&server->nfs_client->cl_lock); + goto out; + } + spin_unlock(&server->nfs_client->cl_lock); + } +out: + rcu_read_unlock(); + return 0; } #endif /* CONFIG_NFS_V4_2 */ diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 377a61654a88..96d5f8135eb9 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -886,6 +886,7 @@ struct nfs_server *nfs_alloc_server(void) INIT_LIST_HEAD(&server->delegations); INIT_LIST_HEAD(&server->layouts); INIT_LIST_HEAD(&server->state_owners_lru); + INIT_LIST_HEAD(&server->ss_copies); atomic_set(&server->active, 0); diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index 5f59b6f65a42..023aea8f6cf1 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -130,6 +130,37 @@ out_unlock: return err; } +static int handle_async_copy(struct nfs42_copy_res *res, + struct nfs_server *server, + struct file *src, + struct file *dst, + nfs4_stateid *src_stateid) +{ + struct nfs4_copy_state *copy; + int status = NFS4_OK; + + copy = kzalloc(sizeof(struct nfs4_copy_state), GFP_NOFS); + if (!copy) + return -ENOMEM; + memcpy(©->stateid, &res->write_res.stateid, NFS4_STATEID_SIZE); + init_completion(©->completion); + + spin_lock(&server->nfs_client->cl_lock); + list_add_tail(©->copies, &server->ss_copies); + spin_unlock(&server->nfs_client->cl_lock); + + wait_for_completion_interruptible(©->completion); + spin_lock(&server->nfs_client->cl_lock); + list_del_init(©->copies); + spin_unlock(&server->nfs_client->cl_lock); + res->write_res.count = copy->count; + memcpy(&res->write_res.verifier, ©->verf, sizeof(copy->verf)); + status = -copy->error; + + kfree(copy); + return status; +} + static ssize_t _nfs42_proc_copy(struct file *src, struct nfs_lock_context *src_lock, struct file *dst, @@ -168,9 +199,13 @@ static ssize_t _nfs42_proc_copy(struct file *src, if (status) return status; - res->commit_res.verf = kzalloc(sizeof(struct nfs_writeverf), GFP_NOFS); - if (!res->commit_res.verf) - return -ENOMEM; + res->commit_res.verf = NULL; + if (args->sync) { + res->commit_res.verf = + kzalloc(sizeof(struct nfs_writeverf), GFP_NOFS); + if (!res->commit_res.verf) + return -ENOMEM; + } status = nfs4_call_sync(server->client, server, &msg, &args->seq_args, &res->seq_res, 0); if (status == -ENOTSUPP) @@ -178,18 +213,27 @@ static ssize_t _nfs42_proc_copy(struct file *src, if (status) goto out; - if (nfs_write_verifier_cmp(&res->write_res.verifier.verifier, + if (args->sync && + nfs_write_verifier_cmp(&res->write_res.verifier.verifier, &res->commit_res.verf->verifier)) { status = -EAGAIN; goto out; } + if (!res->synchronous) { + status = handle_async_copy(res, server, src, dst, + &args->src_stateid); + if (status) + return status; + } + truncate_pagecache_range(dst_inode, pos_dst, pos_dst + res->write_res.count); status = res->write_res.count; out: - kfree(res->commit_res.verf); + if (args->sync) + kfree(res->commit_res.verf); return status; } @@ -206,6 +250,7 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src, .dst_fh = NFS_FH(file_inode(dst)), .dst_pos = pos_dst, .count = count, + .sync = false, }; struct nfs42_copy_res res; struct nfs4_exception src_exception = { diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c index 205c3567ea08..69f72ed2bf87 100644 --- a/fs/nfs/nfs42xdr.c +++ b/fs/nfs/nfs42xdr.c @@ -150,7 +150,7 @@ static void encode_copy(struct xdr_stream *xdr, encode_uint64(xdr, args->count); encode_uint32(xdr, 1); /* consecutive = true */ - encode_uint32(xdr, 1); /* synchronous = true */ + encode_uint32(xdr, args->sync); encode_uint32(xdr, 0); /* src server list */ } @@ -273,7 +273,8 @@ static void nfs4_xdr_enc_copy(struct rpc_rqst *req, encode_savefh(xdr, &hdr); encode_putfh(xdr, args->dst_fh, &hdr); encode_copy(xdr, args, &hdr); - encode_copy_commit(xdr, args, &hdr); + if (args->sync) + encode_copy_commit(xdr, args, &hdr); encode_nops(&hdr); } @@ -551,7 +552,8 @@ static int nfs4_xdr_dec_copy(struct rpc_rqst *rqstp, status = decode_copy(xdr, res); if (status) goto out; - status = decode_commit(xdr, &res->commit_res); + if (res->commit_res.verf) + status = decode_commit(xdr, &res->commit_res); out: return status; } diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 2f129bbfaae8..645ad8e342f6 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -185,6 +185,15 @@ struct nfs_inode { struct inode vfs_inode; }; +struct nfs4_copy_state { + struct list_head copies; + nfs4_stateid stateid; + struct completion completion; + uint64_t count; + struct nfs_writeverf verf; + int error; +}; + /* * Access bit flags */ diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index fbc735f08d7e..f88952d7b9fb 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -208,6 +208,7 @@ struct nfs_server { struct list_head state_owners_lru; struct list_head layouts; struct list_head delegations; + struct list_head ss_copies; unsigned long mig_gen; unsigned long mig_status; diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 06ddfa31cbef..bd1c889a9ed9 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1388,6 +1388,7 @@ struct nfs42_copy_args { u64 dst_pos; u64 count; + bool sync; }; struct nfs42_write_res { -- cgit v1.2.3 From bc0c9079b48ddcf1f8a6e1aaa277288b263c78d8 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 9 Jul 2018 15:13:32 -0400 Subject: NFS handle COPY reply CB_OFFLOAD call race It's possible that server replies back with CB_OFFLOAD call and COPY reply at the same time such that client will process CB_OFFLOAD before reply to COPY. For that keep a list of pending callback stateids received and then before waiting on completion check the pending list. Cleanup any pending copies on the client shutdown. Signed-off-by: Olga Kornievskaia Signed-off-by: Anna Schumaker --- fs/nfs/callback_proc.c | 17 ++++++++++++++--- fs/nfs/nfs42proc.c | 22 ++++++++++++++++++++-- fs/nfs/nfs4client.c | 15 +++++++++++++++ include/linux/nfs_fs_sb.h | 1 + 4 files changed, 50 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c index acdda259912e..cd733649646b 100644 --- a/fs/nfs/callback_proc.c +++ b/fs/nfs/callback_proc.c @@ -681,11 +681,12 @@ __be32 nfs4_callback_offload(void *data, void *dummy, struct cb_offloadargs *args = data; struct nfs_server *server; struct nfs4_copy_state *copy; + bool found = false; + spin_lock(&cps->clp->cl_lock); rcu_read_lock(); list_for_each_entry_rcu(server, &cps->clp->cl_superblocks, client_link) { - spin_lock(&server->nfs_client->cl_lock); list_for_each_entry(copy, &server->ss_copies, copies) { if (memcmp(args->coa_stateid.other, copy->stateid.other, @@ -693,13 +694,23 @@ __be32 nfs4_callback_offload(void *data, void *dummy, continue; nfs4_copy_cb_args(copy, args); complete(©->completion); - spin_unlock(&server->nfs_client->cl_lock); + found = true; goto out; } - spin_unlock(&server->nfs_client->cl_lock); } out: rcu_read_unlock(); + if (!found) { + copy = kzalloc(sizeof(struct nfs4_copy_state), GFP_NOFS); + if (!copy) { + spin_unlock(&cps->clp->cl_lock); + return htonl(NFS4ERR_SERVERFAULT); + } + memcpy(©->stateid, &args->coa_stateid, NFS4_STATEID_SIZE); + nfs4_copy_cb_args(copy, args); + list_add_tail(©->copies, &cps->clp->pending_cb_stateids); + } + spin_unlock(&cps->clp->cl_lock); return 0; } diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index 023aea8f6cf1..c7d31f72070e 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -138,14 +138,31 @@ static int handle_async_copy(struct nfs42_copy_res *res, { struct nfs4_copy_state *copy; int status = NFS4_OK; + bool found_pending = false; + + spin_lock(&server->nfs_client->cl_lock); + list_for_each_entry(copy, &server->nfs_client->pending_cb_stateids, + copies) { + if (memcmp(&res->write_res.stateid, ©->stateid, + NFS4_STATEID_SIZE)) + continue; + found_pending = true; + list_del(©->copies); + break; + } + if (found_pending) { + spin_unlock(&server->nfs_client->cl_lock); + goto out; + } copy = kzalloc(sizeof(struct nfs4_copy_state), GFP_NOFS); - if (!copy) + if (!copy) { + spin_unlock(&server->nfs_client->cl_lock); return -ENOMEM; + } memcpy(©->stateid, &res->write_res.stateid, NFS4_STATEID_SIZE); init_completion(©->completion); - spin_lock(&server->nfs_client->cl_lock); list_add_tail(©->copies, &server->ss_copies); spin_unlock(&server->nfs_client->cl_lock); @@ -153,6 +170,7 @@ static int handle_async_copy(struct nfs42_copy_res *res, spin_lock(&server->nfs_client->cl_lock); list_del_init(©->copies); spin_unlock(&server->nfs_client->cl_lock); +out: res->write_res.count = copy->count; memcpy(&res->write_res.verifier, ©->verf, sizeof(copy->verf)); status = -copy->error; diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index d7124fb12041..146e30862234 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -156,9 +156,23 @@ nfs4_shutdown_ds_clients(struct nfs_client *clp) } } +static void +nfs4_cleanup_callback(struct nfs_client *clp) +{ + struct nfs4_copy_state *cp_state; + + while (!list_empty(&clp->pending_cb_stateids)) { + cp_state = list_entry(clp->pending_cb_stateids.next, + struct nfs4_copy_state, copies); + list_del(&cp_state->copies); + kfree(cp_state); + } +} + void nfs41_shutdown_client(struct nfs_client *clp) { if (nfs4_has_session(clp)) { + nfs4_cleanup_callback(clp); nfs4_shutdown_ds_clients(clp); nfs4_destroy_session(clp->cl_session); nfs4_destroy_clientid(clp); @@ -202,6 +216,7 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init) #if IS_ENABLED(CONFIG_NFS_V4_1) init_waitqueue_head(&clp->cl_lock_waitq); #endif + INIT_LIST_HEAD(&clp->pending_cb_stateids); return clp; error: diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index f88952d7b9fb..bf39d9c92201 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -121,6 +121,7 @@ struct nfs_client { #endif struct net *cl_net; + struct list_head pending_cb_stateids; }; /* -- cgit v1.2.3 From e4648aa4f98a87cf0a83f73a5864cede073053a0 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Mon, 13 Aug 2018 15:33:01 -0400 Subject: NFS recover from destination server reboot for copies Mark the destination state to indicate a server-side copy is happening. On detecting a reboot and recovering open state check if any state is engaged in a server-side copy, if so, find the copy and mark it and then signal the waiting thread. Upon wakeup, if copy was marked then propage EAGAIN to the nfsd_copy_file_range and restart the copy from scratch. Signed-off-by: Olga Kornievskaia Signed-off-by: Anna Schumaker --- fs/nfs/nfs42proc.c | 16 +++++++++++++--- fs/nfs/nfs4_fs.h | 3 +++ fs/nfs/nfs4file.c | 9 +++++++-- fs/nfs/nfs4state.c | 16 ++++++++++++++++ include/linux/nfs_fs.h | 2 ++ 5 files changed, 41 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index d158526d1044..ac5b784a1de0 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -140,6 +140,7 @@ static int handle_async_copy(struct nfs42_copy_res *res, struct nfs4_copy_state *copy; int status = NFS4_OK; bool found_pending = false; + struct nfs_open_context *ctx = nfs_file_open_context(dst); spin_lock(&server->nfs_client->cl_lock); list_for_each_entry(copy, &server->nfs_client->pending_cb_stateids, @@ -163,6 +164,7 @@ static int handle_async_copy(struct nfs42_copy_res *res, } memcpy(©->stateid, &res->write_res.stateid, NFS4_STATEID_SIZE); init_completion(©->completion); + copy->parent_state = ctx->state; list_add_tail(©->copies, &server->ss_copies); spin_unlock(&server->nfs_client->cl_lock); @@ -172,15 +174,20 @@ static int handle_async_copy(struct nfs42_copy_res *res, list_del_init(©->copies); spin_unlock(&server->nfs_client->cl_lock); if (status == -ERESTARTSYS) { - nfs42_do_offload_cancel_async(dst, ©->stateid); - kfree(copy); - return status; + goto out_cancel; + } else if (copy->flags) { + status = -EAGAIN; + goto out_cancel; } out: res->write_res.count = copy->count; memcpy(&res->write_res.verifier, ©->verf, sizeof(copy->verf)); status = -copy->error; + kfree(copy); + return status; +out_cancel: + nfs42_do_offload_cancel_async(dst, ©->stateid); kfree(copy); return status; } @@ -254,6 +261,9 @@ static ssize_t _nfs42_proc_copy(struct file *src, if (!res->commit_res.verf) return -ENOMEM; } + set_bit(NFS_CLNT_DST_SSC_COPY_STATE, + &dst_lock->open_context->state->flags); + status = nfs4_call_sync(server->client, server, &msg, &args->seq_args, &res->seq_res, 0); if (status == -ENOTSUPP) diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h index e87d414c40d0..542b2fce0447 100644 --- a/fs/nfs/nfs4_fs.h +++ b/fs/nfs/nfs4_fs.h @@ -163,6 +163,9 @@ enum { NFS_STATE_RECOVERY_FAILED, /* OPEN stateid state recovery failed */ NFS_STATE_MAY_NOTIFY_LOCK, /* server may CB_NOTIFY_LOCK */ NFS_STATE_CHANGE_WAIT, /* A state changing operation is outstanding */ +#ifdef CONFIG_NFS_V4_2 + NFS_CLNT_DST_SSC_COPY_STATE, /* dst server open state on client*/ +#endif /* CONFIG_NFS_V4_2 */ }; struct nfs4_state { diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c index 6c03d6b570b2..4288a6ecaf75 100644 --- a/fs/nfs/nfs4file.c +++ b/fs/nfs/nfs4file.c @@ -133,10 +133,15 @@ static ssize_t nfs4_copy_file_range(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, size_t count, unsigned int flags) { + ssize_t ret; + if (file_inode(file_in) == file_inode(file_out)) return -EINVAL; - - return nfs42_proc_copy(file_in, pos_in, file_out, pos_out, count); +retry: + ret = nfs42_proc_copy(file_in, pos_in, file_out, pos_out, count); + if (ret == -EAGAIN) + goto retry; + return ret; } static loff_t nfs4_file_llseek(struct file *filep, loff_t offset, int whence) diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index f1b27e895a94..b6882e09d0f4 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -1589,6 +1589,22 @@ restart: } clear_bit(NFS_STATE_RECLAIM_NOGRACE, &state->flags); +#ifdef CONFIG_NFS_V4_2 + if (test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags)) { + struct nfs4_copy_state *copy; + + spin_lock(&sp->so_server->nfs_client->cl_lock); + list_for_each_entry(copy, &sp->so_server->ss_copies, copies) { + if (memcmp(&state->stateid.other, ©->parent_state->stateid.other, NFS4_STATEID_SIZE)) + continue; + copy->flags = 1; + complete(©->completion); + printk("AGLO: server rebooted waking up the copy\n"); + break; + } + spin_unlock(&sp->so_server->nfs_client->cl_lock); + } +#endif /* CONFIG_NFS_V4_2 */ nfs4_put_open_state(state); spin_lock(&sp->so_lock); goto restart; diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 645ad8e342f6..a0831e9d19c9 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -192,6 +192,8 @@ struct nfs4_copy_state { uint64_t count; struct nfs_writeverf verf; int error; + int flags; + struct nfs4_state *parent_state; }; /* -- cgit v1.2.3