diff --git a/sys/fs/nfs/nfs.h b/sys/fs/nfs/nfs.h index 4e79d3afbd2f..7212a685d902 100644 --- a/sys/fs/nfs/nfs.h +++ b/sys/fs/nfs/nfs.h @@ -747,6 +747,7 @@ struct nfsrv_descript { #define ND_ERELOOKUP 0x40000000000 #define ND_MACHCRED 0x80000000000 #define ND_CANEXTPG 0x100000000000 +#define ND_RDMA 0x200000000000 /* * ND_GSS should be the "or" of all GSS type authentications. diff --git a/sys/fs/nfsserver/nfs_nfsdkrpc.c b/sys/fs/nfsserver/nfs_nfsdkrpc.c index 16c21b4e1607..20b76d6152cd 100644 --- a/sys/fs/nfsserver/nfs_nfsdkrpc.c +++ b/sys/fs/nfsserver/nfs_nfsdkrpc.c @@ -120,6 +120,10 @@ VNET_DEFINE(struct nfsv4lock, nfsd_suspend_lock); VNET_DEFINE_STATIC(bool, nfsrvd_inited) = false; +/* Server RDMA listen hook, set by nfsrdma at MOD_LOAD. */ +svc_rdma_listen_ftype *svc_rdma_listen = NULL; +int nfsrvd_rdma_port = 0; + /* * NFS server system calls */ @@ -181,7 +185,8 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt) nd.nd_mreq = NULL; nd.nd_cred = NULL; - if (VNET(nfs_privport) != 0) { + /* xp_socket is NULL for RDMA. */ + if (VNET(nfs_privport) != 0 && xprt->xp_socket != NULL) { /* Check if source port is privileged */ u_short port; struct sockaddr *nam = nd.nd_nam; @@ -260,6 +265,18 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt) goto out; } + /* + * For RDMA, ND_GSSINTEGRITY and ND_GSSPRIVACY are not + * supported. + */ + if ((nd.nd_flag & (ND_GSSINTEGRITY | ND_GSSPRIVACY)) != 0 && + xprt->xp_socket == NULL) { + svcerr_auth(rqst, AUTH_FAILED); + svc_freereq(rqst); + m_freem(nd.nd_mrep); + goto out; + } + /* Acquire the principal name for the RPCSEC_GSS cases. */ if ((nd.nd_flag & (ND_NFSV4 | ND_GSS)) == (ND_NFSV4 | ND_GSS)) { rcredp = NULL; @@ -332,6 +349,10 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt) if ((nfsrv_mextpg || xprt->xp_extpg) && nd.nd_nam2 == NULL && PMAP_HAS_DMAP != 0) nd.nd_flag |= ND_CANEXTPG; + + /* If the xprt xp_socket == NULL, this is a RDMA call. */ + if (xprt->xp_socket == NULL) + nd.nd_flag |= ND_RDMA; #ifdef MAC mac_cred_associate_nfsd(nd.nd_cred); #endif @@ -620,6 +641,21 @@ nfsrvd_nfsd(struct thread *td, struct nfsd_nfsd_args *args) VNET(nfsrv_numnfsd)++; /* Num for this vnet. */ NFSD_UNLOCK(); + /* + * If nfsrvd_rdma_port has been set and the nfsrdma.ko module + * has been loaded, start the server side RDMA. + * RDMA does not work within a vnet jail. + */ + if (!jailed(curthread->td_ucred) && svc_rdma_listen != NULL && + nfsrvd_rdma_port != 0) { + error = svc_rdma_listen(VNET(nfsrvd_pool), + nfsrvd_rdma_port); + if (error != 0) { + printf("nfsrvd_nfsd: RDMA listen failed=%d\n", + error); + nfsrvd_rdma_port = 0; + } + } error = nfsrv_createdevids(args, td); if (error == 0) { /* An empty string implies AUTH_SYS only. */ @@ -695,6 +731,11 @@ nfsrvd_init(int terminating) if (terminating) { VNET(nfsd_master_proc) = NULL; NFSD_UNLOCK(); + + /* Turn off the RDMA listener, if enabled. */ + if (!jailed(curthread->td_ucred) && svc_rdma_listen != NULL) + (void)svc_rdma_listen(VNET(nfsrvd_pool), 0); + nfsrv_freealllayoutsanddevids(); nfsrv_freeallbackchannel_xprts(); svcpool_close(VNET(nfsrvd_pool)); diff --git a/sys/fs/nfsserver/nfs_nfsdserv.c b/sys/fs/nfsserver/nfs_nfsdserv.c index 704667bd27b6..dd6e4805606a 100644 --- a/sys/fs/nfsserver/nfs_nfsdserv.c +++ b/sys/fs/nfsserver/nfs_nfsdserv.c @@ -49,6 +49,7 @@ #include #include #include +#include /* Global vars */ extern u_int32_t newnfs_false, newnfs_true; @@ -1076,6 +1077,20 @@ nfsrvd_read(struct nfsrv_descript *nd, __unused int isdgram, } *tl = txdr_unsigned(cnt); if (m3) { + /* + * For RDMA, inform the server side rdma the reduction's + * position. + */ + if ((nd->nd_flag & ND_RDMA) != 0 && nd->nd_xprt != NULL) { + KASSERT(cnt > 0, + ("nfsrvd_read: m3 != NULL when cnt == 0")); + struct rpcrdma_reduce ddp; + + ddp.xid = nd->nd_retxid; + ddp.off = (uint32_t)m_length(nd->nd_mreq, NULL); + ddp.len = (uint32_t)cnt; + (void)SVC_CONTROL(nd->nd_xprt, SVCSET_READDDP, &ddp); + } nd->nd_mb->m_next = m3; nd->nd_mb = m2; if ((m2->m_flags & M_EXTPG) != 0) { diff --git a/sys/rpc/clnt_bck.c b/sys/rpc/clnt_bck.c index 9ff85b1fa2c0..65eb95c56fa5 100644 --- a/sys/rpc/clnt_bck.c +++ b/sys/rpc/clnt_bck.c @@ -83,6 +83,9 @@ #include #include +/* The RDMA module will set this function pointer non-NULL. */ +clnt_bck_rdma_send_ftype *clnt_bck_rdma_send = NULL; + struct cmessage { struct cmsghdr cmsg; struct cmsgcred cmcred; @@ -269,12 +272,15 @@ call_again: } mreq->m_pkthdr.len = m_length(mreq, NULL); - /* - * Prepend a record marker containing the packet length. - */ - M_PREPEND(mreq, sizeof(uint32_t), M_WAITOK); - *mtod(mreq, uint32_t *) = - htonl(0x80000000 | (mreq->m_pkthdr.len - sizeof(uint32_t))); + /* RDMA doesn't need a record marker. */ + if (xprt->xp_socket != NULL) { + /* + * Prepend a record marker containing the packet length. + */ + M_PREPEND(mreq, sizeof(uint32_t), M_WAITOK); + *mtod(mreq, uint32_t *) = + htonl(0x80000000 | (mreq->m_pkthdr.len - sizeof(uint32_t))); + } cr->cr_xid = xid; mtx_lock(&ct->ct_lock); @@ -297,36 +303,47 @@ call_again: TAILQ_INSERT_TAIL(&ct->ct_pending, cr, cr_link); mtx_unlock(&ct->ct_lock); - /* For RPC-over-TLS, copy mrep to a chain of ext_pgs. */ - if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) { - /* - * Copy the mbuf chain to a chain of - * ext_pgs mbuf(s) as required by KERN_TLS. - */ - maxextsiz = TLS_MAX_MSG_SIZE_V10_2; + if (xprt->xp_socket != NULL) { + /* Handle TCP sockets. */ + /* For RPC-over-TLS, copy mrep to a chain of ext_pgs. */ + if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) { + /* + * Copy the mbuf chain to a chain of + * ext_pgs mbuf(s) as required by KERN_TLS. + */ + maxextsiz = TLS_MAX_MSG_SIZE_V10_2; #ifdef KERN_TLS - if (rpctls_getinfo(&maxlen, false, false)) - maxextsiz = min(maxextsiz, maxlen); + if (rpctls_getinfo(&maxlen, false, false)) + maxextsiz = min(maxextsiz, maxlen); #endif - mreq = _rpc_copym_into_ext_pgs(mreq, maxextsiz); - } - /* - * sosend consumes mreq. - */ - sx_xlock(&xprt->xp_lock); - error = sosend(xprt->xp_socket, NULL, NULL, mreq, NULL, 0, curthread); - mreq = NULL; - if (error == EMSGSIZE) { - SOCK_SENDBUF_LOCK(xprt->xp_socket); - sbwait(xprt->xp_socket, SO_SND); - SOCK_SENDBUF_UNLOCK(xprt->xp_socket); + mreq = _rpc_copym_into_ext_pgs(mreq, maxextsiz); + } + /* + * sosend consumes mreq. + */ + sx_xlock(&xprt->xp_lock); + error = sosend(xprt->xp_socket, NULL, NULL, mreq, NULL, 0, + curthread); + mreq = NULL; + if (error == EMSGSIZE) { + SOCK_SENDBUF_LOCK(xprt->xp_socket); + sbwait(xprt->xp_socket, SO_SND); + SOCK_SENDBUF_UNLOCK(xprt->xp_socket); + sx_xunlock(&xprt->xp_lock); + AUTH_VALIDATE(auth, xid, NULL, NULL); + mtx_lock(&ct->ct_lock); + TAILQ_REMOVE(&ct->ct_pending, cr, cr_link); + goto call_again; + } sx_xunlock(&xprt->xp_lock); - AUTH_VALIDATE(auth, xid, NULL, NULL); - mtx_lock(&ct->ct_lock); - TAILQ_REMOVE(&ct->ct_pending, cr, cr_link); - goto call_again; + } else { + /* Handle RDMA. */ + if (clnt_bck_rdma_send != NULL) { + clnt_bck_rdma_send(xprt, mreq); + mreq = NULL; + } else + error = ENOTCONN; } - sx_xunlock(&xprt->xp_lock); reply_msg.acpted_rply.ar_verf.oa_flavor = AUTH_NULL; reply_msg.acpted_rply.ar_verf.oa_base = cr->cr_verf; diff --git a/sys/rpc/krpc.h b/sys/rpc/krpc.h index f77e2d2d7428..f53b07ed86b6 100644 --- a/sys/rpc/krpc.h +++ b/sys/rpc/krpc.h @@ -42,6 +42,10 @@ enum clnt_stat clnt_bck_call(CLIENT *, struct rpc_callextra *, rpcproc_t, struct mbuf *, struct mbuf **, struct timeval, SVCXPRT *); struct mbuf *_rpc_copym_into_ext_pgs(struct mbuf *, int); +/* Callback function for server side RDMA. */ +typedef int clnt_bck_rdma_send_ftype(SVCXPRT *xprt, struct mbuf *m); +extern clnt_bck_rdma_send_ftype *clnt_bck_rdma_send; + /* * A pending RPC request which awaits a reply. Requests which have * received their reply will have cr_xid set to zero and cr_mrep to @@ -136,6 +140,29 @@ struct cf_conn { /* kept in xprt->xp_p1 for actual connection */ void rpcnl_init(void); +/* RDMA procedures. */ +enum rdma_proc { + RDMA_MSG = 0, + RDMA_NOMSG = 1, + RDMA_MSGP = 2, /* Not used. */ + RDMA_DONE = 3, /* Not used. */ + RDMA_ERROR = 4 +}; + +enum rdma_errcode { + RDMA_ERR_VERS = 1, + RDMA_ERR_CHUNK = 2 +}; + +/* Structure use by both client and server RDMA for reductions (RFC8166). */ +struct rpcrdma_reduce { + struct iovec *iov; + uint32_t xid; + uint32_t off; + uint32_t len; + uint8_t into_mem; +}; + #endif /* _KERNEL */ #endif /* _RPC_KRPC_H_ */ diff --git a/sys/rpc/svc.h b/sys/rpc/svc.h index e73e61e5ac93..a7cbb639f48e 100644 --- a/sys/rpc/svc.h +++ b/sys/rpc/svc.h @@ -76,6 +76,7 @@ #define SVCSET_VERSQUIET 2 #define SVCGET_CONNMAXREC 3 #define SVCSET_CONNMAXREC 4 +#define SVCSET_READDDP 5 /* * Operations for rpc_control(). @@ -342,6 +343,16 @@ typedef struct __rpc_svcpool { SVCGROUP sp_groups[SVC_MAXGROUPS]; /* Thread/port groups. */ } SVCPOOL; +/* + * svc_rdma_listen is the function in the nfsrdma.ko module called to + * enable the RDMA server side listener. + * nfsrvd_rdma_port - The port# that RDMA should listen on for the NFS server. + */ +typedef int svc_rdma_listen_ftype(SVCPOOL *pool, int port); +extern svc_rdma_listen_ftype *svc_rdma_listen; +extern int nfsrvd_rdma_port; +extern int newnfs_numnfsd; + /* * Operations defined on an SVCXPRT handle * diff --git a/sys/rpc/svc_auth.c b/sys/rpc/svc_auth.c index acbb1112e270..030f55e44a97 100644 --- a/sys/rpc/svc_auth.c +++ b/sys/rpc/svc_auth.c @@ -107,6 +107,9 @@ _authenticate(struct svc_req *rqst, struct rpc_msg *msg) dummy = _svcauth_rpcsec_gss(rqst, msg); return (dummy); case AUTH_TLS: + /* Not supported for RDMA. */ + if (rqst->rq_xprt->xp_socket == NULL) + return (AUTH_REJECTEDCRED); dummy = _svcauth_rpcsec_tls(rqst, msg); return (dummy); default: