cxgbe: Use partial GCM mode for partial TLS records on T7

T7's crypto co-processor adds a new partial GCM mode.  This permits an
AES-GCM operation to be split into multiple requests.  After each
request, the partial GHASH state is returned to the host and must be
included in the subsequent request to continue the GHASH computation.

Make use of this when sending a TLS record that spans multiple TCP
"request" (where a request can use TSO to span multiple segments).
This permits computing the final GHASH value across multiple requests
without having to re-send the entire TLS record for the final request.

To ensure that intermediate GHASH results are available when needed,
mbuf chains are queued in the TLS pcb and only dispatched one at a
time to the NIC TXQ.  Packets which do not request a GHASH result
queue the next mbuf from the connection as soon as they are written
into the TXQ.  Packets which do request a GHASH result queue the next
mbuf after the the GHASH result is returned by a message on a NIC RXQ.

Note that partial GCM mode is only used for the in-order data at the
"tip" of a connection and not for retransmits of earlier data.
Retransmits will not request a GHASH result so will enqueue the next
mbuf from the connection to the NIC TXQ after being written.

Partial GCM mode is also not TLS-aware and only supports "plain"
AES-GCM.  Thus, TLS AAD must be explicitly constructed and sent as
part of the work request in the TXQ at the start of each TLS record.
However, by avoiding the need to retransmit the entire TLS record's
contents for the last mbuf spanning a TLS record, the overall amount
of "wasted" DMA is reduced by a factor of about 30.

MFC after:	3 days
Sponsored by:	Chelsio Communications

sys/dev/cxgbe/adapter.h

@@ -418,6 +418,7 @@ enum {
  */
 enum {
 	CPL_FW6_COOKIE_CCR = 0,
+	CPL_FW6_COOKIE_KTLS,
 
 	NUM_CPL_FW6_COOKIES = 2	/* Low bits of cookie value. */
 };
@@ -669,8 +670,12 @@ struct sge_txq {
 		};
 		struct {
 			/* T7 only. */
+			uint64_t kern_tls_ghash_received;
+			uint64_t kern_tls_ghash_requested;
 			uint64_t kern_tls_lso;
+			uint64_t kern_tls_partial_ghash;
 			uint64_t kern_tls_splitmode;
+			uint64_t kern_tls_trailer;
 		};
 	};
 
@@ -1456,6 +1461,8 @@ int t6_ktls_write_wr(struct sge_txq *, void *, struct mbuf *, u_int);
 /* t7_kern_tls.c */
 int t7_tls_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *,
     struct m_snd_tag **);
+void t7_ktls_modload(void);
+void t7_ktls_modunload(void);
 int t7_ktls_parse_pkt(struct mbuf *);
 int t7_ktls_write_wr(struct sge_txq *, void *, struct mbuf *, u_int);
 #endif

sys/dev/cxgbe/offload.h

@@ -230,10 +230,15 @@ struct iw_tunables {
 struct tls_tunables {
 	int inline_keys;
 	union {
-		/* T6 only. */
-		int combo_wrs;
-		/* T7 only. */
-		int short_records;
+		struct {
+			/* T6 only. */
+			int combo_wrs;
+		};
+		struct {
+			/* T7 only. */
+			int short_records;
+			int partial_ghash;
+		};
 	};
 };
 

sys/dev/cxgbe/t4_main.c

@@ -765,6 +765,10 @@ SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, combo_wrs, CTLFLAG_RDTUN, &t4_tls_combo_wrs,
 static int t4_tls_short_records = 1;
 SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, short_records, CTLFLAG_RDTUN,
     &t4_tls_short_records, 0, "Use cipher-only mode for short records.");
+
+static int t4_tls_partial_ghash = 1;
+SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, partial_ghash, CTLFLAG_RDTUN,
+    &t4_tls_partial_ghash, 0, "Use partial GHASH for AES-GCM records.");
 #endif
 
 /* Functions used by VIs to obtain unique MAC addresses for each VI. */
@@ -6139,8 +6143,10 @@ set_params__post_init(struct adapter *sc)
 		if (t4_kern_tls != 0 && is_t6(sc)) {
 			sc->tlst.combo_wrs = t4_tls_combo_wrs;
 			t6_config_kern_tls(sc, true);
-		} else
+		} else {
 			sc->tlst.short_records = t4_tls_short_records;
+			sc->tlst.partial_ghash = t4_tls_partial_ghash;
+		}
 	}
 #endif
 	return (0);
@@ -8113,10 +8119,14 @@ t4_sysctls(struct adapter *sc)
 			    CTLFLAG_RW, &sc->tlst.combo_wrs, 0, "Attempt to "
 			    "combine TCB field updates with TLS record work "
 			    "requests.");
-		else
+		else {
 			SYSCTL_ADD_INT(ctx, children, OID_AUTO, "short_records",
 			    CTLFLAG_RW, &sc->tlst.short_records, 0,
 			    "Use cipher-only mode for short records.");
+			SYSCTL_ADD_INT(ctx, children, OID_AUTO, "partial_ghash",
+			    CTLFLAG_RW, &sc->tlst.partial_ghash, 0,
+			    "Use partial GHASH for AES-GCM records.");
+		}
 	}
 #endif
 
@@ -12949,12 +12959,21 @@ clear_stats(struct adapter *sc, u_int port_id)
 				txq->kern_tls_full = 0;
 				txq->kern_tls_octets = 0;
 				txq->kern_tls_waste = 0;
-				txq->kern_tls_options = 0;
 				txq->kern_tls_header = 0;
-				txq->kern_tls_fin = 0;
 				txq->kern_tls_fin_short = 0;
 				txq->kern_tls_cbc = 0;
 				txq->kern_tls_gcm = 0;
+				if (is_t6(sc)) {
+					txq->kern_tls_options = 0;
+					txq->kern_tls_fin = 0;
+				} else {
+					txq->kern_tls_ghash_received = 0;
+					txq->kern_tls_ghash_requested = 0;
+					txq->kern_tls_lso = 0;
+					txq->kern_tls_partial_ghash = 0;
+					txq->kern_tls_splitmode = 0;
+					txq->kern_tls_trailer = 0;
+				}
 				mp_ring_reset_stats(txq->r);
 			}
 
@@ -14122,6 +14141,7 @@ mod_event(module_t mod, int cmd, void *arg)
 #endif
 #ifdef KERN_TLS
 			t6_ktls_modload();
+			t7_ktls_modload();
 #endif
 			t4_tracer_modload();
 			tweak_tunables();
@@ -14185,6 +14205,7 @@ mod_event(module_t mod, int cmd, void *arg)
 				    vxlan_stop_evtag);
 				t4_tracer_modunload();
 #ifdef KERN_TLS
+				t7_ktls_modunload();
 				t6_ktls_modunload();
 #endif
 #ifdef INET6

sys/dev/cxgbe/t4_sge.c

@@ -4874,14 +4874,30 @@ add_txq_sysctls(struct vi_info *vi, struct sysctl_ctx_list *ctx,
 			    "kern_tls_fin", CTLFLAG_RD, &txq->kern_tls_fin,
 			    "# of NIC TLS FIN-only packets transmitted");
 		} else {
+			SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO,
+			    "kern_tls_ghash_received", CTLFLAG_RD,
+			    &txq->kern_tls_ghash_received,
+			    "# of NIC TLS GHASHes received");
+			SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO,
+			    "kern_tls_ghash_requested", CTLFLAG_RD,
+			    &txq->kern_tls_ghash_requested,
+			    "# of NIC TLS GHASHes requested");
 			SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO,
 			    "kern_tls_lso", CTLFLAG_RD,
 			    &txq->kern_tls_lso,
 			    "# of NIC TLS records transmitted using LSO");
+			SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO,
+			    "kern_tls_partial_ghash", CTLFLAG_RD,
+			    &txq->kern_tls_partial_ghash,
+			    "# of NIC TLS records encrypted using a partial GHASH");
 			SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO,
 			    "kern_tls_splitmode", CTLFLAG_RD,
 			    &txq->kern_tls_splitmode,
 			    "# of NIC TLS records using SplitMode");
+			SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO,
+			    "kern_tls_trailer", CTLFLAG_RD,
+			    &txq->kern_tls_trailer,
+			    "# of NIC TLS trailer-only packets transmitted");
 		}
 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_cbc",
 		    CTLFLAG_RD, &txq->kern_tls_cbc,