libutil: avoid an out-of-bounds read in trimdomain(3)

memchr(3) will happily believe we've passed in a valid object, but
hostsize could easily exceed the bounds of fullhost.  Clamp it down to
the string size to be safe and avoid UB.  This plugs a potential
overread noted in the compat shim that was just added.

Reviewed by:	des
Sponsored by:	Klara, Inc.
Differential Revision:	https://reviews.freebsd.org/D54623

lib/libutil/tests/trimdomain_test.c

@@ -78,11 +78,13 @@ int
 main(void)
 {
 
-	printf("1..5\n");
+	printf("1..6\n");
 
 	testit(TESTFQDN, -1, TESTHOST, "self");
 	testit("XXX" TESTDOMAIN, -1, "XXX", "different host, same domain");
 	testit("XXX" TESTDOMAIN, 1, NULL, "short hostsize");
+	testit("XXX" TESTDOMAIN, strlen(TESTDOMAIN) + 256, "XXX",
+	    "long hostsize");
 	testit("bogus.example.net", -1, NULL, "arbitrary host");
 	testit("XXX." TESTFQDN, -1, NULL, "domain is local hostname");
 

lib/libutil/trimdomain.c

@@ -73,6 +73,12 @@ trimdomain(char *fullhost, size_t hostsize)
 	if (domain[0] == '\0')
 		return;
 
+	/*
+	 * Clamp hostsize down if it's out-of-bounds of fullhost, to avoid any
+	 * kind of out-of-bounds read in the below memchr().
+	 */
+	hostsize = strnlen(fullhost, hostsize);
+
 	s = fullhost;
 	end = s + hostsize + 1;
 	if ((s = memchr(s, '.', (size_t)(end - s))) != NULL) {