dbm_nextkey: Always return an error if we've reached the end of the database

POSIX.1 states that `dbm_nextkey` must return an invalid key
(i.e., `key.dptr == NULL`) after the end of the database was reached.
The current implementation of `hash_seq` will incorrectly restart
the key sequence after the end of the database is reached.

Fix this by checking the "current bucket" index when R_NEXT is passed.

Sponsored by:   Klara, Inc.
Differential Revision:	https://reviews.freebsd.org/D51635
Reviewed by:	markj

lib/libc/db/hash/hash.c

@@ -704,17 +704,19 @@ hash_seq(const DB *dbp, DBT *key, DBT *data, u_int32_t flag)
 	u_int16_t *bp, ndx;
 
 	hashp = (HTAB *)dbp->internal;
-	if (flag && flag != R_FIRST && flag != R_NEXT) {
+	if (flag != R_FIRST || flag != R_NEXT) {
 		hashp->error = errno = EINVAL;
 		return (ERROR);
 	}
 #ifdef HASH_STATISTICS
 	hash_accesses++;
 #endif
-	if ((hashp->cbucket < 0) || (flag == R_FIRST)) {
+	if (flag == R_FIRST) {
 		hashp->cbucket = 0;
 		hashp->cndx = 1;
 		hashp->cpage = NULL;
+	} else if (hashp->cbucket < 0) { /* R_NEXT */
+		return (ABNORMAL);
 	}
 next_bucket:
 	for (bp = NULL; !bp || !bp[0]; ) {

lib/libc/tests/db/Makefile

@@ -9,6 +9,7 @@ ${PACKAGE}FILES+=		README
 
 ATF_TESTS_C+=		dbm_open_test
 ATF_TESTS_C+=		dbm_perm_test
+ATF_TESTS_C+=		dbm_nextkey_test
 
 NETBSD_ATF_TESTS_C+=	db_hash_seq_test
 NETBSD_ATF_TESTS_SH+=	db_test

lib/libc/tests/db/dbm_nextkey_test.c

@@ -0,0 +1,53 @@
+/*-
+ * Copyright (c) 2025 Klara, Inc.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <fcntl.h>
+#include <ndbm.h>
+#include <stdio.h>
+
+#include <atf-c.h>
+
+static const char *path = "tmp";
+static const char *dbname = "tmp.db";
+
+ATF_TC(dbm_nextkey_test);
+ATF_TC_HEAD(dbm_nextkey_test, tc)
+{
+	atf_tc_set_md_var(tc, "descr",
+	    "Check that dbm_nextkey always returns NULL after reaching the end of the database");
+}
+
+ATF_TC_BODY(dbm_nextkey_test, tc)
+{
+	DBM *db;
+	datum key, data;
+
+	data.dptr = "bar";
+	data.dsize = strlen("bar");
+	key.dptr = "foo";
+	key.dsize = strlen("foo");
+
+	db = dbm_open(path, O_RDWR | O_CREAT, 0755);
+	ATF_CHECK(db != NULL);
+	ATF_REQUIRE(atf_utils_file_exists(dbname));
+	ATF_REQUIRE(dbm_store(db, key, data, DBM_INSERT) != -1);
+
+	key = dbm_firstkey(db);
+	ATF_REQUIRE(key.dptr != NULL);
+	key = dbm_nextkey(db);
+	ATF_REQUIRE(key.dptr == NULL);
+	key = dbm_nextkey(db);
+	ATF_REQUIRE(key.dptr == NULL);
+
+	dbm_close(db);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+	ATF_TP_ADD_TC(tp, dbm_nextkey_test);
+
+	return (atf_no_error());
+}