diff --git a/ChangeLog b/ChangeLog
index 996619300a412f06170697d83dde49450afae03f..278de08ba7cdc1193a3fb68e145a0bc818dcae0c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
 2023-10-06  Niels Möller  <nisse@lysator.liu.se>
 
+	* testsuite/rsa-sec-decrypt-test.c (test_main): Skip side-channel
+	test if built with mini-gmp.
+
+	* testsuite/sc-valgrind.sh (with_valgrind): Pass
+	--exit-on-first-error=yes.
+
 	* aclocal.m4 (NETTLE_PROG_VALGRIND): New macro.
 	* configure.ac: Use it.
 	* testsuite/Makefile.in (TS_SH): Include side-channel tests only
diff --git a/testsuite/rsa-sec-decrypt-test.c b/testsuite/rsa-sec-decrypt-test.c
index f257723bb5dfef31d1a6a6e83fd2068a2b8e31a4..cc5d49aa74b48e3340e4347632517e82ccdc291a 100644
--- a/testsuite/rsa-sec-decrypt-test.c
+++ b/testsuite/rsa-sec-decrypt-test.c
@@ -70,6 +70,10 @@ test_main(void)
   mpz_t garbage;
   unsigned count;
 
+#if NETTLE_USE_MINI_GMP
+  if (test_side_channel)
+    SKIP();
+#endif
   rsa_private_key_init(&key);
   rsa_public_key_init(&pub);
   mpz_init(gibberish);
diff --git a/testsuite/sc-valgrind.sh b/testsuite/sc-valgrind.sh
index 39e2e941797232f0145e2ec6016a1d9612d7dbeb..61fb7c7d73bb55f7699f42e1df3d61f0cefcaa5d 100644
--- a/testsuite/sc-valgrind.sh
+++ b/testsuite/sc-valgrind.sh
@@ -2,6 +2,6 @@
 # using valgrind.
 
 with_valgrind () {
-    type valgrind >/dev/null || exit 77
-    NETTLE_TEST_SIDE_CHANNEL=1 valgrind -q --error-exitcode=1 "$@"
+    type valgrind >/dev/null 2>&1 || exit 77
+    NETTLE_TEST_SIDE_CHANNEL=1 valgrind -q --exit-on-first-error=yes --error-exitcode=1 "$@"
 }