diff -r eb83daa69495 -r 280c9db41038 cubicweb/cwconfig.py
--- a/cubicweb/cwconfig.py	Mon Jul 22 11:21:10 2019 +0200
+++ b/cubicweb/cwconfig.py	Wed Jul 24 15:14:56 2019 +0200
@@ -1243,6 +1243,8 @@
             except Exception as ex:
                 self.exception("can't connect to smtp server %s:%s (%s)",
                                server, port, ex)
+                if self.mode == 'test':
+                    raise
                 return False
             for msg, recipients in msgs:
                 try:
@@ -1250,6 +1252,8 @@
                 except Exception as ex:
                     self.exception("error sending mail to %s (%s)",
                                    recipients, ex)
+                    if self.mode == 'test':
+                        raise
             smtp.close()
         finally:
             SMTP_LOCK.release()