--- a/cubicweb/cwconfig.py	Fri Jun 07 14:53:03 2019 +0200
+++ b/cubicweb/cwconfig.py	Thu Jun 13 15:04:13 2019 +0200
@@ -1341,6 +1341,8 @@
             except Exception as ex:
                 self.exception("can't connect to smtp server %s:%s (%s)",
                                server, port, ex)
+                if self.mode == 'test':
+                    raise
                 return False
             for msg, recipients in msgs:
                 try:
@@ -1348,6 +1350,8 @@
                 except Exception as ex:
                     self.exception("error sending mail to %s (%s)",
                                    recipients, ex)
+                    if self.mode == 'test':
+                        raise
             smtp.close()
         finally:
             SMTP_LOCK.release()