# HG changeset patch
# User Aurelien Campeas <aurelien.campeas@logilab.fr>
# Date 1366971037 -7200
# Node ID 7b26fe71404f8e3899c588482bff1b687271ee37
# Parent  ae898a084da2aeb55531523fc9977714f66f711d
drop xhtml content-type support (closes #2065651)

* HTMLStream does not care about xml any more

* reqquest.demote_to_html and .xhtml_browser are deprecated

* web config: drop force-html-content-type option

* adjust tests

diff -r ae898a084da2 -r 7b26fe71404f devtools/htmlparser.py
--- a/devtools/htmlparser.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/devtools/htmlparser.py	Fri Apr 26 12:10:37 2013 +0200
@@ -39,7 +39,7 @@
         except etree.XMLSyntaxError as exc:
             def save_in(fname=''):
                 file(fname, 'w').write(data)
-            new_exc = AssertionError(u'invalid xml %s' % exc)
+            new_exc = AssertionError(u'invalid document: %s' % exc)
             new_exc.position = exc.position
             raise new_exc
 
diff -r ae898a084da2 -r 7b26fe71404f devtools/httptest.py
--- a/devtools/httptest.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/devtools/httptest.py	Fri Apr 26 12:10:37 2013 +0200
@@ -78,10 +78,6 @@
     def pyro_enabled(self):
         return False
 
-    def load_configuration(self):
-        super(CubicWebServerConfig, self).load_configuration()
-        self.global_set_option('force-html-content-type', True)
-
 
 class CubicWebServerTC(CubicWebTC):
     """Class for running test web server. See :class:`CubicWebServerConfig`.
diff -r ae898a084da2 -r 7b26fe71404f devtools/test/unittest_testlib.py
--- a/devtools/test/unittest_testlib.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/devtools/test/unittest_testlib.py	Fri Apr 26 12:10:37 2013 +0200
@@ -97,7 +97,9 @@
 class HTMLPageInfoTC(TestCase):
     """test cases for PageInfo"""
     def setUp(self):
-        parser = htmlparser.DTDValidator()
+        parser = htmlparser.HTMLValidator()
+        # disable cleanup that would remove doctype
+        parser.preprocess_data = lambda data: data
         self.page_info = parser.parse_string(HTML_PAGE2)
 
     def test_source1(self):
diff -r ae898a084da2 -r 7b26fe71404f utils.py
--- a/utils.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/utils.py	Fri Apr 26 12:10:37 2013 +0200
@@ -229,11 +229,8 @@
     jQuery(window).unload(unloadPageData);
     pageDataUnloaded = true;
 }'''
-    # Making <script> tag content work properly with all possible
-    # content-types (xml/html) and all possible browsers is very
-    # tricky, see http://www.hixie.ch/advocacy/xhtml for an in-depth discussion
-    xhtml_safe_script_opening = u'<script type="text/javascript"><!--//--><![CDATA[//><!--\n'
-    xhtml_safe_script_closing = u'\n//--><!]]></script>'
+    script_opening = u'<script type="text/javascript">\n'
+    script_closing = u'\n</script>'
 
     def __init__(self, req):
         super(HTMLHead, self).__init__()
@@ -344,14 +341,14 @@
         w = self.write
         # 1/ variable declaration if any
         if self.jsvars:
-            w(self.xhtml_safe_script_opening)
+            w(self.script_opening)
             for var, value, override in self.jsvars:
                 vardecl = u'%s = %s;' % (var, json.dumps(value))
                 if not override:
                     vardecl = (u'if (typeof %s == "undefined") {%s}' %
                                (var, vardecl))
                 w(vardecl + u'\n')
-            w(self.xhtml_safe_script_closing)
+            w(self.script_closing)
         # 2/ css files
         ie_cssfiles = ((x, (y, z)) for x, y, z in self.ie_cssfiles)
         if self.datadir_url and self._cw.vreg.config['concat-resources']:
@@ -397,9 +394,9 @@
                     w(xml_escape(script))
                     w(u'</pre>')
             else:
-                w(self.xhtml_safe_script_opening)
+                w(self.script_opening)
                 w(u'\n\n'.join(self.post_inlined_scripts))
-                w(self.xhtml_safe_script_closing)
+                w(self.script_closing)
         header = super(HTMLHead, self).getvalue()
         if skiphead:
             return header
@@ -422,20 +419,17 @@
         # main stream
         self.body = UStringIO()
         self.doctype = u''
-        # xmldecl and html opening tag
-        self.xmldecl = u'<?xml version="1.0" encoding="%s"?>\n' % req.encoding
-        self._namespaces = [('xmlns', 'http://www.w3.org/1999/xhtml'),
-                            ('xmlns:cubicweb','http://www.logilab.org/2008/cubicweb')]
-        self._htmlattrs = [('xml:lang', req.lang),
-                           ('lang', req.lang)]
+        self._htmlattrs = [('lang', req.lang)]
         # keep main_stream's reference on req for easier text/html demoting
         req.main_stream = self
 
+    @deprecated('[3.17] there are no namespaces in html, xhtml is not served any longer')
     def add_namespace(self, prefix, uri):
-        self._namespaces.append( (prefix, uri) )
+        pass
 
+    @deprecated('[3.17] there are no namespaces in html, xhtml is not served any longer')
     def set_namespaces(self, namespaces):
-        self._namespaces = namespaces
+        pass
 
     def add_htmlattr(self, attrname, attrvalue):
         self._htmlattrs.append( (attrname, attrvalue) )
@@ -443,10 +437,11 @@
     def set_htmlattrs(self, attrs):
         self._htmlattrs = attrs
 
-    def set_doctype(self, doctype, reset_xmldecl=True):
+    def set_doctype(self, doctype, reset_xmldecl=None):
         self.doctype = doctype
-        if reset_xmldecl:
-            self.xmldecl = u''
+        if reset_xmldecl is not None:
+            warn('[3.17] xhtml is no more supported',
+                 DeprecationWarning, stacklevel=2)
 
     def write(self, data):
         """StringIO interface: this method will be assigned to self.w
@@ -456,17 +451,17 @@
     @property
     def htmltag(self):
         attrs = ' '.join('%s="%s"' % (attr, xml_escape(value))
-                         for attr, value in (self._namespaces + self._htmlattrs))
+                         for attr, value in self._htmlattrs)
         if attrs:
             return '<html %s>' % attrs
         return '<html>'
 
     def getvalue(self):
         """writes HTML headers, closes </head> tag and writes HTML body"""
-        return u'%s\n%s\n%s\n%s\n%s\n</html>' % (self.xmldecl, self.doctype,
-                                                 self.htmltag,
-                                                 self.head.getvalue(),
-                                                 self.body.getvalue())
+        return u'%s\n%s\n%s\n%s\n</html>' % (self.doctype,
+                                             self.htmltag,
+                                             self.head.getvalue(),
+                                             self.body.getvalue())
 
 try:
     # may not be there if cubicweb-web not installed
diff -r ae898a084da2 -r 7b26fe71404f view.py
--- a/view.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/view.py	Fri Apr 26 12:10:37 2013 +0200
@@ -42,50 +42,11 @@
 NOINDEX = u'<meta name="ROBOTS" content="NOINDEX" />'
 NOFOLLOW = u'<meta name="ROBOTS" content="NOFOLLOW" />'
 
-CW_XHTML_EXTENSIONS = '''[
-  <!ATTLIST html xmlns:cubicweb CDATA  #FIXED \'http://www.logilab.org/2008/cubicweb\'  >
-
-<!ENTITY % coreattrs
- "id          ID            #IMPLIED
-  class       CDATA         #IMPLIED
-  style       CDATA         #IMPLIED
-  title       CDATA         #IMPLIED
+TRANSITIONAL_DOCTYPE_NOEXT = u'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n'
+TRANSITIONAL_DOCTYPE = TRANSITIONAL_DOCTYPE_NOEXT # bw compat
 
- cubicweb:accesskey         CDATA   #IMPLIED
- cubicweb:actualrql         CDATA   #IMPLIED
- cubicweb:dataurl           CDATA   #IMPLIED
- cubicweb:facetName         CDATA   #IMPLIED
- cubicweb:facetargs         CDATA   #IMPLIED
- cubicweb:fallbackvid       CDATA   #IMPLIED
- cubicweb:fname             CDATA   #IMPLIED
- cubicweb:initfunc          CDATA   #IMPLIED
- cubicweb:inputid           CDATA   #IMPLIED
- cubicweb:inputname         CDATA   #IMPLIED
- cubicweb:limit             CDATA   #IMPLIED
- cubicweb:loadtype          CDATA   #IMPLIED
- cubicweb:loadurl           CDATA   #IMPLIED
- cubicweb:maxlength         CDATA   #IMPLIED
- cubicweb:required          CDATA   #IMPLIED
- cubicweb:rooteid           CDATA   #IMPLIED
- cubicweb:rql               CDATA   #IMPLIED
- cubicweb:size              CDATA   #IMPLIED
- cubicweb:sortvalue         CDATA   #IMPLIED
- cubicweb:target            CDATA   #IMPLIED
- cubicweb:tindex            CDATA   #IMPLIED
- cubicweb:tlunit            CDATA   #IMPLIED
- cubicweb:type              CDATA   #IMPLIED
- cubicweb:unselimg          CDATA   #IMPLIED
- cubicweb:uselabel          CDATA   #IMPLIED
- cubicweb:value             CDATA   #IMPLIED
- cubicweb:variables         CDATA   #IMPLIED
- cubicweb:vid               CDATA   #IMPLIED
- cubicweb:wdgtype           CDATA   #IMPLIED
-  "> ] '''
-
-TRANSITIONAL_DOCTYPE = u'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd" %s>\n' % CW_XHTML_EXTENSIONS
-TRANSITIONAL_DOCTYPE_NOEXT = u'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n'
-STRICT_DOCTYPE = u'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd" %s>\n' % CW_XHTML_EXTENSIONS
 STRICT_DOCTYPE_NOEXT = u'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n'
+STRICT_DOCTYPE = STRICT_DOCTYPE_NOEXT # bw compat
 
 # base view object ############################################################
 
@@ -510,11 +471,7 @@
     one to display error if the first one failed
     """
 
-    @property
-    def doctype(self):
-        if self._cw.xhtml_browser():
-            return STRICT_DOCTYPE
-        return STRICT_DOCTYPE_NOEXT
+    doctype = STRICT_DOCTYPE
 
     def set_stream(self, w=None):
         if self.w is not None:
diff -r ae898a084da2 -r 7b26fe71404f web/data/cubicweb.edition.js
--- a/web/data/cubicweb.edition.js	Fri Apr 26 11:53:47 2013 +0200
+++ b/web/data/cubicweb.edition.js	Fri Apr 26 12:10:37 2013 +0200
@@ -543,7 +543,8 @@
  *
  * .. note::
  *
- *    this is a hack to make the XHTML compliant.
+ *    This was a hack to make form loop handling XHTML compliant.
+ *    Since we do not care about xhtml any longer, this may go away.
  *
  * .. note::
  *
@@ -551,8 +552,10 @@
  *
  * .. note::
  *
- *    there is a XHTML module allowing iframe elements but there
- *    is still the problem of the form's `target` attribute
+ *    The form's `target` attribute should probably become a simple data-target
+ *    immediately generated server-side.
+ *    Since we don't do xhtml any longer, the iframe should probably be either
+ *    reconsidered or at least emitted server-side.
  */
 function setFormsTarget(node) {
     var $node = jQuery(node || document.body);
diff -r ae898a084da2 -r 7b26fe71404f web/request.py
--- a/web/request.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/web/request.py	Fri Apr 26 12:10:37 2013 +0200
@@ -42,7 +42,7 @@
 from cubicweb.dbapi import DBAPIRequest
 from cubicweb.uilib import remove_html_tags, js
 from cubicweb.utils import SizeConstrainedList, HTMLHead, make_uid
-from cubicweb.view import STRICT_DOCTYPE, TRANSITIONAL_DOCTYPE_NOEXT
+from cubicweb.view import TRANSITIONAL_DOCTYPE_NOEXT
 from cubicweb.web import (INTERNAL_FIELD_VALUE, LOGGER, NothingToEdit,
                           RequestError, StatusResponse)
 from cubicweb.web.httpcache import GMTOFFSET, get_validators
@@ -902,29 +902,26 @@
         values = _parse_accept_header(accepteds, value_parser, value_sort_key)
         return (raw_value for (raw_value, parsed_value, score) in values)
 
+    @deprecated('[3.17] demote_to_html is deprecated as we always serve html')
     def demote_to_html(self):
         """helper method to dynamically set request content type to text/html
 
         The global doctype and xmldec must also be changed otherwise the browser
         will display '<[' at the beginning of the page
         """
-        if not self.vreg.config['force-html-content-type']:
-            if not hasattr(self, 'main_stream'):
-                raise Exception("Can't demote to html from an ajax context. You "
-                                "should change force-html-content-type to yes "
-                                "in the instance configuration file.")
-            self.set_content_type('text/html')
-            self.main_stream.set_doctype(TRANSITIONAL_DOCTYPE_NOEXT)
+        pass
+
 
     # xml doctype #############################################################
 
-    def set_doctype(self, doctype, reset_xmldecl=True):
+    def set_doctype(self, doctype, reset_xmldecl=None):
         """helper method to dynamically change page doctype
 
         :param doctype: the new doctype, e.g. '<!DOCTYPE html>'
-        :param reset_xmldecl: if True, remove the '<?xml version="1.0"?>'
-                              declaration from the page
         """
+        if reset_xmldecl is not None:
+            warn('[3.17] reset_xmldecl is deprecated as we only serve html',
+                 DeprecationWarning, stacklevel=2)
         self.main_stream.set_doctype(doctype, reset_xmldecl)
 
     # page data management ####################################################
@@ -965,6 +962,7 @@
         useragent = self.useragent()
         return useragent and 'MSIE' in useragent
 
+    @deprecated('[3.17] xhtml_browser is deprecated (xhtml is no longer served)')
     def xhtml_browser(self):
         """return True if the browser is considered as xhtml compatible.
 
@@ -972,26 +970,12 @@
         application/xhtml+xml, this method will always return False, even though
         this is semantically different
         """
-        if self.vreg.config['force-html-content-type']:
-            return False
-        useragent = self.useragent()
-        # * MSIE/Konqueror does not support xml content-type
-        # * Opera supports xhtml and handles namespaces properly but it breaks
-        #   jQuery.attr()
-        if useragent and ('MSIE' in useragent or 'KHTML' in useragent
-                          or 'Opera' in useragent):
-            return False
-        return True
+        return False
 
     def html_content_type(self):
-        if self.xhtml_browser():
-            return 'application/xhtml+xml'
         return 'text/html'
 
     def document_surrounding_div(self):
-        if self.xhtml_browser():
-            return (u'<?xml version="1.0"?>\n' + STRICT_DOCTYPE + # XXX encoding ?
-                    u'<div xmlns="http://www.w3.org/1999/xhtml" xmlns:cubicweb="http://www.logilab.org/2008/cubicweb">')
         return u'<div>'
 
     @deprecated('[3.9] use req.uiprops[rid]')
diff -r ae898a084da2 -r 7b26fe71404f web/test/test_views.py
--- a/web/test/test_views.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/web/test/test_views.py	Fri Apr 26 12:10:37 2013 +0200
@@ -1,4 +1,4 @@
-# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
 #
 # This file is part of CubicWeb.
@@ -16,7 +16,7 @@
 # You should have received a copy of the GNU Lesser General Public License along
 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
 """automatic tests"""
-
+from cubicweb.devtools import htmlparser
 from cubicweb.devtools.testlib import CubicWebTC, AutoPopulateTest, AutomaticWebTest
 from cubicweb.view import AnyRsetView
 
@@ -51,7 +51,7 @@
         self.assertFalse('jquery.tablesorter.js' in self.view('oneline', rset))
         # but should be included by the tableview
         rset = self.execute('Any P,F,S LIMIT 1 WHERE P is CWUser, P firstname F, P surname S')
-        self.assertTrue('jquery.tablesorter.js' in self.view('table', rset))
+        self.assertIn('jquery.tablesorter.js', self.view('table', rset).source)
 
     def test_js_added_only_once(self):
         self.vreg._loadedmods[__name__] = {}
diff -r ae898a084da2 -r 7b26fe71404f web/test/unittest_idownloadable.py
--- a/web/test/unittest_idownloadable.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/web/test/unittest_idownloadable.py	Fri Apr 26 12:10:37 2013 +0200
@@ -146,7 +146,7 @@
         finally:
             self.app.error_handler = errhdlr
         get = req.headers_out.getRawHeaders
-        self.assertEqual(['application/xhtml+xml'],
+        self.assertEqual(['text/html;charset=UTF-8'],
                          get('content-type'))
         self.assertEqual(None,
                          get('content-disposition'))
diff -r ae898a084da2 -r 7b26fe71404f web/test/unittest_views_basecontrollers.py
--- a/web/test/unittest_views_basecontrollers.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/web/test/unittest_views_basecontrollers.py	Fri Apr 26 12:10:37 2013 +0200
@@ -566,11 +566,6 @@
         rset = self.john.as_rset()
         rset.req = req
         source = ctrl.publish()
-        self.assertTrue(source.startswith('<?xml version="1.0"?>\n' + STRICT_DOCTYPE +
-                                          u'<div xmlns="http://www.w3.org/1999/xhtml" xmlns:cubicweb="http://www.logilab.org/2008/cubicweb">')
-                        )
-        req.xhtml_browser = lambda: False
-        source = ctrl.publish()
         self.assertTrue(source.startswith('<div>'))
 
 #     def test_json_exec(self):
@@ -744,9 +739,7 @@
         def js_foo(self):
             return u'hello'
         res, req = self.remote_call('foo')
-        self.assertEqual(res,
-                         '<?xml version="1.0"?>\n' + STRICT_DOCTYPE +
-                         u'<div xmlns="http://www.w3.org/1999/xhtml" xmlns:cubicweb="http://www.logilab.org/2008/cubicweb">hello</div>')
+        self.assertEqual(u'<div>hello</div>', res)
 
     def test_monkeypatch_jsoncontroller_jsonize(self):
         self.assertRaises(RemoteCallFailed, self.remote_call, 'foo')
diff -r ae898a084da2 -r 7b26fe71404f web/test/unittest_views_baseviews.py
--- a/web/test/unittest_views_baseviews.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/web/test/unittest_views_baseviews.py	Fri Apr 26 12:10:37 2013 +0200
@@ -21,7 +21,7 @@
 
 from cubicweb.devtools.testlib import CubicWebTC
 from cubicweb.utils import json
-from cubicweb.view import StartupView, TRANSITIONAL_DOCTYPE_NOEXT
+from cubicweb.view import StartupView, TRANSITIONAL_DOCTYPE
 from cubicweb.web.htmlwidgets import TableWidget
 from cubicweb.web.views import vid_from_rset
 
@@ -133,31 +133,26 @@
             html_source = self.view('my-view').source
             source_lines = [line.strip() for line in html_source.splitlines(False)
                             if line.strip()]
-            self.assertListEqual(source_lines[:2],
-                                 ['<!DOCTYPE html>',
-                                  '<html xmlns="http://www.w3.org/1999/xhtml" xmlns:cubicweb="http://www.logilab.org/2008/cubicweb" xml:lang="en" lang="en">'])
+            self.assertListEqual(['<!DOCTYPE html>', '<html lang="en">'], source_lines[:2])
 
     def test_set_doctype_no_reset_xmldecl(self):
         """
         tests `cubicweb.web.request.CubicWebRequestBase.set_doctype`
         with no xmldecl reset
         """
-        html_doctype = TRANSITIONAL_DOCTYPE_NOEXT.strip()
+        html_doctype = TRANSITIONAL_DOCTYPE.strip()
         class MyView(StartupView):
             __regid__ = 'my-view'
             def call(self):
                 self._cw.set_doctype(html_doctype, reset_xmldecl=False)
-                self._cw.main_stream.set_namespaces([('xmlns', 'http://www.w3.org/1999/xhtml')])
                 self._cw.main_stream.set_htmlattrs([('lang', 'cz')])
 
         with self.temporary_appobjects(MyView):
             html_source = self.view('my-view').source
             source_lines = [line.strip() for line in html_source.splitlines(False)
                             if line.strip()]
-            self.assertListEqual(source_lines[:3],
-                                 ['<?xml version="1.0" encoding="UTF-8"?>',
-                                  html_doctype,
-                                  '<html xmlns="http://www.w3.org/1999/xhtml" lang="cz">'])
+            self.assertListEqual([html_doctype, '<html lang="cz">', '<head>'],
+                                 source_lines[:3])
 
 if __name__ == '__main__':
     unittest_main()
diff -r ae898a084da2 -r 7b26fe71404f web/views/calendar.py
--- a/web/views/calendar.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/web/views/calendar.py	Fri Apr 26 12:10:37 2013 +0200
@@ -1,4 +1,4 @@
-# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
 #
 # This file is part of CubicWeb.
@@ -188,7 +188,6 @@
         }
 
     def call(self):
-        self._cw.demote_to_html()
         self._cw.add_css(('fullcalendar.css', 'cubicweb.calendar.css'))
         self._cw.add_js(('jquery.ui.js', 'fullcalendar.min.js', 'jquery.qtip.min.js', 'fullcalendar.locale.js'))
         self.calendar_id = 'cal' + make_uid('uid')
diff -r ae898a084da2 -r 7b26fe71404f web/webconfig.py
--- a/web/webconfig.py	Fri Apr 26 11:53:47 2013 +0200
+++ b/web/webconfig.py	Fri Apr 26 12:10:37 2013 +0200
@@ -170,13 +170,6 @@
           'transparent to the user. Default to 5min.',
           'group': 'web', 'level': 3,
           }),
-        ('force-html-content-type',
-         {'type' : 'yn',
-          'default': False,
-          'help': 'force text/html content type for your html pages instead of cubicweb user-agent based'\
-          'deduction of an appropriate content type',
-          'group': 'web', 'level': 3,
-          }),
         ('embed-allowed',
          {'type' : 'regexp',
           'default': None,