# HG changeset patch
# User Sylvain Thénault <sylvain.thenault@logilab.fr>
# Date 1276778596 -7200
# Node ID 1e73a466aa69eef4d0846847f4505cfe6ac6d8f0
# Parent  c397819f2482f8508be1d972034022c62bc1452a
[fti] support for fti ranking: has_text query results sorted by relevance, and provides a way to control weight per entity / entity's attribute

diff -r c397819f2482 -r 1e73a466aa69 __pkginfo__.py
--- a/__pkginfo__.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/__pkginfo__.py	Thu Jun 17 14:43:16 2010 +0200
@@ -52,7 +52,7 @@
     'Twisted': '',
     # XXX graphviz
     # server dependencies
-    'logilab-database': '',
+    'logilab-database': '1.1.0',
     'pysqlite': '>= 2.5.5', # XXX install pysqlite2
     }
 
diff -r c397819f2482 -r 1e73a466aa69 cwconfig.py
--- a/cwconfig.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/cwconfig.py	Thu Jun 17 14:43:16 2010 +0200
@@ -1107,6 +1107,7 @@
 def register_stored_procedures():
     from logilab.database import FunctionDescr
     from rql.utils import register_function, iter_funcnode_variables
+    from rql.nodes import SortTerm, Constant, VariableRef
 
     global _EXT_REGISTERED
     if _EXT_REGISTERED:
@@ -1152,6 +1153,34 @@
     register_function(TEXT_LIMIT_SIZE)
 
 
+    class FTIRANK(FunctionDescr):
+        """return ranking of a variable that must be used as some has_text
+        relation subject in the query's restriction. Usually used to sort result
+        of full-text search by ranking.
+        """
+        supported_backends = ('postgres',)
+        rtype = 'Float'
+
+        def st_check_backend(self, backend, funcnode):
+            """overriden so that on backend not supporting fti ranking, the
+            function is removed when in an orderby clause, or replaced by a 1.0
+            constant.
+            """
+            if not self.supports(backend):
+                parent = funcnode.parent
+                while parent is not None and not isinstance(parent, SortTerm):
+                    parent = parent.parent
+                if isinstance(parent, SortTerm):
+                    parent.parent.remove(parent)
+                else:
+                    funcnode.parent.replace(funcnode, Constant(1.0, 'Float'))
+                    parent = funcnode
+                for vref in parent.iget_nodes(VariableRef):
+                    vref.unregister_reference()
+
+    register_function(FTIRANK)
+
+
     class FSPATH(FunctionDescr):
         """return path of some bytes attribute stored using the Bytes
         File-System Storage (bfss)
diff -r c397819f2482 -r 1e73a466aa69 debian/control
--- a/debian/control	Thu Jun 17 12:13:38 2010 +0200
+++ b/debian/control	Thu Jun 17 14:43:16 2010 +0200
@@ -33,7 +33,7 @@
 Conflicts: cubicweb-multisources
 Replaces: cubicweb-multisources
 Provides: cubicweb-multisources
-Depends: ${python:Depends}, cubicweb-common (= ${source:Version}), cubicweb-ctl (= ${source:Version}), python-logilab-database (>= 1.0.2), cubicweb-postgresql-support | cubicweb-mysql-support | python-pysqlite2
+Depends: ${python:Depends}, cubicweb-common (= ${source:Version}), cubicweb-ctl (= ${source:Version}), python-logilab-database (>= 1.1.0), cubicweb-postgresql-support | cubicweb-mysql-support | python-pysqlite2
 Recommends: pyro, cubicweb-documentation (= ${source:Version})
 Description: server part of the CubicWeb framework
  CubicWeb is a semantic web application framework.
diff -r c397819f2482 -r 1e73a466aa69 devtools/fake.py
--- a/devtools/fake.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/devtools/fake.py	Thu Jun 17 14:43:16 2010 +0200
@@ -16,8 +16,8 @@
 # You should have received a copy of the GNU Lesser General Public License along
 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
 """Fake objects to ease testing of cubicweb without a fully working environment
+"""
 
-"""
 __docformat__ = "restructuredtext en"
 
 from logilab.database import get_db_helper
@@ -46,7 +46,7 @@
         return self._cubes
 
     def sources(self):
-        return {}
+        return {'system': {'db-driver': 'sqlite'}}
 
 
 class FakeRequest(CubicWebRequestBase):
diff -r c397819f2482 -r 1e73a466aa69 devtools/repotest.py
--- a/devtools/repotest.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/devtools/repotest.py	Thu Jun 17 14:43:16 2010 +0200
@@ -18,8 +18,8 @@
 """some utilities to ease repository testing
 
 This module contains functions to initialize a new repository.
+"""
 
-"""
 __docformat__ = "restructuredtext en"
 
 from pprint import pprint
@@ -134,24 +134,32 @@
             schema._eid_index[rdef.eid] = rdef
 
 
-from logilab.common.testlib import TestCase
+from logilab.common.testlib import TestCase, mock_object
+from logilab.database import get_db_helper
+
 from rql import RQLHelper
+
 from cubicweb.devtools.fake import FakeRepo, FakeSession
 from cubicweb.server import set_debug
 from cubicweb.server.querier import QuerierHelper
 from cubicweb.server.session import Session
-from cubicweb.server.sources.rql2sql import remove_unused_solutions
+from cubicweb.server.sources.rql2sql import SQLGenerator, remove_unused_solutions
 
 class RQLGeneratorTC(TestCase):
-    schema = None # set this in concret test
+    schema = backend = None # set this in concret test
 
     def setUp(self):
         self.repo = FakeRepo(self.schema)
+        self.repo.system_source = mock_object(dbdriver=self.backend)
         self.rqlhelper = RQLHelper(self.schema, special_relations={'eid': 'uid',
-                                                                   'has_text': 'fti'})
+                                                                   'has_text': 'fti'},
+                                   backend=self.backend)
         self.qhelper = QuerierHelper(self.repo, self.schema)
         ExecutionPlan._check_permissions = _dummy_check_permissions
         rqlannotation._select_principal = _select_principal
+        if self.backend is not None:
+            dbhelper = get_db_helper(self.backend)
+            self.o = SQLGenerator(self.schema, dbhelper)
 
     def tearDown(self):
         ExecutionPlan._check_permissions = _orig_check_permissions
@@ -270,6 +278,7 @@
         self.system = self.sources[-1]
         do_monkey_patch()
         self._dumb_sessions = [] # by hi-jacked parent setup
+        self.repo.vreg.rqlhelper.backend = 'postgres' # so FTIRANK is considered
 
     def add_source(self, sourcecls, uri):
         self.sources.append(sourcecls(self.repo, self.o.schema,
diff -r c397819f2482 -r 1e73a466aa69 entities/adapters.py
--- a/entities/adapters.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/entities/adapters.py	Thu Jun 17 14:43:16 2010 +0200
@@ -108,6 +108,10 @@
         else:
             yield entity
 
+    # weight in ABCD
+    entity_weight = 1.0
+    attr_weight = {}
+
     def get_words(self):
         """used by the full text indexer to get words to index
 
@@ -121,10 +125,11 @@
         # take care to cases where we're modyfying the schema
         entity = self.entity
         pending = self._cw.transaction_data.setdefault('pendingrdefs', set())
-        words = []
+        words = {}
         for rschema in entity.e_schema.indexable_attributes():
             if (entity.e_schema, rschema) in pending:
                 continue
+            weight = self.attr_weight.get(rschema, 'C')
             try:
                 value = entity.printable_value(rschema, format='text/plain')
             except TransformError:
@@ -134,16 +139,19 @@
                                rschema, entity.eid)
                 continue
             if value:
-                words += tokenize(value)
+                words.setdefault(weight, []).extend(tokenize(value))
         for rschema, role in entity.e_schema.fulltext_relations():
             if role == 'subject':
                 for entity_ in getattr(entity, rschema.type):
-                    words += entity_.cw_adapt_to('IFTIndexable').get_words()
+                    merge_weight_dict(words, entity_.cw_adapt_to('IFTIndexable').get_words())
             else: # if role == 'object':
                 for entity_ in getattr(entity, 'reverse_%s' % rschema.type):
-                    words += entity_.cw_adapt_to('IFTIndexable').get_words()
+                    merge_weight_dict(words, entity_.cw_adapt_to('IFTIndexable').get_words())
         return words
 
+def merge_weight_dict(maindict, newdict):
+    for weight, words in newdict.iteritems():
+        maindict.setdefault(weight, []).extend(words)
 
 class IDownloadableAdapter(EntityAdapter):
     """interface for downloadable entities"""
diff -r c397819f2482 -r 1e73a466aa69 misc/migration/3.9.0_Any.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/misc/migration/3.9.0_Any.py	Thu Jun 17 14:43:16 2010 +0200
@@ -0,0 +1,3 @@
+if repo.system_source.dbdriver == 'postgres':
+    sql('ALTER TABLE appears ADD COLUMN weight float')
+    sql('UPDATE appears SET weight=1.0 ')
diff -r c397819f2482 -r 1e73a466aa69 schema.py
--- a/schema.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/schema.py	Thu Jun 17 14:43:16 2010 +0200
@@ -577,7 +577,6 @@
         except BadSchemaDefinition:
             reversed_etype_map = dict( (v, k) for k, v in ETYPE_NAME_MAP.iteritems() )
             if rdef.subject in reversed_etype_map or rdef.object in reversed_etype_map:
-                self.warning('huuuu')
                 return
             raise
         if rdefs:
diff -r c397819f2482 -r 1e73a466aa69 server/msplanner.py
--- a/server/msplanner.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/server/msplanner.py	Thu Jun 17 14:43:16 2010 +0200
@@ -96,7 +96,7 @@
 
 from rql.stmts import Union, Select
 from rql.nodes import (VariableRef, Comparison, Relation, Constant, Variable,
-                       Not, Exists)
+                       Not, Exists, SortTerm, Function)
 
 from cubicweb import server
 from cubicweb.utils import make_uid
@@ -1330,6 +1330,12 @@
                                                orderby.append)
                 if orderby:
                     newroot.set_orderby(orderby)
+            elif rqlst.orderby:
+                for sortterm in rqlst.orderby:
+                    if any(f for f in sortterm.iget_nodes(Function) if f.name == 'FTIRANK'):
+                        newnode, oldnode = sortterm.accept(self, newroot, terms)
+                        if newnode is not None:
+                            newroot.add_sort_term(newnode)
             self.process_selection(newroot, terms, rqlst)
         elif not newroot.where:
             # no restrictions have been copied, just select terms and add
@@ -1530,12 +1536,38 @@
             copy.operator = '='
         return copy, node
 
+    def visit_function(self, node, newroot, terms):
+        if node.name == 'FTIRANK':
+            # FTIRANK is somewhat special... Rank function should be included in
+            # the same query has the has_text relation, potentially added to
+            # selection for latter usage
+            if not self.hasaggrstep and self.final and node not in self.skip:
+                return self.visit_default(node, newroot, terms)
+            elif any(s for s in self.sources if s.uri != 'system'):
+                return None, node
+            # p = node.parent
+            # while p is not None and not isinstance(p, SortTerm):
+            #     p = p.parent
+            # if isinstance(p, SortTerm):
+            if not self.hasaggrstep and self.final and node in self.skip:
+                return Constant(self.skip[node], 'Int'), node
+            # XXX only if not yet selected
+            newroot.append_selected(node.copy(newroot))
+            self.skip[node] = len(newroot.selection)
+            return None, node
+        return self.visit_default(node, newroot, terms)
+
     def visit_default(self, node, newroot, terms):
         subparts, node = self._visit_children(node, newroot, terms)
         return copy_node(newroot, node, subparts), node
 
-    visit_mathexpression = visit_constant = visit_function = visit_default
-    visit_sort = visit_sortterm = visit_default
+    visit_mathexpression = visit_constant = visit_default
+
+    def visit_sortterm(self, node, newroot, terms):
+        subparts, node = self._visit_children(node, newroot, terms)
+        if not subparts:
+            return None, node
+        return copy_node(newroot, node, subparts), node
 
     def _visit_children(self, node, newroot, terms):
         subparts = []
diff -r c397819f2482 -r 1e73a466aa69 server/mssteps.py
--- a/server/mssteps.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/server/mssteps.py	Thu Jun 17 14:43:16 2010 +0200
@@ -140,13 +140,6 @@
 
     def mytest_repr(self):
         """return a representation of this step suitable for test"""
-        sel = self.select.selection
-        restr = self.select.where
-        self.select.selection = self.selection
-        self.select.where = None
-        rql = self.select.as_string(kwargs=self.plan.args)
-        self.select.selection = sel
-        self.select.where = restr
         try:
             # rely on a monkey patch (cf unittest_querier)
             table = self.plan.tablesinorder[self.table]
@@ -155,12 +148,19 @@
             # not monkey patched
             table = self.table
             outputtable = self.outputtable
-        return (self.__class__.__name__, rql, self.limit, self.offset, table,
-                outputtable)
+        sql = self.get_sql().replace(self.table, table)
+        return (self.__class__.__name__, sql, outputtable)
 
     def execute(self):
         """execute this step"""
         self.execute_children()
+        sql = self.get_sql()
+        if self.outputtable:
+            self.plan.create_temp_table(self.outputtable)
+            sql = 'INSERT INTO %s %s' % (self.outputtable, sql)
+        return self.plan.sqlexec(sql, self.plan.args)
+
+    def get_sql(self):
         self.inputmap = inputmap = self.children[-1].outputmap
         # get the select clause
         clause = []
@@ -223,17 +223,15 @@
             sql.append('LIMIT %s' % self.limit)
         if self.offset:
             sql.append('OFFSET %s' % self.offset)
-        #print 'DATA', plan.sqlexec('SELECT * FROM %s' % self.table, None)
-        sql = ' '.join(sql)
-        if self.outputtable:
-            self.plan.create_temp_table(self.outputtable)
-            sql = 'INSERT INTO %s %s' % (self.outputtable, sql)
-        return self.plan.sqlexec(sql, self.plan.args)
+        return ' '.join(sql)
 
     def visit_function(self, function):
         """generate SQL name for a function"""
-        return '%s(%s)' % (function.name,
-                           ','.join(c.accept(self) for c in function.children))
+        try:
+            return self.children[0].outputmap[str(function)]
+        except KeyError:
+            return '%s(%s)' % (function.name,
+                               ','.join(c.accept(self) for c in function.children))
 
     def visit_variableref(self, variableref):
         """get the sql name for a variable reference"""
diff -r c397819f2482 -r 1e73a466aa69 server/querier.py
--- a/server/querier.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/server/querier.py	Thu Jun 17 14:43:16 2010 +0200
@@ -29,7 +29,7 @@
 from logilab.common.compat import any
 from rql import RQLSyntaxError
 from rql.stmts import Union, Select
-from rql.nodes import Relation, VariableRef, Constant, SubQuery
+from rql.nodes import Relation, VariableRef, Constant, SubQuery, Function
 
 from cubicweb import Unauthorized, QueryError, UnknownEid, typed_eid
 from cubicweb import server
@@ -50,7 +50,8 @@
         key = term.as_string()
         value = '%s.C%s' % (table, i)
         if varmap.get(key, value) != value:
-            raise Exception('variable name conflict on %s' % key)
+            raise Exception('variable name conflict on %s: got %s / %s'
+                            % (key, value, varmap))
         varmap[key] = value
 
 # permission utilities ########################################################
@@ -285,7 +286,26 @@
                     for term in origselection:
                         newselect.append_selected(term.copy(newselect))
                     if select.orderby:
-                        newselect.set_orderby([s.copy(newselect) for s in select.orderby])
+                        sortterms = []
+                        for sortterm in select.orderby:
+                            sortterms.append(sortterm.copy(newselect))
+                            for fnode in sortterm.get_nodes(Function):
+                                if fnode.name == 'FTIRANK':
+                                    # we've to fetch the has_text relation as well
+                                    var = fnode.children[0].variable
+                                    rel = iter(var.stinfo['ftirels']).next()
+                                    assert not rel.ored(), 'unsupported'
+                                    newselect.add_restriction(rel.copy(newselect))
+                                    # remove relation from the orig select and
+                                    # cleanup variable stinfo
+                                    rel.parent.remove(rel)
+                                    var.stinfo['ftirels'].remove(rel)
+                                    var.stinfo['relations'].remove(rel)
+                                    # XXX not properly re-annotated after security insertion?
+                                    newvar = newselect.get_variable(var.name)
+                                    newvar.stinfo.setdefault('ftirels', set()).add(rel)
+                                    newvar.stinfo.setdefault('relations', set()).add(rel)
+                        newselect.set_orderby(sortterms)
                         _expand_selection(select.orderby, selected, aliases, select, newselect)
                         select.orderby = () # XXX dereference?
                     if select.groupby:
@@ -562,6 +582,8 @@
         # rql parsing / analysing helper
         self.solutions = repo.vreg.solutions
         rqlhelper = repo.vreg.rqlhelper
+        # set backend on the rql helper, will be used for function checking
+        rqlhelper.backend = repo.config.sources()['system']['db-driver']
         self._parse = rqlhelper.parse
         self._annotate = rqlhelper.annotate
         # rql planner
diff -r c397819f2482 -r 1e73a466aa69 server/sources/native.py
--- a/server/sources/native.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/server/sources/native.py	Thu Jun 17 14:43:16 2010 +0200
@@ -22,8 +22,8 @@
   from which it comes from) are stored in a varchar column encoded as a base64
   string. This is because it should actually be Bytes but we want an index on
   it for fast querying.
+"""
 
-"""
 from __future__ import with_statement
 
 __docformat__ = "restructuredtext en"
diff -r c397819f2482 -r 1e73a466aa69 server/sources/rql2sql.py
--- a/server/sources/rql2sql.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/server/sources/rql2sql.py	Thu Jun 17 14:43:16 2010 +0200
@@ -568,12 +568,14 @@
                 sql += '\nHAVING %s' % having
             # sort
             if sorts:
-                sql += '\nORDER BY %s' % ','.join(self._sortterm_sql(sortterm,
-                                                                     fselectidx)
-                                                  for sortterm in sorts)
-                if fneedwrap:
-                    selection = ['T1.C%s' % i for i in xrange(len(origselection))]
-                    sql = 'SELECT %s FROM (%s) AS T1' % (','.join(selection), sql)
+                sqlsortterms = [self._sortterm_sql(sortterm, fselectidx)
+                                for sortterm in sorts]
+                sqlsortterms = [x for x in sqlsortterms if x is not None]
+                if sqlsortterms:
+                    sql += '\nORDER BY %s' % ','.join(sqlsortterms)
+                    if sorts and fneedwrap:
+                        selection = ['T1.C%s' % i for i in xrange(len(origselection))]
+                        sql = 'SELECT %s FROM (%s) AS T1' % (','.join(selection), sql)
             state.finalize_source_cbs()
         finally:
             select.selection = origselection
@@ -651,12 +653,14 @@
     def _sortterm_sql(self, sortterm, selectidx):
         term = sortterm.term
         try:
-            sqlterm = str(selectidx.index(str(term)) + 1)
+            sqlterm = selectidx.index(str(term)) + 1
         except ValueError:
             # Constant node or non selected term
-            sqlterm = str(term.accept(self))
+            sqlterm = term.accept(self)
+            if sqlterm is None:
+                return None
         if sortterm.asc:
-            return sqlterm
+            return str(sqlterm)
         else:
             return '%s DESC' % sqlterm
 
@@ -1014,7 +1018,8 @@
             not_ = True
         else:
             not_ = False
-        return self.dbhelper.fti_restriction_sql(alias, const.eval(self._args),
+        query = const.eval(self._args)
+        return self.dbhelper.fti_restriction_sql(alias, query,
                                                  jointo, not_) + restriction
 
     def visit_comparison(self, cmp):
@@ -1057,6 +1062,15 @@
 
     def visit_function(self, func):
         """generate SQL name for a function"""
+        if func.name == 'FTIRANK':
+            try:
+                rel = iter(func.children[0].variable.stinfo['ftirels']).next()
+            except KeyError:
+                raise BadRQLQuery("can't use FTIRANK on variable not used in an"
+                                  " 'has_text' relation (eg full-text search)")
+            const = rel.get_parts()[1].children[0]
+            return self.dbhelper.fti_rank_order(self._fti_table(rel),
+                                                const.eval(self._args))
         args = [c.accept(self) for c in func.children]
         if func in self._state.source_cb_funcs:
             # function executed as a callback on the source
diff -r c397819f2482 -r 1e73a466aa69 server/sqlutils.py
--- a/server/sqlutils.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/server/sqlutils.py	Thu Jun 17 14:43:16 2010 +0200
@@ -15,9 +15,8 @@
 #
 # You should have received a copy of the GNU Lesser General Public License along
 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
-"""SQL utilities functions and classes.
+"""SQL utilities functions and classes."""
 
-"""
 __docformat__ = "restructuredtext en"
 
 import os
diff -r c397819f2482 -r 1e73a466aa69 server/test/data/site_cubicweb.py
--- a/server/test/data/site_cubicweb.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/server/test/data/site_cubicweb.py	Thu Jun 17 14:43:16 2010 +0200
@@ -15,9 +15,6 @@
 #
 # You should have received a copy of the GNU Lesser General Public License along
 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
-"""
-
-"""
 
 from logilab.database import FunctionDescr
 from logilab.database.sqlite import register_sqlite_pyfunc
@@ -25,7 +22,7 @@
 
 try:
     class DUMB_SORT(FunctionDescr):
-        supported_backends = ('sqlite',)
+        pass
 
     register_function(DUMB_SORT)
     def dumb_sort(something):
diff -r c397819f2482 -r 1e73a466aa69 server/test/data/sources_fti
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/server/test/data/sources_fti	Thu Jun 17 14:43:16 2010 +0200
@@ -0,0 +1,14 @@
+[system]
+
+db-driver   = postgres
+db-host     = localhost
+db-port     = 
+adapter     = native
+db-name     = cw_fti_test
+db-encoding = UTF-8
+db-user     = syt
+db-password = syt
+
+[admin]
+login = admin
+password = gingkow
diff -r c397819f2482 -r 1e73a466aa69 server/test/unittest_fti.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/server/test/unittest_fti.py	Thu Jun 17 14:43:16 2010 +0200
@@ -0,0 +1,52 @@
+from __future__ import with_statement
+
+from cubicweb.devtools import ApptestConfiguration
+from cubicweb.devtools.testlib import CubicWebTC
+from cubicweb.selectors import implements
+from cubicweb.entities.adapters import IFTIndexableAdapter
+
+class PostgresFTITC(CubicWebTC):
+    config = ApptestConfiguration('data', sourcefile='sources_fti')
+
+    def test_occurence_count(self):
+        req = self.request()
+        c1 = req.create_entity('Card', title=u'c1',
+                               content=u'cubicweb cubicweb cubicweb')
+        c2 = req.create_entity('Card', title=u'c3',
+                               content=u'cubicweb')
+        c3 = req.create_entity('Card', title=u'c2',
+                               content=u'cubicweb cubicweb')
+        self.commit()
+        self.assertEquals(req.execute('Card X ORDERBY FTIRANK(X) DESC WHERE X has_text "cubicweb"').rows,
+                          [[c1.eid], [c3.eid], [c2.eid]])
+
+
+    def test_attr_weight(self):
+        class CardIFTIndexableAdapter(IFTIndexableAdapter):
+            __select__ = implements('Card')
+            attr_weight = {'title': 'A'}
+        with self.temporary_appobjects(CardIFTIndexableAdapter):
+            req = self.request()
+            c1 = req.create_entity('Card', title=u'c1',
+                                   content=u'cubicweb cubicweb cubicweb')
+            c2 = req.create_entity('Card', title=u'c2',
+                                   content=u'cubicweb cubicweb')
+            c3 = req.create_entity('Card', title=u'cubicweb',
+                                   content=u'autre chose')
+            self.commit()
+            self.assertEquals(req.execute('Card X ORDERBY FTIRANK(X) DESC WHERE X has_text "cubicweb"').rows,
+                              [[c3.eid], [c1.eid], [c2.eid]])
+
+
+    def test_entity_weight(self):
+        class PersonneIFTIndexableAdapter(IFTIndexableAdapter):
+            __select__ = implements('Personne')
+            entity_weight = 2.0
+        with self.temporary_appobjects(PersonneIFTIndexableAdapter):
+            req = self.request()
+            c1 = req.create_entity('Personne', nom=u'c1', prenom=u'cubicweb')
+            c2 = req.create_entity('Comment', content=u'cubicweb cubicweb', comments=c1)
+            c3 = req.create_entity('Comment', content=u'cubicweb cubicweb cubicweb', comments=c1)
+            self.commit()
+            self.assertEquals(req.execute('Any X ORDERBY FTIRANK(X) DESC WHERE X has_text "cubicweb"').rows,
+                              [[c1.eid], [c3.eid], [c2.eid]])
diff -r c397819f2482 -r 1e73a466aa69 server/test/unittest_msplanner.py
--- a/server/test/unittest_msplanner.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/server/test/unittest_msplanner.py	Thu Jun 17 14:43:16 2010 +0200
@@ -413,7 +413,7 @@
         """retrieve CWUser X from both sources and return concatenation of results
         """
         self._test('CWUser X ORDERBY X LIMIT 10 OFFSET 10',
-                   [('AggrStep', 'Any X ORDERBY X', 10, 10, 'table0', None, [
+                   [('AggrStep', 'SELECT table0.C0 FROM table0 ORDER BY table0.C0 LIMIT 10 OFFSET 10', None, [
                        ('FetchStep', [('Any X WHERE X is CWUser', [{'X': 'CWUser'}])],
                         [self.ldap, self.system], {}, {'X': 'table0.C0'}, []),
                        ]),
@@ -423,7 +423,7 @@
         """
         # COUNT(X) is kept in sub-step and transformed into SUM(X) in the AggrStep
         self._test('Any COUNT(X) WHERE X is CWUser',
-                   [('AggrStep', 'Any COUNT(X)', None, None, 'table0', None, [
+                   [('AggrStep', 'SELECT SUM(table0.C0) FROM table0', None, [
                        ('FetchStep', [('Any COUNT(X) WHERE X is CWUser', [{'X': 'CWUser'}])],
                         [self.ldap, self.system], {}, {'COUNT(X)': 'table0.C0'}, []),
                        ]),
@@ -498,7 +498,7 @@
 
     def test_complex_ordered(self):
         self._test('Any L ORDERBY L WHERE X login L',
-                   [('AggrStep', 'Any L ORDERBY L', None, None, 'table0', None,
+                   [('AggrStep', 'SELECT table0.C0 FROM table0 ORDER BY table0.C0', None,
                      [('FetchStep', [('Any L WHERE X login L, X is CWUser',
                                       [{'X': 'CWUser', 'L': 'String'}])],
                        [self.ldap, self.system], {}, {'X.login': 'table0.C0', 'L': 'table0.C0'}, []),
@@ -507,7 +507,7 @@
 
     def test_complex_ordered_limit_offset(self):
         self._test('Any L ORDERBY L LIMIT 10 OFFSET 10 WHERE X login L',
-                   [('AggrStep', 'Any L ORDERBY L', 10, 10, 'table0', None,
+                   [('AggrStep', 'SELECT table0.C0 FROM table0 ORDER BY table0.C0 LIMIT 10 OFFSET 10', None,
                      [('FetchStep', [('Any L WHERE X login L, X is CWUser',
                                       [{'X': 'CWUser', 'L': 'String'}])],
                        [self.ldap, self.system], {}, {'X.login': 'table0.C0', 'L': 'table0.C0'}, []),
@@ -593,7 +593,7 @@
         2. return content of the table sorted
         """
         self._test('Any X,F ORDERBY F WHERE X firstname F',
-                   [('AggrStep', 'Any X,F ORDERBY F', None, None, 'table0', None,
+                   [('AggrStep', 'SELECT table0.C0, table0.C1 FROM table0 ORDER BY table0.C1', None,
                      [('FetchStep', [('Any X,F WHERE X firstname F, X is CWUser',
                                       [{'X': 'CWUser', 'F': 'String'}])],
                        [self.ldap, self.system], {},
@@ -657,7 +657,7 @@
 
     def test_complex_typed_aggregat(self):
         self._test('Any MAX(X) WHERE X is Card',
-                   [('AggrStep', 'Any MAX(X)', None, None, 'table0',  None,
+                   [('AggrStep', 'SELECT MAX(table0.C0) FROM table0',  None,
                      [('FetchStep',
                        [('Any MAX(X) WHERE X is Card', [{'X': 'Card'}])],
                        [self.cards, self.system], {}, {'MAX(X)': 'table0.C0'}, [])
@@ -1299,9 +1299,66 @@
                         ]),
                     ])
 
+    def test_has_text_orderby_rank(self):
+        self._test('Any X ORDERBY FTIRANK(X) WHERE X has_text "bla", X firstname "bla"',
+                   [('FetchStep', [('Any X WHERE X firstname "bla", X is CWUser', [{'X': 'CWUser'}])],
+                     [self.ldap, self.system], None, {'X': 'table0.C0'}, []),
+                    ('AggrStep', 'SELECT table1.C1 FROM table1 ORDER BY table1.C0', None, [
+                        ('FetchStep', [('Any FTIRANK(X),X WHERE X has_text "bla", X is CWUser',
+                                        [{'X': 'CWUser'}])],
+                         [self.system], {'X': 'table0.C0'}, {'FTIRANK(X)': 'table1.C0', 'X': 'table1.C1'}, []),
+                        ('FetchStep', [('Any FTIRANK(X),X WHERE X has_text "bla", X firstname "bla", X is Personne',
+                                        [{'X': 'Personne'}])],
+                         [self.system], {}, {'FTIRANK(X)': 'table1.C0', 'X': 'table1.C1'}, []),
+                        ]),
+                    ])
+
+    def test_security_has_text_orderby_rank(self):
+        # use a guest user
+        self.session = self.user_groups_session('guests')
+        self._test('Any X ORDERBY FTIRANK(X) WHERE X has_text "bla", X firstname "bla"',
+                   [('FetchStep', [('Any X WHERE X firstname "bla", X is CWUser', [{'X': 'CWUser'}])],
+                     [self.ldap, self.system], None, {'X': 'table1.C0'}, []),
+                    ('UnionFetchStep',
+                     [('FetchStep', [('Any X WHERE X firstname "bla", X is Personne', [{'X': 'Personne'}])],
+                       [self.system], {}, {'X': 'table0.C0'}, []),
+                      ('FetchStep', [('Any X WHERE EXISTS(X owned_by 5), X is CWUser', [{'X': 'CWUser'}])],
+                       [self.system], {'X': 'table1.C0'}, {'X': 'table0.C0'}, [])]),
+                    ('OneFetchStep', [('Any X ORDERBY FTIRANK(X) WHERE X has_text "bla"',
+                                       [{'X': 'CWUser'}, {'X': 'Personne'}])],
+                     None, None, [self.system], {'X': 'table0.C0'}, []),
+                    ])
+
+    def test_has_text_select_rank(self):
+        self._test('Any X, FTIRANK(X) WHERE X has_text "bla", X firstname "bla"',
+                   # XXX unecessary duplicate selection
+                   [('FetchStep', [('Any X,X WHERE X firstname "bla", X is CWUser', [{'X': 'CWUser'}])],
+                     [self.ldap, self.system], None, {'X': 'table0.C1'}, []),
+                    ('UnionStep', None, None, [
+                        ('OneFetchStep', [('Any X,FTIRANK(X) WHERE X has_text "bla", X is CWUser', [{'X': 'CWUser'}])],
+                         None, None, [self.system], {'X': 'table0.C1'}, []),
+                        ('OneFetchStep', [('Any X,FTIRANK(X) WHERE X has_text "bla", X firstname "bla", X is Personne', [{'X': 'Personne'}])],
+                         None, None, [self.system], {}, []),
+                        ]),
+                    ])
+
+    def test_security_has_text_select_rank(self):
+        # use a guest user
+        self.session = self.user_groups_session('guests')
+        self._test('Any X, FTIRANK(X) WHERE X has_text "bla", X firstname "bla"',
+                   [('FetchStep', [('Any X,X WHERE X firstname "bla", X is CWUser', [{'X': 'CWUser'}])],
+                     [self.ldap, self.system], None, {'X': 'table0.C1'}, []),
+                    ('UnionStep', None, None, [
+                        ('OneFetchStep', [('Any X,FTIRANK(X) WHERE X has_text "bla", EXISTS(X owned_by 5), X is CWUser', [{'X': 'CWUser'}])],
+                         None, None, [self.system], {'X': 'table0.C1'}, []),
+                        ('OneFetchStep', [('Any X,FTIRANK(X) WHERE X has_text "bla", X firstname "bla", X is Personne', [{'X': 'Personne'}])],
+                         None, None, [self.system], {}, []),
+                        ]),
+                    ])
+
     def test_sort_func(self):
         self._test('Note X ORDERBY DUMB_SORT(RF) WHERE X type RF',
-                   [('AggrStep', 'Any X ORDERBY DUMB_SORT(RF)', None, None, 'table0', None, [
+                   [('AggrStep', 'SELECT table0.C0 FROM table0 ORDER BY DUMB_SORT(table0.C1)', None, [
                        ('FetchStep', [('Any X,RF WHERE X type RF, X is Note',
                                        [{'X': 'Note', 'RF': 'String'}])],
                         [self.cards, self.system], {}, {'X': 'table0.C0', 'X.type': 'table0.C1', 'RF': 'table0.C1'}, []),
@@ -1310,8 +1367,7 @@
 
     def test_ambigous_sort_func(self):
         self._test('Any X ORDERBY DUMB_SORT(RF) WHERE X title RF, X is IN (Bookmark, Card, EmailThread)',
-                   [('AggrStep', 'Any X ORDERBY DUMB_SORT(RF)',
-                     None, None, 'table0', None,
+                   [('AggrStep', 'SELECT table0.C0 FROM table0 ORDER BY DUMB_SORT(table0.C1)', None,
                      [('FetchStep', [('Any X,RF WHERE X title RF, X is Card',
                                       [{'X': 'Card', 'RF': 'String'}])],
                        [self.cards, self.system], {},
@@ -1728,7 +1784,7 @@
                     ('FetchStep', [('Any X,D WHERE X modification_date D, X is CWUser',
                                     [{'X': 'CWUser', 'D': 'Datetime'}])],
                      [self.ldap, self.system], None, {'X': 'table1.C0', 'X.modification_date': 'table1.C1', 'D': 'table1.C1'}, []),
-                    ('AggrStep', 'Any X ORDERBY D DESC', None, None, 'table2', None, [
+                    ('AggrStep', 'SELECT table2.C0 FROM table2 ORDER BY table2.C1 DESC', None, [
                         ('FetchStep', [('Any X,D WHERE E eid %s, E wf_info_for X, X modification_date D, E is TrInfo, X is Affaire'%treid,
                                         [{'X': 'Affaire', 'E': 'TrInfo', 'D': 'Datetime'}])],
                          [self.system],
@@ -1871,8 +1927,7 @@
                                     [{'X': 'Note', 'Z': 'Datetime'}])],
                      [self.cards, self.system], None, {'X': 'table0.C0', 'X.modification_date': 'table0.C1', 'Z': 'table0.C1'},
                      []),
-                    ('AggrStep', 'Any X ORDERBY Z DESC',
-                     None, None, 'table1', None,
+                    ('AggrStep', 'SELECT table1.C0 FROM table1 ORDER BY table1.C1 DESC', None,
                      [('FetchStep', [('Any X,Z WHERE X modification_date Z, 999999 see_also X, X is Bookmark',
                                       [{'X': 'Bookmark', 'Z': 'Datetime'}])],
                        [self.system], {},   {'X': 'table1.C0', 'X.modification_date': 'table1.C1',
diff -r c397819f2482 -r 1e73a466aa69 server/test/unittest_multisources.py
--- a/server/test/unittest_multisources.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/server/test/unittest_multisources.py	Thu Jun 17 14:43:16 2010 +0200
@@ -134,6 +134,8 @@
         self.repo.sources_by_uri['extern'].synchronize(MTIME) # in case fti_update has been run before
         self.failUnless(self.sexecute('Any X WHERE X has_text "affref"'))
         self.failUnless(self.sexecute('Affaire X WHERE X has_text "affref"'))
+        self.failUnless(self.sexecute('Any X ORDERBY FTIRANK(X) WHERE X has_text "affref"'))
+        self.failUnless(self.sexecute('Affaire X ORDERBY FTIRANK(X) WHERE X has_text "affref"'))
 
     def test_anon_has_text(self):
         self.repo.sources_by_uri['extern'].synchronize(MTIME) # in case fti_update has been run before
@@ -145,6 +147,9 @@
         cnx = self.login('anon')
         cu = cnx.cursor()
         rset = cu.execute('Any X WHERE X has_text "card"')
+        # 5: 4 card + 1 readable affaire
+        self.assertEquals(len(rset), 5, zip(rset.rows, rset.description))
+        rset = cu.execute('Any X ORDERBY FTIRANK(X) WHERE X has_text "card"')
         self.assertEquals(len(rset), 5, zip(rset.rows, rset.description))
         Connection_close(cnx)
 
diff -r c397819f2482 -r 1e73a466aa69 server/test/unittest_rql2sql.py
--- a/server/test/unittest_rql2sql.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/server/test/unittest_rql2sql.py	Thu Jun 17 14:43:16 2010 +0200
@@ -22,11 +22,13 @@
 from logilab.common.testlib import TestCase, unittest_main, mock_object
 
 from rql import BadRQLQuery
+from rql.utils import register_function, FunctionDescr
 
-#from cubicweb.server.sources.native import remove_unused_solutions
-from cubicweb.server.sources.rql2sql import SQLGenerator, remove_unused_solutions
+from cubicweb.devtools import TestServerConfiguration
+from cubicweb.devtools.repotest import RQLGeneratorTC
+from cubicweb.server.sources.rql2sql import remove_unused_solutions
 
-from rql.utils import register_function, FunctionDescr
+
 # add a dumb registered procedure
 class stockproc(FunctionDescr):
     supported_backends = ('postgres', 'sqlite', 'mysql')
@@ -35,8 +37,6 @@
 except AssertionError, ex:
     pass # already registered
 
-from cubicweb.devtools import TestServerConfiguration
-from cubicweb.devtools.repotest import RQLGeneratorTC
 
 config = TestServerConfiguration('data')
 config.bootstrap_cubes()
@@ -1060,11 +1060,9 @@
 WHERE rel_is0.eid_to=2'''),
 
     ]
-from logilab.database import get_db_helper
-
 class CWRQLTC(RQLGeneratorTC):
     schema = schema
-
+    backend = 'sqlite'
     def test_nonregr_sol(self):
         delete = self.rqlhelper.parse(
             'DELETE X read_permission READ_PERMISSIONSUBJECT,X add_permission ADD_PERMISSIONSUBJECT,'
@@ -1090,12 +1088,7 @@
 
 class PostgresSQLGeneratorTC(RQLGeneratorTC):
     schema = schema
-
-    #capture = True
-    def setUp(self):
-        RQLGeneratorTC.setUp(self)
-        dbhelper = get_db_helper('postgres')
-        self.o = SQLGenerator(schema, dbhelper)
+    backend = 'postgres'
 
     def _norm_sql(self, sql):
         return sql.strip()
@@ -1355,13 +1348,53 @@
 UNION ALL
 SELECT _X.cw_eid
 FROM appears AS appears0, cw_Folder AS _X
-WHERE appears0.words @@ to_tsquery('default', 'toto&tata') AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
-"""),
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata') AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu"""),
 
             ('Personne X where X has_text %(text)s, X travaille S, S has_text %(text)s',
              """SELECT _X.eid
 FROM appears AS appears0, appears AS appears2, entities AS _X, travaille_relation AS rel_travaille1
 WHERE appears0.words @@ to_tsquery('default', 'hip&hop&momo') AND appears0.uid=_X.eid AND _X.type='Personne' AND _X.eid=rel_travaille1.eid_from AND appears2.uid=rel_travaille1.eid_to AND appears2.words @@ to_tsquery('default', 'hip&hop&momo')"""),
+
+            ('Any X ORDERBY FTIRANK(X) DESC WHERE X has_text "toto tata"',
+             """SELECT appears0.uid
+FROM appears AS appears0
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata')
+ORDER BY ts_rank(appears0.words, to_tsquery('default', 'toto&tata'))*appears0.weight DESC"""),
+
+            ('Personne X ORDERBY FTIRANK(X) WHERE X has_text "toto tata"',
+             """SELECT _X.eid
+FROM appears AS appears0, entities AS _X
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata') AND appears0.uid=_X.eid AND _X.type='Personne'
+ORDER BY ts_rank(appears0.words, to_tsquery('default', 'toto&tata'))*appears0.weight"""),
+
+            ('Personne X ORDERBY FTIRANK(X) WHERE X has_text %(text)s',
+             """SELECT _X.eid
+FROM appears AS appears0, entities AS _X
+WHERE appears0.words @@ to_tsquery('default', 'hip&hop&momo') AND appears0.uid=_X.eid AND _X.type='Personne'
+ORDER BY ts_rank(appears0.words, to_tsquery('default', 'hip&hop&momo'))*appears0.weight"""),
+
+            ('Any X ORDERBY FTIRANK(X) WHERE X has_text "toto tata", X name "tutu", X is IN (Basket,Folder)',
+             """SELECT T1.C0 FROM (SELECT _X.cw_eid AS C0, ts_rank(appears0.words, to_tsquery('default', 'toto&tata'))*appears0.weight AS C1
+FROM appears AS appears0, cw_Basket AS _X
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata') AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
+UNION ALL
+SELECT _X.cw_eid AS C0, ts_rank(appears0.words, to_tsquery('default', 'toto&tata'))*appears0.weight AS C1
+FROM appears AS appears0, cw_Folder AS _X
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata') AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
+ORDER BY 2) AS T1"""),
+
+            ('Personne X ORDERBY FTIRANK(X),FTIRANK(S) WHERE X has_text %(text)s, X travaille S, S has_text %(text)s',
+             """SELECT _X.eid
+FROM appears AS appears0, appears AS appears2, entities AS _X, travaille_relation AS rel_travaille1
+WHERE appears0.words @@ to_tsquery('default', 'hip&hop&momo') AND appears0.uid=_X.eid AND _X.type='Personne' AND _X.eid=rel_travaille1.eid_from AND appears2.uid=rel_travaille1.eid_to AND appears2.words @@ to_tsquery('default', 'hip&hop&momo')
+ORDER BY ts_rank(appears0.words, to_tsquery('default', 'hip&hop&momo'))*appears0.weight,ts_rank(appears2.words, to_tsquery('default', 'hip&hop&momo'))*appears2.weight"""),
+
+
+            ('Any X, FTIRANK(X) WHERE X has_text "toto tata"',
+             """SELECT appears0.uid, ts_rank(appears0.words, to_tsquery('default', 'toto&tata'))*appears0.weight
+FROM appears AS appears0
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata')"""),
+
             )):
             yield t
 
@@ -1411,11 +1444,7 @@
 
 
 class SqliteSQLGeneratorTC(PostgresSQLGeneratorTC):
-
-    def setUp(self):
-        RQLGeneratorTC.setUp(self)
-        dbhelper = get_db_helper('sqlite')
-        self.o = SQLGenerator(schema, dbhelper)
+    backend = 'sqlite'
 
     def _norm_sql(self, sql):
         return sql.strip().replace(' ILIKE ', ' LIKE ')
@@ -1513,17 +1542,33 @@
 FROM appears AS appears0, cw_Folder AS _X
 WHERE appears0.word_id IN (SELECT word_id FROM word WHERE word in ('toto', 'tata')) AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
 """),
+
+            ('Any X ORDERBY FTIRANK(X) WHERE X has_text "toto tata"',
+             """SELECT DISTINCT appears0.uid
+FROM appears AS appears0
+WHERE appears0.word_id IN (SELECT word_id FROM word WHERE word in ('toto', 'tata'))"""),
+
+            ('Any X ORDERBY FTIRANK(X) WHERE X has_text "toto tata", X name "tutu", X is IN (Basket,Folder)',
+             """SELECT DISTINCT _X.cw_eid
+FROM appears AS appears0, cw_Basket AS _X
+WHERE appears0.word_id IN (SELECT word_id FROM word WHERE word in ('toto', 'tata')) AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
+UNION
+SELECT DISTINCT _X.cw_eid
+FROM appears AS appears0, cw_Folder AS _X
+WHERE appears0.word_id IN (SELECT word_id FROM word WHERE word in ('toto', 'tata')) AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
+"""),
+
+            ('Any X, FTIRANK(X) WHERE X has_text "toto tata"',
+             """SELECT DISTINCT appears0.uid, 1.0
+FROM appears AS appears0
+WHERE appears0.word_id IN (SELECT word_id FROM word WHERE word in ('toto', 'tata'))"""),
             )):
             yield t
 
 
 
 class MySQLGenerator(PostgresSQLGeneratorTC):
-
-    def setUp(self):
-        RQLGeneratorTC.setUp(self)
-        dbhelper = get_db_helper('mysql')
-        self.o = SQLGenerator(schema, dbhelper)
+    backend = 'mysql'
 
     def _norm_sql(self, sql):
         sql = sql.strip().replace(' ILIKE ', ' LIKE ').replace('TRUE', '1').replace('FALSE', '0')
diff -r c397819f2482 -r 1e73a466aa69 test/unittest_entity.py
--- a/test/unittest_entity.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/test/unittest_entity.py	Thu Jun 17 14:43:16 2010 +0200
@@ -442,8 +442,8 @@
         e['data_format'] = 'text/html'
         e['data_encoding'] = 'ascii'
         e._cw.transaction_data = {} # XXX req should be a session
-        self.assertEquals(set(e.cw_adapt_to('IFTIndexable').get_words()),
-                          set(['an', 'html', 'file', 'du', 'html', 'some', 'data']))
+        self.assertEquals(e.cw_adapt_to('IFTIndexable').get_words(),
+                          {'C': [u'du', u'html', 'an', 'html', 'file', u'some', u'data']})
 
 
     def test_nonregr_relation_cache(self):
diff -r c397819f2482 -r 1e73a466aa69 web/facet.py
--- a/web/facet.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/web/facet.py	Thu Jun 17 14:43:16 2010 +0200
@@ -17,8 +17,8 @@
 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
 """contains utility functions and some visual component to restrict results of
 a search
+"""
 
-"""
 __docformat__ = "restructuredtext en"
 
 from copy import deepcopy
diff -r c397819f2482 -r 1e73a466aa69 web/test/test_views.py
--- a/web/test/test_views.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/web/test/test_views.py	Thu Jun 17 14:43:16 2010 +0200
@@ -15,9 +15,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public License along
 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
-"""automatic tests
-
-"""
+"""automatic tests"""
 
 from cubicweb.devtools.testlib import CubicWebTC, AutoPopulateTest, AutomaticWebTest
 from cubicweb.view import AnyRsetView
diff -r c397819f2482 -r 1e73a466aa69 web/test/unittest_magicsearch.py
--- a/web/test/unittest_magicsearch.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/web/test/unittest_magicsearch.py	Thu Jun 17 14:43:16 2010 +0200
@@ -16,9 +16,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public License along
 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
-"""Unit tests for magic_search service
-
-"""
+"""Unit tests for cw.web.views.magicsearch"""
 
 import sys
 
@@ -128,11 +126,11 @@
         self.assertEquals(transform('CWUser', 'E'),
                           ("CWUser E",))
         self.assertEquals(transform('CWUser', 'Smith'),
-                          ('CWUser C WHERE C has_text %(text)s', {'text': 'Smith'}))
+                          ('CWUser C ORDERBY FTIRANK(C) DESC WHERE C has_text %(text)s', {'text': 'Smith'}))
         self.assertEquals(transform('utilisateur', 'Smith'),
-                          ('CWUser C WHERE C has_text %(text)s', {'text': 'Smith'}))
+                          ('CWUser C ORDERBY FTIRANK(C) DESC WHERE C has_text %(text)s', {'text': 'Smith'}))
         self.assertEquals(transform(u'adresse', 'Logilab'),
-                          ('EmailAddress E WHERE E has_text %(text)s', {'text': 'Logilab'}))
+                          ('EmailAddress E ORDERBY FTIRANK(E) DESC WHERE E has_text %(text)s', {'text': 'Logilab'}))
         self.assertEquals(transform(u'adresse', 'Logi%'),
                           ('EmailAddress E WHERE E alias LIKE %(text)s', {'text': 'Logi%'}))
         self.assertRaises(BadRQLQuery, transform, "pers", "taratata")
@@ -152,7 +150,7 @@
                           ('CWUser C WHERE C firstname LIKE %(text)s', {'text': 'cubicweb%'}))
         # expanded shortcuts
         self.assertEquals(transform('CWUser', 'use_email', 'Logilab'),
-                          ('CWUser C WHERE C use_email C1, C1 has_text %(text)s', {'text': 'Logilab'}))
+                          ('CWUser C ORDERBY FTIRANK(C1) DESC WHERE C use_email C1, C1 has_text %(text)s', {'text': 'Logilab'}))
         self.assertEquals(transform('CWUser', 'use_email', '%Logilab'),
                           ('CWUser C WHERE C use_email C1, C1 alias LIKE %(text)s', {'text': '%Logilab'}))
         self.assertRaises(BadRQLQuery, transform, 'word1', 'word2', 'word3')
@@ -160,7 +158,7 @@
     def test_quoted_queries(self):
         """tests how quoted queries are handled"""
         queries = [
-            (u'Adresse "My own EmailAddress"', ('EmailAddress E WHERE E has_text %(text)s', {'text': u'My own EmailAddress'})),
+            (u'Adresse "My own EmailAddress"', ('EmailAddress E ORDERBY FTIRANK(E) DESC WHERE E has_text %(text)s', {'text': u'My own EmailAddress'})),
             (u'Utilisateur prénom "Jean Paul"', ('CWUser C WHERE C firstname %(text)s', {'text': 'Jean Paul'})),
             (u'Utilisateur firstname "Jean Paul"', ('CWUser C WHERE C firstname %(text)s', {'text': 'Jean Paul'})),
             (u'CWUser firstname "Jean Paul"', ('CWUser C WHERE C firstname %(text)s', {'text': 'Jean Paul'})),
@@ -177,7 +175,7 @@
         queries = [
             (u'Utilisateur', (u"CWUser C",)),
             (u'Utilisateur P', (u"CWUser P",)),
-            (u'Utilisateur cubicweb', (u'CWUser C WHERE C has_text %(text)s', {'text': u'cubicweb'})),
+            (u'Utilisateur cubicweb', (u'CWUser C ORDERBY FTIRANK(C) DESC WHERE C has_text %(text)s', {'text': u'cubicweb'})),
             (u'CWUser prénom cubicweb', (u'CWUser C WHERE C firstname %(text)s', {'text': 'cubicweb'},)),
             ]
         for query, expected in queries:
@@ -203,11 +201,11 @@
         """tests QUERY_PROCESSOR"""
         queries = [
             (u'foo',
-             ("Any X WHERE X has_text %(text)s", {'text': u'foo'})),
+             ("Any X ORDERBY FTIRANK(X) DESC WHERE X has_text %(text)s", {'text': u'foo'})),
             # XXX this sounds like a language translator test...
             # and it fails
             (u'Utilisateur Smith',
-             ('CWUser C WHERE C has_text %(text)s', {'text': u'Smith'})),
+             ('CWUser C ORDERBY FTIRANK(C) DESC WHERE C has_text %(text)s', {'text': u'Smith'})),
             (u'utilisateur nom Smith',
              ('CWUser C WHERE C surname %(text)s', {'text': u'Smith'})),
             (u'Any P WHERE P is Utilisateur, P nom "Smith"',
@@ -217,11 +215,11 @@
             rset = self.proc.process_query(query)
             self.assertEquals((rset.rql, rset.args), expected)
 
-    def test_iso88591_fulltext(self):
+    def test_accentuated_fulltext(self):
         """we must be able to type accentuated characters in the search field"""
-        rset = self.proc.process_query(u'écrire')
-        self.assertEquals(rset.rql, "Any X WHERE X has_text %(text)s")
-        self.assertEquals(rset.args, {'text': u'écrire'})
+        rset = self.proc.process_query(u'écrire')
+        self.assertEquals(rset.rql, "Any X ORDERBY FTIRANK(X) DESC WHERE X has_text %(text)s")
+        self.assertEquals(rset.args, {'text': u'écrire'})
 
     def test_explicit_component(self):
         self.assertRaises(RQLSyntaxError,
@@ -229,7 +227,7 @@
         self.assertRaises(BadRQLQuery,
                           self.proc.process_query, u'rql: CWUser E WHERE E noattr "Smith"')
         rset = self.proc.process_query(u'text: utilisateur Smith')
-        self.assertEquals(rset.rql, 'Any X WHERE X has_text %(text)s')
+        self.assertEquals(rset.rql, 'Any X ORDERBY FTIRANK(X) DESC WHERE X has_text %(text)s')
         self.assertEquals(rset.args, {'text': u'utilisateur Smith'})
 
 if __name__ == '__main__':
diff -r c397819f2482 -r 1e73a466aa69 web/views/magicsearch.py
--- a/web/views/magicsearch.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/web/views/magicsearch.py	Thu Jun 17 14:43:16 2010 +0200
@@ -15,10 +15,7 @@
 #
 # You should have received a copy of the GNU Lesser General Public License along
 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
-"""a query preprocesser to handle quick search shortcuts for cubicweb
-
-
-"""
+"""a query processor to handle quick search shortcuts for cubicweb"""
 
 __docformat__ = "restructuredtext en"
 
@@ -282,7 +279,13 @@
         if len(word2) == 1 and word2.isupper():
             return '%s %s' % (etype, word2),
         # else, suppose it's a shortcut like : Person Smith
-        rql = '%s %s WHERE %s' % (etype, etype[0], self._complete_rql(word2, etype))
+        restriction = self._complete_rql(word2, etype)
+        if ' has_text ' in restriction:
+            rql = '%s %s ORDERBY FTIRANK(%s) DESC WHERE %s' % (
+                etype, etype[0], etype[0], restriction)
+        else:
+            rql = '%s %s WHERE %s' % (
+                etype, etype[0], restriction)
         return rql, {'text': word2}
 
     def _three_words_query(self, word1, word2, word3):
@@ -314,10 +317,17 @@
         # by 'rtype'
         mainvar = etype[0]
         searchvar = mainvar  + '1'
-        rql =  '%s %s WHERE %s %s %s, %s' % (etype, mainvar,  # Person P
-                                             mainvar, rtype, searchvar, # P worksAt C
-                                             self._complete_rql(searchstr, etype,
-                                                                rtype=rtype, var=searchvar))
+        restriction = self._complete_rql(searchstr, etype, rtype=rtype,
+                                         var=searchvar)
+        if ' has_text ' in restriction:
+            rql =  ('%s %s ORDERBY FTIRANK(%s) DESC '
+                    'WHERE %s %s %s, %s' % (etype, mainvar, searchvar,
+                                            mainvar, rtype, searchvar, # P worksAt C
+                                            restriction))
+        else:
+            rql =  ('%s %s WHERE %s %s %s, %s' % (etype, mainvar,
+                                            mainvar, rtype, searchvar, # P worksAt C
+                                            restriction))
         return rql, {'text': searchstr}
 
 
@@ -352,7 +362,7 @@
 
     def preprocess_query(self, uquery):
         """suppose it's a plain text query"""
-        return 'Any X WHERE X has_text %(text)s', {'text': uquery}
+        return 'Any X ORDERBY FTIRANK(X) DESC WHERE X has_text %(text)s', {'text': uquery}
 
 
 
@@ -385,7 +395,6 @@
                     try:
                         return proc.process_query(uquery)
                     except TypeError, exc: # cw 3.5 compat
-                        print "EXC", exc
                         warn("[3.6] %s.%s.process_query() should now accept uquery "
                              "as unique argument, use self._cw instead of req"
                              % (proc.__module__, proc.__class__.__name__),
diff -r c397819f2482 -r 1e73a466aa69 web/views/urlrewrite.py
--- a/web/views/urlrewrite.py	Thu Jun 17 12:13:38 2010 +0200
+++ b/web/views/urlrewrite.py	Thu Jun 17 14:43:16 2010 +0200
@@ -207,7 +207,7 @@
     __regid__ = 'schemabased'
     rules = [
         # rgxp : callback
-        (rgx('/search/(.+)'), build_rset(rql=r'Any X WHERE X has_text %(text)s',
+        (rgx('/search/(.+)'), build_rset(rql=r'Any X ORDERBY FTIRANK(X) DESC WHERE X has_text %(text)s',
                                          rgxgroups=[('text', 1)])),
         ]