[fti] support for fti ranking: has_text query results sorted by relevance, and provides a way to control weight per entity / entity's attribute
--- a/__pkginfo__.py Thu Jun 17 12:13:38 2010 +0200
+++ b/__pkginfo__.py Thu Jun 17 14:43:16 2010 +0200
@@ -52,7 +52,7 @@
'Twisted': '',
# XXX graphviz
# server dependencies
- 'logilab-database': '',
+ 'logilab-database': '1.1.0',
'pysqlite': '>= 2.5.5', # XXX install pysqlite2
}
--- a/cwconfig.py Thu Jun 17 12:13:38 2010 +0200
+++ b/cwconfig.py Thu Jun 17 14:43:16 2010 +0200
@@ -1107,6 +1107,7 @@
def register_stored_procedures():
from logilab.database import FunctionDescr
from rql.utils import register_function, iter_funcnode_variables
+ from rql.nodes import SortTerm, Constant, VariableRef
global _EXT_REGISTERED
if _EXT_REGISTERED:
@@ -1152,6 +1153,34 @@
register_function(TEXT_LIMIT_SIZE)
+ class FTIRANK(FunctionDescr):
+ """return ranking of a variable that must be used as some has_text
+ relation subject in the query's restriction. Usually used to sort result
+ of full-text search by ranking.
+ """
+ supported_backends = ('postgres',)
+ rtype = 'Float'
+
+ def st_check_backend(self, backend, funcnode):
+ """overriden so that on backend not supporting fti ranking, the
+ function is removed when in an orderby clause, or replaced by a 1.0
+ constant.
+ """
+ if not self.supports(backend):
+ parent = funcnode.parent
+ while parent is not None and not isinstance(parent, SortTerm):
+ parent = parent.parent
+ if isinstance(parent, SortTerm):
+ parent.parent.remove(parent)
+ else:
+ funcnode.parent.replace(funcnode, Constant(1.0, 'Float'))
+ parent = funcnode
+ for vref in parent.iget_nodes(VariableRef):
+ vref.unregister_reference()
+
+ register_function(FTIRANK)
+
+
class FSPATH(FunctionDescr):
"""return path of some bytes attribute stored using the Bytes
File-System Storage (bfss)
--- a/debian/control Thu Jun 17 12:13:38 2010 +0200
+++ b/debian/control Thu Jun 17 14:43:16 2010 +0200
@@ -33,7 +33,7 @@
Conflicts: cubicweb-multisources
Replaces: cubicweb-multisources
Provides: cubicweb-multisources
-Depends: ${python:Depends}, cubicweb-common (= ${source:Version}), cubicweb-ctl (= ${source:Version}), python-logilab-database (>= 1.0.2), cubicweb-postgresql-support | cubicweb-mysql-support | python-pysqlite2
+Depends: ${python:Depends}, cubicweb-common (= ${source:Version}), cubicweb-ctl (= ${source:Version}), python-logilab-database (>= 1.1.0), cubicweb-postgresql-support | cubicweb-mysql-support | python-pysqlite2
Recommends: pyro, cubicweb-documentation (= ${source:Version})
Description: server part of the CubicWeb framework
CubicWeb is a semantic web application framework.
--- a/devtools/fake.py Thu Jun 17 12:13:38 2010 +0200
+++ b/devtools/fake.py Thu Jun 17 14:43:16 2010 +0200
@@ -16,8 +16,8 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""Fake objects to ease testing of cubicweb without a fully working environment
+"""
-"""
__docformat__ = "restructuredtext en"
from logilab.database import get_db_helper
@@ -46,7 +46,7 @@
return self._cubes
def sources(self):
- return {}
+ return {'system': {'db-driver': 'sqlite'}}
class FakeRequest(CubicWebRequestBase):
--- a/devtools/repotest.py Thu Jun 17 12:13:38 2010 +0200
+++ b/devtools/repotest.py Thu Jun 17 14:43:16 2010 +0200
@@ -18,8 +18,8 @@
"""some utilities to ease repository testing
This module contains functions to initialize a new repository.
+"""
-"""
__docformat__ = "restructuredtext en"
from pprint import pprint
@@ -134,24 +134,32 @@
schema._eid_index[rdef.eid] = rdef
-from logilab.common.testlib import TestCase
+from logilab.common.testlib import TestCase, mock_object
+from logilab.database import get_db_helper
+
from rql import RQLHelper
+
from cubicweb.devtools.fake import FakeRepo, FakeSession
from cubicweb.server import set_debug
from cubicweb.server.querier import QuerierHelper
from cubicweb.server.session import Session
-from cubicweb.server.sources.rql2sql import remove_unused_solutions
+from cubicweb.server.sources.rql2sql import SQLGenerator, remove_unused_solutions
class RQLGeneratorTC(TestCase):
- schema = None # set this in concret test
+ schema = backend = None # set this in concret test
def setUp(self):
self.repo = FakeRepo(self.schema)
+ self.repo.system_source = mock_object(dbdriver=self.backend)
self.rqlhelper = RQLHelper(self.schema, special_relations={'eid': 'uid',
- 'has_text': 'fti'})
+ 'has_text': 'fti'},
+ backend=self.backend)
self.qhelper = QuerierHelper(self.repo, self.schema)
ExecutionPlan._check_permissions = _dummy_check_permissions
rqlannotation._select_principal = _select_principal
+ if self.backend is not None:
+ dbhelper = get_db_helper(self.backend)
+ self.o = SQLGenerator(self.schema, dbhelper)
def tearDown(self):
ExecutionPlan._check_permissions = _orig_check_permissions
@@ -270,6 +278,7 @@
self.system = self.sources[-1]
do_monkey_patch()
self._dumb_sessions = [] # by hi-jacked parent setup
+ self.repo.vreg.rqlhelper.backend = 'postgres' # so FTIRANK is considered
def add_source(self, sourcecls, uri):
self.sources.append(sourcecls(self.repo, self.o.schema,
--- a/entities/adapters.py Thu Jun 17 12:13:38 2010 +0200
+++ b/entities/adapters.py Thu Jun 17 14:43:16 2010 +0200
@@ -108,6 +108,10 @@
else:
yield entity
+ # weight in ABCD
+ entity_weight = 1.0
+ attr_weight = {}
+
def get_words(self):
"""used by the full text indexer to get words to index
@@ -121,10 +125,11 @@
# take care to cases where we're modyfying the schema
entity = self.entity
pending = self._cw.transaction_data.setdefault('pendingrdefs', set())
- words = []
+ words = {}
for rschema in entity.e_schema.indexable_attributes():
if (entity.e_schema, rschema) in pending:
continue
+ weight = self.attr_weight.get(rschema, 'C')
try:
value = entity.printable_value(rschema, format='text/plain')
except TransformError:
@@ -134,16 +139,19 @@
rschema, entity.eid)
continue
if value:
- words += tokenize(value)
+ words.setdefault(weight, []).extend(tokenize(value))
for rschema, role in entity.e_schema.fulltext_relations():
if role == 'subject':
for entity_ in getattr(entity, rschema.type):
- words += entity_.cw_adapt_to('IFTIndexable').get_words()
+ merge_weight_dict(words, entity_.cw_adapt_to('IFTIndexable').get_words())
else: # if role == 'object':
for entity_ in getattr(entity, 'reverse_%s' % rschema.type):
- words += entity_.cw_adapt_to('IFTIndexable').get_words()
+ merge_weight_dict(words, entity_.cw_adapt_to('IFTIndexable').get_words())
return words
+def merge_weight_dict(maindict, newdict):
+ for weight, words in newdict.iteritems():
+ maindict.setdefault(weight, []).extend(words)
class IDownloadableAdapter(EntityAdapter):
"""interface for downloadable entities"""
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/misc/migration/3.9.0_Any.py Thu Jun 17 14:43:16 2010 +0200
@@ -0,0 +1,3 @@
+if repo.system_source.dbdriver == 'postgres':
+ sql('ALTER TABLE appears ADD COLUMN weight float')
+ sql('UPDATE appears SET weight=1.0 ')
--- a/schema.py Thu Jun 17 12:13:38 2010 +0200
+++ b/schema.py Thu Jun 17 14:43:16 2010 +0200
@@ -577,7 +577,6 @@
except BadSchemaDefinition:
reversed_etype_map = dict( (v, k) for k, v in ETYPE_NAME_MAP.iteritems() )
if rdef.subject in reversed_etype_map or rdef.object in reversed_etype_map:
- self.warning('huuuu')
return
raise
if rdefs:
--- a/server/msplanner.py Thu Jun 17 12:13:38 2010 +0200
+++ b/server/msplanner.py Thu Jun 17 14:43:16 2010 +0200
@@ -96,7 +96,7 @@
from rql.stmts import Union, Select
from rql.nodes import (VariableRef, Comparison, Relation, Constant, Variable,
- Not, Exists)
+ Not, Exists, SortTerm, Function)
from cubicweb import server
from cubicweb.utils import make_uid
@@ -1330,6 +1330,12 @@
orderby.append)
if orderby:
newroot.set_orderby(orderby)
+ elif rqlst.orderby:
+ for sortterm in rqlst.orderby:
+ if any(f for f in sortterm.iget_nodes(Function) if f.name == 'FTIRANK'):
+ newnode, oldnode = sortterm.accept(self, newroot, terms)
+ if newnode is not None:
+ newroot.add_sort_term(newnode)
self.process_selection(newroot, terms, rqlst)
elif not newroot.where:
# no restrictions have been copied, just select terms and add
@@ -1530,12 +1536,38 @@
copy.operator = '='
return copy, node
+ def visit_function(self, node, newroot, terms):
+ if node.name == 'FTIRANK':
+ # FTIRANK is somewhat special... Rank function should be included in
+ # the same query has the has_text relation, potentially added to
+ # selection for latter usage
+ if not self.hasaggrstep and self.final and node not in self.skip:
+ return self.visit_default(node, newroot, terms)
+ elif any(s for s in self.sources if s.uri != 'system'):
+ return None, node
+ # p = node.parent
+ # while p is not None and not isinstance(p, SortTerm):
+ # p = p.parent
+ # if isinstance(p, SortTerm):
+ if not self.hasaggrstep and self.final and node in self.skip:
+ return Constant(self.skip[node], 'Int'), node
+ # XXX only if not yet selected
+ newroot.append_selected(node.copy(newroot))
+ self.skip[node] = len(newroot.selection)
+ return None, node
+ return self.visit_default(node, newroot, terms)
+
def visit_default(self, node, newroot, terms):
subparts, node = self._visit_children(node, newroot, terms)
return copy_node(newroot, node, subparts), node
- visit_mathexpression = visit_constant = visit_function = visit_default
- visit_sort = visit_sortterm = visit_default
+ visit_mathexpression = visit_constant = visit_default
+
+ def visit_sortterm(self, node, newroot, terms):
+ subparts, node = self._visit_children(node, newroot, terms)
+ if not subparts:
+ return None, node
+ return copy_node(newroot, node, subparts), node
def _visit_children(self, node, newroot, terms):
subparts = []
--- a/server/mssteps.py Thu Jun 17 12:13:38 2010 +0200
+++ b/server/mssteps.py Thu Jun 17 14:43:16 2010 +0200
@@ -140,13 +140,6 @@
def mytest_repr(self):
"""return a representation of this step suitable for test"""
- sel = self.select.selection
- restr = self.select.where
- self.select.selection = self.selection
- self.select.where = None
- rql = self.select.as_string(kwargs=self.plan.args)
- self.select.selection = sel
- self.select.where = restr
try:
# rely on a monkey patch (cf unittest_querier)
table = self.plan.tablesinorder[self.table]
@@ -155,12 +148,19 @@
# not monkey patched
table = self.table
outputtable = self.outputtable
- return (self.__class__.__name__, rql, self.limit, self.offset, table,
- outputtable)
+ sql = self.get_sql().replace(self.table, table)
+ return (self.__class__.__name__, sql, outputtable)
def execute(self):
"""execute this step"""
self.execute_children()
+ sql = self.get_sql()
+ if self.outputtable:
+ self.plan.create_temp_table(self.outputtable)
+ sql = 'INSERT INTO %s %s' % (self.outputtable, sql)
+ return self.plan.sqlexec(sql, self.plan.args)
+
+ def get_sql(self):
self.inputmap = inputmap = self.children[-1].outputmap
# get the select clause
clause = []
@@ -223,17 +223,15 @@
sql.append('LIMIT %s' % self.limit)
if self.offset:
sql.append('OFFSET %s' % self.offset)
- #print 'DATA', plan.sqlexec('SELECT * FROM %s' % self.table, None)
- sql = ' '.join(sql)
- if self.outputtable:
- self.plan.create_temp_table(self.outputtable)
- sql = 'INSERT INTO %s %s' % (self.outputtable, sql)
- return self.plan.sqlexec(sql, self.plan.args)
+ return ' '.join(sql)
def visit_function(self, function):
"""generate SQL name for a function"""
- return '%s(%s)' % (function.name,
- ','.join(c.accept(self) for c in function.children))
+ try:
+ return self.children[0].outputmap[str(function)]
+ except KeyError:
+ return '%s(%s)' % (function.name,
+ ','.join(c.accept(self) for c in function.children))
def visit_variableref(self, variableref):
"""get the sql name for a variable reference"""
--- a/server/querier.py Thu Jun 17 12:13:38 2010 +0200
+++ b/server/querier.py Thu Jun 17 14:43:16 2010 +0200
@@ -29,7 +29,7 @@
from logilab.common.compat import any
from rql import RQLSyntaxError
from rql.stmts import Union, Select
-from rql.nodes import Relation, VariableRef, Constant, SubQuery
+from rql.nodes import Relation, VariableRef, Constant, SubQuery, Function
from cubicweb import Unauthorized, QueryError, UnknownEid, typed_eid
from cubicweb import server
@@ -50,7 +50,8 @@
key = term.as_string()
value = '%s.C%s' % (table, i)
if varmap.get(key, value) != value:
- raise Exception('variable name conflict on %s' % key)
+ raise Exception('variable name conflict on %s: got %s / %s'
+ % (key, value, varmap))
varmap[key] = value
# permission utilities ########################################################
@@ -285,7 +286,26 @@
for term in origselection:
newselect.append_selected(term.copy(newselect))
if select.orderby:
- newselect.set_orderby([s.copy(newselect) for s in select.orderby])
+ sortterms = []
+ for sortterm in select.orderby:
+ sortterms.append(sortterm.copy(newselect))
+ for fnode in sortterm.get_nodes(Function):
+ if fnode.name == 'FTIRANK':
+ # we've to fetch the has_text relation as well
+ var = fnode.children[0].variable
+ rel = iter(var.stinfo['ftirels']).next()
+ assert not rel.ored(), 'unsupported'
+ newselect.add_restriction(rel.copy(newselect))
+ # remove relation from the orig select and
+ # cleanup variable stinfo
+ rel.parent.remove(rel)
+ var.stinfo['ftirels'].remove(rel)
+ var.stinfo['relations'].remove(rel)
+ # XXX not properly re-annotated after security insertion?
+ newvar = newselect.get_variable(var.name)
+ newvar.stinfo.setdefault('ftirels', set()).add(rel)
+ newvar.stinfo.setdefault('relations', set()).add(rel)
+ newselect.set_orderby(sortterms)
_expand_selection(select.orderby, selected, aliases, select, newselect)
select.orderby = () # XXX dereference?
if select.groupby:
@@ -562,6 +582,8 @@
# rql parsing / analysing helper
self.solutions = repo.vreg.solutions
rqlhelper = repo.vreg.rqlhelper
+ # set backend on the rql helper, will be used for function checking
+ rqlhelper.backend = repo.config.sources()['system']['db-driver']
self._parse = rqlhelper.parse
self._annotate = rqlhelper.annotate
# rql planner
--- a/server/sources/native.py Thu Jun 17 12:13:38 2010 +0200
+++ b/server/sources/native.py Thu Jun 17 14:43:16 2010 +0200
@@ -22,8 +22,8 @@
from which it comes from) are stored in a varchar column encoded as a base64
string. This is because it should actually be Bytes but we want an index on
it for fast querying.
+"""
-"""
from __future__ import with_statement
__docformat__ = "restructuredtext en"
--- a/server/sources/rql2sql.py Thu Jun 17 12:13:38 2010 +0200
+++ b/server/sources/rql2sql.py Thu Jun 17 14:43:16 2010 +0200
@@ -568,12 +568,14 @@
sql += '\nHAVING %s' % having
# sort
if sorts:
- sql += '\nORDER BY %s' % ','.join(self._sortterm_sql(sortterm,
- fselectidx)
- for sortterm in sorts)
- if fneedwrap:
- selection = ['T1.C%s' % i for i in xrange(len(origselection))]
- sql = 'SELECT %s FROM (%s) AS T1' % (','.join(selection), sql)
+ sqlsortterms = [self._sortterm_sql(sortterm, fselectidx)
+ for sortterm in sorts]
+ sqlsortterms = [x for x in sqlsortterms if x is not None]
+ if sqlsortterms:
+ sql += '\nORDER BY %s' % ','.join(sqlsortterms)
+ if sorts and fneedwrap:
+ selection = ['T1.C%s' % i for i in xrange(len(origselection))]
+ sql = 'SELECT %s FROM (%s) AS T1' % (','.join(selection), sql)
state.finalize_source_cbs()
finally:
select.selection = origselection
@@ -651,12 +653,14 @@
def _sortterm_sql(self, sortterm, selectidx):
term = sortterm.term
try:
- sqlterm = str(selectidx.index(str(term)) + 1)
+ sqlterm = selectidx.index(str(term)) + 1
except ValueError:
# Constant node or non selected term
- sqlterm = str(term.accept(self))
+ sqlterm = term.accept(self)
+ if sqlterm is None:
+ return None
if sortterm.asc:
- return sqlterm
+ return str(sqlterm)
else:
return '%s DESC' % sqlterm
@@ -1014,7 +1018,8 @@
not_ = True
else:
not_ = False
- return self.dbhelper.fti_restriction_sql(alias, const.eval(self._args),
+ query = const.eval(self._args)
+ return self.dbhelper.fti_restriction_sql(alias, query,
jointo, not_) + restriction
def visit_comparison(self, cmp):
@@ -1057,6 +1062,15 @@
def visit_function(self, func):
"""generate SQL name for a function"""
+ if func.name == 'FTIRANK':
+ try:
+ rel = iter(func.children[0].variable.stinfo['ftirels']).next()
+ except KeyError:
+ raise BadRQLQuery("can't use FTIRANK on variable not used in an"
+ " 'has_text' relation (eg full-text search)")
+ const = rel.get_parts()[1].children[0]
+ return self.dbhelper.fti_rank_order(self._fti_table(rel),
+ const.eval(self._args))
args = [c.accept(self) for c in func.children]
if func in self._state.source_cb_funcs:
# function executed as a callback on the source
--- a/server/sqlutils.py Thu Jun 17 12:13:38 2010 +0200
+++ b/server/sqlutils.py Thu Jun 17 14:43:16 2010 +0200
@@ -15,9 +15,8 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-"""SQL utilities functions and classes.
+"""SQL utilities functions and classes."""
-"""
__docformat__ = "restructuredtext en"
import os
--- a/server/test/data/site_cubicweb.py Thu Jun 17 12:13:38 2010 +0200
+++ b/server/test/data/site_cubicweb.py Thu Jun 17 14:43:16 2010 +0200
@@ -15,9 +15,6 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-"""
-
-"""
from logilab.database import FunctionDescr
from logilab.database.sqlite import register_sqlite_pyfunc
@@ -25,7 +22,7 @@
try:
class DUMB_SORT(FunctionDescr):
- supported_backends = ('sqlite',)
+ pass
register_function(DUMB_SORT)
def dumb_sort(something):
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/server/test/data/sources_fti Thu Jun 17 14:43:16 2010 +0200
@@ -0,0 +1,14 @@
+[system]
+
+db-driver = postgres
+db-host = localhost
+db-port =
+adapter = native
+db-name = cw_fti_test
+db-encoding = UTF-8
+db-user = syt
+db-password = syt
+
+[admin]
+login = admin
+password = gingkow
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/server/test/unittest_fti.py Thu Jun 17 14:43:16 2010 +0200
@@ -0,0 +1,52 @@
+from __future__ import with_statement
+
+from cubicweb.devtools import ApptestConfiguration
+from cubicweb.devtools.testlib import CubicWebTC
+from cubicweb.selectors import implements
+from cubicweb.entities.adapters import IFTIndexableAdapter
+
+class PostgresFTITC(CubicWebTC):
+ config = ApptestConfiguration('data', sourcefile='sources_fti')
+
+ def test_occurence_count(self):
+ req = self.request()
+ c1 = req.create_entity('Card', title=u'c1',
+ content=u'cubicweb cubicweb cubicweb')
+ c2 = req.create_entity('Card', title=u'c3',
+ content=u'cubicweb')
+ c3 = req.create_entity('Card', title=u'c2',
+ content=u'cubicweb cubicweb')
+ self.commit()
+ self.assertEquals(req.execute('Card X ORDERBY FTIRANK(X) DESC WHERE X has_text "cubicweb"').rows,
+ [[c1.eid], [c3.eid], [c2.eid]])
+
+
+ def test_attr_weight(self):
+ class CardIFTIndexableAdapter(IFTIndexableAdapter):
+ __select__ = implements('Card')
+ attr_weight = {'title': 'A'}
+ with self.temporary_appobjects(CardIFTIndexableAdapter):
+ req = self.request()
+ c1 = req.create_entity('Card', title=u'c1',
+ content=u'cubicweb cubicweb cubicweb')
+ c2 = req.create_entity('Card', title=u'c2',
+ content=u'cubicweb cubicweb')
+ c3 = req.create_entity('Card', title=u'cubicweb',
+ content=u'autre chose')
+ self.commit()
+ self.assertEquals(req.execute('Card X ORDERBY FTIRANK(X) DESC WHERE X has_text "cubicweb"').rows,
+ [[c3.eid], [c1.eid], [c2.eid]])
+
+
+ def test_entity_weight(self):
+ class PersonneIFTIndexableAdapter(IFTIndexableAdapter):
+ __select__ = implements('Personne')
+ entity_weight = 2.0
+ with self.temporary_appobjects(PersonneIFTIndexableAdapter):
+ req = self.request()
+ c1 = req.create_entity('Personne', nom=u'c1', prenom=u'cubicweb')
+ c2 = req.create_entity('Comment', content=u'cubicweb cubicweb', comments=c1)
+ c3 = req.create_entity('Comment', content=u'cubicweb cubicweb cubicweb', comments=c1)
+ self.commit()
+ self.assertEquals(req.execute('Any X ORDERBY FTIRANK(X) DESC WHERE X has_text "cubicweb"').rows,
+ [[c1.eid], [c3.eid], [c2.eid]])
--- a/server/test/unittest_msplanner.py Thu Jun 17 12:13:38 2010 +0200
+++ b/server/test/unittest_msplanner.py Thu Jun 17 14:43:16 2010 +0200
@@ -413,7 +413,7 @@
"""retrieve CWUser X from both sources and return concatenation of results
"""
self._test('CWUser X ORDERBY X LIMIT 10 OFFSET 10',
- [('AggrStep', 'Any X ORDERBY X', 10, 10, 'table0', None, [
+ [('AggrStep', 'SELECT table0.C0 FROM table0 ORDER BY table0.C0 LIMIT 10 OFFSET 10', None, [
('FetchStep', [('Any X WHERE X is CWUser', [{'X': 'CWUser'}])],
[self.ldap, self.system], {}, {'X': 'table0.C0'}, []),
]),
@@ -423,7 +423,7 @@
"""
# COUNT(X) is kept in sub-step and transformed into SUM(X) in the AggrStep
self._test('Any COUNT(X) WHERE X is CWUser',
- [('AggrStep', 'Any COUNT(X)', None, None, 'table0', None, [
+ [('AggrStep', 'SELECT SUM(table0.C0) FROM table0', None, [
('FetchStep', [('Any COUNT(X) WHERE X is CWUser', [{'X': 'CWUser'}])],
[self.ldap, self.system], {}, {'COUNT(X)': 'table0.C0'}, []),
]),
@@ -498,7 +498,7 @@
def test_complex_ordered(self):
self._test('Any L ORDERBY L WHERE X login L',
- [('AggrStep', 'Any L ORDERBY L', None, None, 'table0', None,
+ [('AggrStep', 'SELECT table0.C0 FROM table0 ORDER BY table0.C0', None,
[('FetchStep', [('Any L WHERE X login L, X is CWUser',
[{'X': 'CWUser', 'L': 'String'}])],
[self.ldap, self.system], {}, {'X.login': 'table0.C0', 'L': 'table0.C0'}, []),
@@ -507,7 +507,7 @@
def test_complex_ordered_limit_offset(self):
self._test('Any L ORDERBY L LIMIT 10 OFFSET 10 WHERE X login L',
- [('AggrStep', 'Any L ORDERBY L', 10, 10, 'table0', None,
+ [('AggrStep', 'SELECT table0.C0 FROM table0 ORDER BY table0.C0 LIMIT 10 OFFSET 10', None,
[('FetchStep', [('Any L WHERE X login L, X is CWUser',
[{'X': 'CWUser', 'L': 'String'}])],
[self.ldap, self.system], {}, {'X.login': 'table0.C0', 'L': 'table0.C0'}, []),
@@ -593,7 +593,7 @@
2. return content of the table sorted
"""
self._test('Any X,F ORDERBY F WHERE X firstname F',
- [('AggrStep', 'Any X,F ORDERBY F', None, None, 'table0', None,
+ [('AggrStep', 'SELECT table0.C0, table0.C1 FROM table0 ORDER BY table0.C1', None,
[('FetchStep', [('Any X,F WHERE X firstname F, X is CWUser',
[{'X': 'CWUser', 'F': 'String'}])],
[self.ldap, self.system], {},
@@ -657,7 +657,7 @@
def test_complex_typed_aggregat(self):
self._test('Any MAX(X) WHERE X is Card',
- [('AggrStep', 'Any MAX(X)', None, None, 'table0', None,
+ [('AggrStep', 'SELECT MAX(table0.C0) FROM table0', None,
[('FetchStep',
[('Any MAX(X) WHERE X is Card', [{'X': 'Card'}])],
[self.cards, self.system], {}, {'MAX(X)': 'table0.C0'}, [])
@@ -1299,9 +1299,66 @@
]),
])
+ def test_has_text_orderby_rank(self):
+ self._test('Any X ORDERBY FTIRANK(X) WHERE X has_text "bla", X firstname "bla"',
+ [('FetchStep', [('Any X WHERE X firstname "bla", X is CWUser', [{'X': 'CWUser'}])],
+ [self.ldap, self.system], None, {'X': 'table0.C0'}, []),
+ ('AggrStep', 'SELECT table1.C1 FROM table1 ORDER BY table1.C0', None, [
+ ('FetchStep', [('Any FTIRANK(X),X WHERE X has_text "bla", X is CWUser',
+ [{'X': 'CWUser'}])],
+ [self.system], {'X': 'table0.C0'}, {'FTIRANK(X)': 'table1.C0', 'X': 'table1.C1'}, []),
+ ('FetchStep', [('Any FTIRANK(X),X WHERE X has_text "bla", X firstname "bla", X is Personne',
+ [{'X': 'Personne'}])],
+ [self.system], {}, {'FTIRANK(X)': 'table1.C0', 'X': 'table1.C1'}, []),
+ ]),
+ ])
+
+ def test_security_has_text_orderby_rank(self):
+ # use a guest user
+ self.session = self.user_groups_session('guests')
+ self._test('Any X ORDERBY FTIRANK(X) WHERE X has_text "bla", X firstname "bla"',
+ [('FetchStep', [('Any X WHERE X firstname "bla", X is CWUser', [{'X': 'CWUser'}])],
+ [self.ldap, self.system], None, {'X': 'table1.C0'}, []),
+ ('UnionFetchStep',
+ [('FetchStep', [('Any X WHERE X firstname "bla", X is Personne', [{'X': 'Personne'}])],
+ [self.system], {}, {'X': 'table0.C0'}, []),
+ ('FetchStep', [('Any X WHERE EXISTS(X owned_by 5), X is CWUser', [{'X': 'CWUser'}])],
+ [self.system], {'X': 'table1.C0'}, {'X': 'table0.C0'}, [])]),
+ ('OneFetchStep', [('Any X ORDERBY FTIRANK(X) WHERE X has_text "bla"',
+ [{'X': 'CWUser'}, {'X': 'Personne'}])],
+ None, None, [self.system], {'X': 'table0.C0'}, []),
+ ])
+
+ def test_has_text_select_rank(self):
+ self._test('Any X, FTIRANK(X) WHERE X has_text "bla", X firstname "bla"',
+ # XXX unecessary duplicate selection
+ [('FetchStep', [('Any X,X WHERE X firstname "bla", X is CWUser', [{'X': 'CWUser'}])],
+ [self.ldap, self.system], None, {'X': 'table0.C1'}, []),
+ ('UnionStep', None, None, [
+ ('OneFetchStep', [('Any X,FTIRANK(X) WHERE X has_text "bla", X is CWUser', [{'X': 'CWUser'}])],
+ None, None, [self.system], {'X': 'table0.C1'}, []),
+ ('OneFetchStep', [('Any X,FTIRANK(X) WHERE X has_text "bla", X firstname "bla", X is Personne', [{'X': 'Personne'}])],
+ None, None, [self.system], {}, []),
+ ]),
+ ])
+
+ def test_security_has_text_select_rank(self):
+ # use a guest user
+ self.session = self.user_groups_session('guests')
+ self._test('Any X, FTIRANK(X) WHERE X has_text "bla", X firstname "bla"',
+ [('FetchStep', [('Any X,X WHERE X firstname "bla", X is CWUser', [{'X': 'CWUser'}])],
+ [self.ldap, self.system], None, {'X': 'table0.C1'}, []),
+ ('UnionStep', None, None, [
+ ('OneFetchStep', [('Any X,FTIRANK(X) WHERE X has_text "bla", EXISTS(X owned_by 5), X is CWUser', [{'X': 'CWUser'}])],
+ None, None, [self.system], {'X': 'table0.C1'}, []),
+ ('OneFetchStep', [('Any X,FTIRANK(X) WHERE X has_text "bla", X firstname "bla", X is Personne', [{'X': 'Personne'}])],
+ None, None, [self.system], {}, []),
+ ]),
+ ])
+
def test_sort_func(self):
self._test('Note X ORDERBY DUMB_SORT(RF) WHERE X type RF',
- [('AggrStep', 'Any X ORDERBY DUMB_SORT(RF)', None, None, 'table0', None, [
+ [('AggrStep', 'SELECT table0.C0 FROM table0 ORDER BY DUMB_SORT(table0.C1)', None, [
('FetchStep', [('Any X,RF WHERE X type RF, X is Note',
[{'X': 'Note', 'RF': 'String'}])],
[self.cards, self.system], {}, {'X': 'table0.C0', 'X.type': 'table0.C1', 'RF': 'table0.C1'}, []),
@@ -1310,8 +1367,7 @@
def test_ambigous_sort_func(self):
self._test('Any X ORDERBY DUMB_SORT(RF) WHERE X title RF, X is IN (Bookmark, Card, EmailThread)',
- [('AggrStep', 'Any X ORDERBY DUMB_SORT(RF)',
- None, None, 'table0', None,
+ [('AggrStep', 'SELECT table0.C0 FROM table0 ORDER BY DUMB_SORT(table0.C1)', None,
[('FetchStep', [('Any X,RF WHERE X title RF, X is Card',
[{'X': 'Card', 'RF': 'String'}])],
[self.cards, self.system], {},
@@ -1728,7 +1784,7 @@
('FetchStep', [('Any X,D WHERE X modification_date D, X is CWUser',
[{'X': 'CWUser', 'D': 'Datetime'}])],
[self.ldap, self.system], None, {'X': 'table1.C0', 'X.modification_date': 'table1.C1', 'D': 'table1.C1'}, []),
- ('AggrStep', 'Any X ORDERBY D DESC', None, None, 'table2', None, [
+ ('AggrStep', 'SELECT table2.C0 FROM table2 ORDER BY table2.C1 DESC', None, [
('FetchStep', [('Any X,D WHERE E eid %s, E wf_info_for X, X modification_date D, E is TrInfo, X is Affaire'%treid,
[{'X': 'Affaire', 'E': 'TrInfo', 'D': 'Datetime'}])],
[self.system],
@@ -1871,8 +1927,7 @@
[{'X': 'Note', 'Z': 'Datetime'}])],
[self.cards, self.system], None, {'X': 'table0.C0', 'X.modification_date': 'table0.C1', 'Z': 'table0.C1'},
[]),
- ('AggrStep', 'Any X ORDERBY Z DESC',
- None, None, 'table1', None,
+ ('AggrStep', 'SELECT table1.C0 FROM table1 ORDER BY table1.C1 DESC', None,
[('FetchStep', [('Any X,Z WHERE X modification_date Z, 999999 see_also X, X is Bookmark',
[{'X': 'Bookmark', 'Z': 'Datetime'}])],
[self.system], {}, {'X': 'table1.C0', 'X.modification_date': 'table1.C1',
--- a/server/test/unittest_multisources.py Thu Jun 17 12:13:38 2010 +0200
+++ b/server/test/unittest_multisources.py Thu Jun 17 14:43:16 2010 +0200
@@ -134,6 +134,8 @@
self.repo.sources_by_uri['extern'].synchronize(MTIME) # in case fti_update has been run before
self.failUnless(self.sexecute('Any X WHERE X has_text "affref"'))
self.failUnless(self.sexecute('Affaire X WHERE X has_text "affref"'))
+ self.failUnless(self.sexecute('Any X ORDERBY FTIRANK(X) WHERE X has_text "affref"'))
+ self.failUnless(self.sexecute('Affaire X ORDERBY FTIRANK(X) WHERE X has_text "affref"'))
def test_anon_has_text(self):
self.repo.sources_by_uri['extern'].synchronize(MTIME) # in case fti_update has been run before
@@ -145,6 +147,9 @@
cnx = self.login('anon')
cu = cnx.cursor()
rset = cu.execute('Any X WHERE X has_text "card"')
+ # 5: 4 card + 1 readable affaire
+ self.assertEquals(len(rset), 5, zip(rset.rows, rset.description))
+ rset = cu.execute('Any X ORDERBY FTIRANK(X) WHERE X has_text "card"')
self.assertEquals(len(rset), 5, zip(rset.rows, rset.description))
Connection_close(cnx)
--- a/server/test/unittest_rql2sql.py Thu Jun 17 12:13:38 2010 +0200
+++ b/server/test/unittest_rql2sql.py Thu Jun 17 14:43:16 2010 +0200
@@ -22,11 +22,13 @@
from logilab.common.testlib import TestCase, unittest_main, mock_object
from rql import BadRQLQuery
+from rql.utils import register_function, FunctionDescr
-#from cubicweb.server.sources.native import remove_unused_solutions
-from cubicweb.server.sources.rql2sql import SQLGenerator, remove_unused_solutions
+from cubicweb.devtools import TestServerConfiguration
+from cubicweb.devtools.repotest import RQLGeneratorTC
+from cubicweb.server.sources.rql2sql import remove_unused_solutions
-from rql.utils import register_function, FunctionDescr
+
# add a dumb registered procedure
class stockproc(FunctionDescr):
supported_backends = ('postgres', 'sqlite', 'mysql')
@@ -35,8 +37,6 @@
except AssertionError, ex:
pass # already registered
-from cubicweb.devtools import TestServerConfiguration
-from cubicweb.devtools.repotest import RQLGeneratorTC
config = TestServerConfiguration('data')
config.bootstrap_cubes()
@@ -1060,11 +1060,9 @@
WHERE rel_is0.eid_to=2'''),
]
-from logilab.database import get_db_helper
-
class CWRQLTC(RQLGeneratorTC):
schema = schema
-
+ backend = 'sqlite'
def test_nonregr_sol(self):
delete = self.rqlhelper.parse(
'DELETE X read_permission READ_PERMISSIONSUBJECT,X add_permission ADD_PERMISSIONSUBJECT,'
@@ -1090,12 +1088,7 @@
class PostgresSQLGeneratorTC(RQLGeneratorTC):
schema = schema
-
- #capture = True
- def setUp(self):
- RQLGeneratorTC.setUp(self)
- dbhelper = get_db_helper('postgres')
- self.o = SQLGenerator(schema, dbhelper)
+ backend = 'postgres'
def _norm_sql(self, sql):
return sql.strip()
@@ -1355,13 +1348,53 @@
UNION ALL
SELECT _X.cw_eid
FROM appears AS appears0, cw_Folder AS _X
-WHERE appears0.words @@ to_tsquery('default', 'toto&tata') AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
-"""),
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata') AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu"""),
('Personne X where X has_text %(text)s, X travaille S, S has_text %(text)s',
"""SELECT _X.eid
FROM appears AS appears0, appears AS appears2, entities AS _X, travaille_relation AS rel_travaille1
WHERE appears0.words @@ to_tsquery('default', 'hip&hop&momo') AND appears0.uid=_X.eid AND _X.type='Personne' AND _X.eid=rel_travaille1.eid_from AND appears2.uid=rel_travaille1.eid_to AND appears2.words @@ to_tsquery('default', 'hip&hop&momo')"""),
+
+ ('Any X ORDERBY FTIRANK(X) DESC WHERE X has_text "toto tata"',
+ """SELECT appears0.uid
+FROM appears AS appears0
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata')
+ORDER BY ts_rank(appears0.words, to_tsquery('default', 'toto&tata'))*appears0.weight DESC"""),
+
+ ('Personne X ORDERBY FTIRANK(X) WHERE X has_text "toto tata"',
+ """SELECT _X.eid
+FROM appears AS appears0, entities AS _X
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata') AND appears0.uid=_X.eid AND _X.type='Personne'
+ORDER BY ts_rank(appears0.words, to_tsquery('default', 'toto&tata'))*appears0.weight"""),
+
+ ('Personne X ORDERBY FTIRANK(X) WHERE X has_text %(text)s',
+ """SELECT _X.eid
+FROM appears AS appears0, entities AS _X
+WHERE appears0.words @@ to_tsquery('default', 'hip&hop&momo') AND appears0.uid=_X.eid AND _X.type='Personne'
+ORDER BY ts_rank(appears0.words, to_tsquery('default', 'hip&hop&momo'))*appears0.weight"""),
+
+ ('Any X ORDERBY FTIRANK(X) WHERE X has_text "toto tata", X name "tutu", X is IN (Basket,Folder)',
+ """SELECT T1.C0 FROM (SELECT _X.cw_eid AS C0, ts_rank(appears0.words, to_tsquery('default', 'toto&tata'))*appears0.weight AS C1
+FROM appears AS appears0, cw_Basket AS _X
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata') AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
+UNION ALL
+SELECT _X.cw_eid AS C0, ts_rank(appears0.words, to_tsquery('default', 'toto&tata'))*appears0.weight AS C1
+FROM appears AS appears0, cw_Folder AS _X
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata') AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
+ORDER BY 2) AS T1"""),
+
+ ('Personne X ORDERBY FTIRANK(X),FTIRANK(S) WHERE X has_text %(text)s, X travaille S, S has_text %(text)s',
+ """SELECT _X.eid
+FROM appears AS appears0, appears AS appears2, entities AS _X, travaille_relation AS rel_travaille1
+WHERE appears0.words @@ to_tsquery('default', 'hip&hop&momo') AND appears0.uid=_X.eid AND _X.type='Personne' AND _X.eid=rel_travaille1.eid_from AND appears2.uid=rel_travaille1.eid_to AND appears2.words @@ to_tsquery('default', 'hip&hop&momo')
+ORDER BY ts_rank(appears0.words, to_tsquery('default', 'hip&hop&momo'))*appears0.weight,ts_rank(appears2.words, to_tsquery('default', 'hip&hop&momo'))*appears2.weight"""),
+
+
+ ('Any X, FTIRANK(X) WHERE X has_text "toto tata"',
+ """SELECT appears0.uid, ts_rank(appears0.words, to_tsquery('default', 'toto&tata'))*appears0.weight
+FROM appears AS appears0
+WHERE appears0.words @@ to_tsquery('default', 'toto&tata')"""),
+
)):
yield t
@@ -1411,11 +1444,7 @@
class SqliteSQLGeneratorTC(PostgresSQLGeneratorTC):
-
- def setUp(self):
- RQLGeneratorTC.setUp(self)
- dbhelper = get_db_helper('sqlite')
- self.o = SQLGenerator(schema, dbhelper)
+ backend = 'sqlite'
def _norm_sql(self, sql):
return sql.strip().replace(' ILIKE ', ' LIKE ')
@@ -1513,17 +1542,33 @@
FROM appears AS appears0, cw_Folder AS _X
WHERE appears0.word_id IN (SELECT word_id FROM word WHERE word in ('toto', 'tata')) AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
"""),
+
+ ('Any X ORDERBY FTIRANK(X) WHERE X has_text "toto tata"',
+ """SELECT DISTINCT appears0.uid
+FROM appears AS appears0
+WHERE appears0.word_id IN (SELECT word_id FROM word WHERE word in ('toto', 'tata'))"""),
+
+ ('Any X ORDERBY FTIRANK(X) WHERE X has_text "toto tata", X name "tutu", X is IN (Basket,Folder)',
+ """SELECT DISTINCT _X.cw_eid
+FROM appears AS appears0, cw_Basket AS _X
+WHERE appears0.word_id IN (SELECT word_id FROM word WHERE word in ('toto', 'tata')) AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
+UNION
+SELECT DISTINCT _X.cw_eid
+FROM appears AS appears0, cw_Folder AS _X
+WHERE appears0.word_id IN (SELECT word_id FROM word WHERE word in ('toto', 'tata')) AND appears0.uid=_X.cw_eid AND _X.cw_name=tutu
+"""),
+
+ ('Any X, FTIRANK(X) WHERE X has_text "toto tata"',
+ """SELECT DISTINCT appears0.uid, 1.0
+FROM appears AS appears0
+WHERE appears0.word_id IN (SELECT word_id FROM word WHERE word in ('toto', 'tata'))"""),
)):
yield t
class MySQLGenerator(PostgresSQLGeneratorTC):
-
- def setUp(self):
- RQLGeneratorTC.setUp(self)
- dbhelper = get_db_helper('mysql')
- self.o = SQLGenerator(schema, dbhelper)
+ backend = 'mysql'
def _norm_sql(self, sql):
sql = sql.strip().replace(' ILIKE ', ' LIKE ').replace('TRUE', '1').replace('FALSE', '0')
--- a/test/unittest_entity.py Thu Jun 17 12:13:38 2010 +0200
+++ b/test/unittest_entity.py Thu Jun 17 14:43:16 2010 +0200
@@ -442,8 +442,8 @@
e['data_format'] = 'text/html'
e['data_encoding'] = 'ascii'
e._cw.transaction_data = {} # XXX req should be a session
- self.assertEquals(set(e.cw_adapt_to('IFTIndexable').get_words()),
- set(['an', 'html', 'file', 'du', 'html', 'some', 'data']))
+ self.assertEquals(e.cw_adapt_to('IFTIndexable').get_words(),
+ {'C': [u'du', u'html', 'an', 'html', 'file', u'some', u'data']})
def test_nonregr_relation_cache(self):
--- a/web/facet.py Thu Jun 17 12:13:38 2010 +0200
+++ b/web/facet.py Thu Jun 17 14:43:16 2010 +0200
@@ -17,8 +17,8 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""contains utility functions and some visual component to restrict results of
a search
+"""
-"""
__docformat__ = "restructuredtext en"
from copy import deepcopy
--- a/web/test/test_views.py Thu Jun 17 12:13:38 2010 +0200
+++ b/web/test/test_views.py Thu Jun 17 14:43:16 2010 +0200
@@ -15,9 +15,7 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-"""automatic tests
-
-"""
+"""automatic tests"""
from cubicweb.devtools.testlib import CubicWebTC, AutoPopulateTest, AutomaticWebTest
from cubicweb.view import AnyRsetView
--- a/web/test/unittest_magicsearch.py Thu Jun 17 12:13:38 2010 +0200
+++ b/web/test/unittest_magicsearch.py Thu Jun 17 14:43:16 2010 +0200
@@ -16,9 +16,7 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-"""Unit tests for magic_search service
-
-"""
+"""Unit tests for cw.web.views.magicsearch"""
import sys
@@ -128,11 +126,11 @@
self.assertEquals(transform('CWUser', 'E'),
("CWUser E",))
self.assertEquals(transform('CWUser', 'Smith'),
- ('CWUser C WHERE C has_text %(text)s', {'text': 'Smith'}))
+ ('CWUser C ORDERBY FTIRANK(C) DESC WHERE C has_text %(text)s', {'text': 'Smith'}))
self.assertEquals(transform('utilisateur', 'Smith'),
- ('CWUser C WHERE C has_text %(text)s', {'text': 'Smith'}))
+ ('CWUser C ORDERBY FTIRANK(C) DESC WHERE C has_text %(text)s', {'text': 'Smith'}))
self.assertEquals(transform(u'adresse', 'Logilab'),
- ('EmailAddress E WHERE E has_text %(text)s', {'text': 'Logilab'}))
+ ('EmailAddress E ORDERBY FTIRANK(E) DESC WHERE E has_text %(text)s', {'text': 'Logilab'}))
self.assertEquals(transform(u'adresse', 'Logi%'),
('EmailAddress E WHERE E alias LIKE %(text)s', {'text': 'Logi%'}))
self.assertRaises(BadRQLQuery, transform, "pers", "taratata")
@@ -152,7 +150,7 @@
('CWUser C WHERE C firstname LIKE %(text)s', {'text': 'cubicweb%'}))
# expanded shortcuts
self.assertEquals(transform('CWUser', 'use_email', 'Logilab'),
- ('CWUser C WHERE C use_email C1, C1 has_text %(text)s', {'text': 'Logilab'}))
+ ('CWUser C ORDERBY FTIRANK(C1) DESC WHERE C use_email C1, C1 has_text %(text)s', {'text': 'Logilab'}))
self.assertEquals(transform('CWUser', 'use_email', '%Logilab'),
('CWUser C WHERE C use_email C1, C1 alias LIKE %(text)s', {'text': '%Logilab'}))
self.assertRaises(BadRQLQuery, transform, 'word1', 'word2', 'word3')
@@ -160,7 +158,7 @@
def test_quoted_queries(self):
"""tests how quoted queries are handled"""
queries = [
- (u'Adresse "My own EmailAddress"', ('EmailAddress E WHERE E has_text %(text)s', {'text': u'My own EmailAddress'})),
+ (u'Adresse "My own EmailAddress"', ('EmailAddress E ORDERBY FTIRANK(E) DESC WHERE E has_text %(text)s', {'text': u'My own EmailAddress'})),
(u'Utilisateur prénom "Jean Paul"', ('CWUser C WHERE C firstname %(text)s', {'text': 'Jean Paul'})),
(u'Utilisateur firstname "Jean Paul"', ('CWUser C WHERE C firstname %(text)s', {'text': 'Jean Paul'})),
(u'CWUser firstname "Jean Paul"', ('CWUser C WHERE C firstname %(text)s', {'text': 'Jean Paul'})),
@@ -177,7 +175,7 @@
queries = [
(u'Utilisateur', (u"CWUser C",)),
(u'Utilisateur P', (u"CWUser P",)),
- (u'Utilisateur cubicweb', (u'CWUser C WHERE C has_text %(text)s', {'text': u'cubicweb'})),
+ (u'Utilisateur cubicweb', (u'CWUser C ORDERBY FTIRANK(C) DESC WHERE C has_text %(text)s', {'text': u'cubicweb'})),
(u'CWUser prénom cubicweb', (u'CWUser C WHERE C firstname %(text)s', {'text': 'cubicweb'},)),
]
for query, expected in queries:
@@ -203,11 +201,11 @@
"""tests QUERY_PROCESSOR"""
queries = [
(u'foo',
- ("Any X WHERE X has_text %(text)s", {'text': u'foo'})),
+ ("Any X ORDERBY FTIRANK(X) DESC WHERE X has_text %(text)s", {'text': u'foo'})),
# XXX this sounds like a language translator test...
# and it fails
(u'Utilisateur Smith',
- ('CWUser C WHERE C has_text %(text)s', {'text': u'Smith'})),
+ ('CWUser C ORDERBY FTIRANK(C) DESC WHERE C has_text %(text)s', {'text': u'Smith'})),
(u'utilisateur nom Smith',
('CWUser C WHERE C surname %(text)s', {'text': u'Smith'})),
(u'Any P WHERE P is Utilisateur, P nom "Smith"',
@@ -217,11 +215,11 @@
rset = self.proc.process_query(query)
self.assertEquals((rset.rql, rset.args), expected)
- def test_iso88591_fulltext(self):
+ def test_accentuated_fulltext(self):
"""we must be able to type accentuated characters in the search field"""
- rset = self.proc.process_query(u'écrire')
- self.assertEquals(rset.rql, "Any X WHERE X has_text %(text)s")
- self.assertEquals(rset.args, {'text': u'écrire'})
+ rset = self.proc.process_query(u'écrire')
+ self.assertEquals(rset.rql, "Any X ORDERBY FTIRANK(X) DESC WHERE X has_text %(text)s")
+ self.assertEquals(rset.args, {'text': u'écrire'})
def test_explicit_component(self):
self.assertRaises(RQLSyntaxError,
@@ -229,7 +227,7 @@
self.assertRaises(BadRQLQuery,
self.proc.process_query, u'rql: CWUser E WHERE E noattr "Smith"')
rset = self.proc.process_query(u'text: utilisateur Smith')
- self.assertEquals(rset.rql, 'Any X WHERE X has_text %(text)s')
+ self.assertEquals(rset.rql, 'Any X ORDERBY FTIRANK(X) DESC WHERE X has_text %(text)s')
self.assertEquals(rset.args, {'text': u'utilisateur Smith'})
if __name__ == '__main__':
--- a/web/views/magicsearch.py Thu Jun 17 12:13:38 2010 +0200
+++ b/web/views/magicsearch.py Thu Jun 17 14:43:16 2010 +0200
@@ -15,10 +15,7 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-"""a query preprocesser to handle quick search shortcuts for cubicweb
-
-
-"""
+"""a query processor to handle quick search shortcuts for cubicweb"""
__docformat__ = "restructuredtext en"
@@ -282,7 +279,13 @@
if len(word2) == 1 and word2.isupper():
return '%s %s' % (etype, word2),
# else, suppose it's a shortcut like : Person Smith
- rql = '%s %s WHERE %s' % (etype, etype[0], self._complete_rql(word2, etype))
+ restriction = self._complete_rql(word2, etype)
+ if ' has_text ' in restriction:
+ rql = '%s %s ORDERBY FTIRANK(%s) DESC WHERE %s' % (
+ etype, etype[0], etype[0], restriction)
+ else:
+ rql = '%s %s WHERE %s' % (
+ etype, etype[0], restriction)
return rql, {'text': word2}
def _three_words_query(self, word1, word2, word3):
@@ -314,10 +317,17 @@
# by 'rtype'
mainvar = etype[0]
searchvar = mainvar + '1'
- rql = '%s %s WHERE %s %s %s, %s' % (etype, mainvar, # Person P
- mainvar, rtype, searchvar, # P worksAt C
- self._complete_rql(searchstr, etype,
- rtype=rtype, var=searchvar))
+ restriction = self._complete_rql(searchstr, etype, rtype=rtype,
+ var=searchvar)
+ if ' has_text ' in restriction:
+ rql = ('%s %s ORDERBY FTIRANK(%s) DESC '
+ 'WHERE %s %s %s, %s' % (etype, mainvar, searchvar,
+ mainvar, rtype, searchvar, # P worksAt C
+ restriction))
+ else:
+ rql = ('%s %s WHERE %s %s %s, %s' % (etype, mainvar,
+ mainvar, rtype, searchvar, # P worksAt C
+ restriction))
return rql, {'text': searchstr}
@@ -352,7 +362,7 @@
def preprocess_query(self, uquery):
"""suppose it's a plain text query"""
- return 'Any X WHERE X has_text %(text)s', {'text': uquery}
+ return 'Any X ORDERBY FTIRANK(X) DESC WHERE X has_text %(text)s', {'text': uquery}
@@ -385,7 +395,6 @@
try:
return proc.process_query(uquery)
except TypeError, exc: # cw 3.5 compat
- print "EXC", exc
warn("[3.6] %s.%s.process_query() should now accept uquery "
"as unique argument, use self._cw instead of req"
% (proc.__module__, proc.__class__.__name__),
--- a/web/views/urlrewrite.py Thu Jun 17 12:13:38 2010 +0200
+++ b/web/views/urlrewrite.py Thu Jun 17 14:43:16 2010 +0200
@@ -207,7 +207,7 @@
__regid__ = 'schemabased'
rules = [
# rgxp : callback
- (rgx('/search/(.+)'), build_rset(rql=r'Any X WHERE X has_text %(text)s',
+ (rgx('/search/(.+)'), build_rset(rql=r'Any X ORDERBY FTIRANK(X) DESC WHERE X has_text %(text)s',
rgxgroups=[('text', 1)])),
]