author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Thu, 16 Jun 2016 14:19:20 +0200 | |
changeset 11278 | 19fcce6dc6d1 |
parent 11080 | 245f440b814a |
permissions | -rw-r--r-- |
0 | 1 |
# -*- coding: iso-8859-1 -*- |
9837
64c8ee99baf7
[devtools/testlib,fill] use the new connection api (for auto_populate)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9521
diff
changeset
|
2 |
# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
3 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
4 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
5 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
6 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
7 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
8 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
9 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
10 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
11 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
12 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
13 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
14 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
15 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
16 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
17 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4651
diff
changeset
|
18 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
5786
7c16a4e7a592
[test] fix stupd name error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
19 |
"""This modules defines func / methods for creating test repositories""" |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
9837
diff
changeset
|
20 |
from __future__ import print_function |
0 | 21 |
|
22 |
__docformat__ = "restructuredtext en" |
|
23 |
||
7995
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7815
diff
changeset
|
24 |
import logging |
0 | 25 |
from random import randint, choice |
26 |
from copy import deepcopy |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
27 |
from datetime import datetime, date, time, timedelta |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
428
diff
changeset
|
28 |
from decimal import Decimal |
10714
8a2ec43fcf44
[py3k] use inspect module to get a function's arguments
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10704
diff
changeset
|
29 |
import inspect |
0 | 30 |
|
10702
f94c812c3669
[py3k] unicode → six.text_type
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10688
diff
changeset
|
31 |
from six import text_type, add_metaclass |
10609
e2d8e81bfe68
[py3k] import range using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
32 |
from six.moves import range |
e2d8e81bfe68
[py3k] import range using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
33 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
34 |
from logilab.common import attrdict |
7995
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7815
diff
changeset
|
35 |
from logilab.mtconverter import xml_escape |
0 | 36 |
from yams.constraints import (SizeConstraint, StaticVocabularyConstraint, |
6916
3970185a8f8c
[devtools] fix understanding of boundary constraint on db automatic population
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
37 |
IntervalBoundConstraint, BoundaryConstraint, |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
38 |
Attribute, actual_value) |
0 | 39 |
from rql.utils import decompose_b26 as base_decompose_b26 |
40 |
||
41 |
from cubicweb import Binary |
|
42 |
from cubicweb.schema import RQLConstraint |
|
43 |
||
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
44 |
def custom_range(start, stop, step): |
11079
eba98431f705
[devtools] "fix" boundary condition for autofill
Julien Cristau <julien.cristau@logilab.fr>
parents:
10714
diff
changeset
|
45 |
if start == stop: |
eba98431f705
[devtools] "fix" boundary condition for autofill
Julien Cristau <julien.cristau@logilab.fr>
parents:
10714
diff
changeset
|
46 |
yield start |
eba98431f705
[devtools] "fix" boundary condition for autofill
Julien Cristau <julien.cristau@logilab.fr>
parents:
10714
diff
changeset
|
47 |
return |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
48 |
while start < stop: |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
49 |
yield start |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
50 |
start += step |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
51 |
|
0 | 52 |
def decompose_b26(index, ascii=False): |
53 |
"""return a letter (base-26) decomposition of index""" |
|
54 |
if ascii: |
|
55 |
return base_decompose_b26(index) |
|
56 |
return base_decompose_b26(index, u'�abcdefghijklmnopqrstuvwxyz') |
|
57 |
||
58 |
def get_max_length(eschema, attrname): |
|
59 |
"""returns the maximum length allowed for 'attrname'""" |
|
4053
7cc66b1d9183
more api update
Sandrine Ribeau <sandrine.ribeau@logilab.fr>
parents:
3749
diff
changeset
|
60 |
for cst in eschema.rdef(attrname).constraints: |
0 | 61 |
if isinstance(cst, SizeConstraint) and cst.max: |
62 |
return cst.max |
|
63 |
return 300 |
|
64 |
#raise AttributeError('No Size constraint on attribute "%s"' % attrname) |
|
65 |
||
66 |
_GENERATED_VALUES = {} |
|
67 |
||
68 |
class _ValueGenerator(object): |
|
69 |
"""generates integers / dates / strings / etc. to fill a DB table""" |
|
70 |
||
71 |
def __init__(self, eschema, choice_func=None): |
|
72 |
"""<choice_func> is a function that returns a list of possible |
|
73 |
choices for a given entity type and an attribute name. It should |
|
74 |
looks like : |
|
75 |
def values_for(etype, attrname): |
|
76 |
# some stuff ... |
|
77 |
return alist_of_acceptable_values # or None |
|
78 |
""" |
|
79 |
self.choice_func = choice_func |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
80 |
self.eschema = eschema |
0 | 81 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
82 |
def generate_attribute_value(self, entity, attrname, index=1, **kwargs): |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
83 |
if attrname in entity: |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
84 |
return entity[attrname] |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
85 |
eschema = self.eschema |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
86 |
if not eschema.has_unique_values(attrname): |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
87 |
value = self.__generate_value(entity, attrname, index, **kwargs) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
88 |
else: |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
89 |
value = self.__generate_value(entity, attrname, index, **kwargs) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
90 |
while value in _GENERATED_VALUES.get((eschema, attrname), ()): |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
91 |
index += 1 |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
92 |
value = self.__generate_value(entity, attrname, index, **kwargs) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
93 |
_GENERATED_VALUES.setdefault((eschema, attrname), set()).add(value) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
94 |
entity[attrname] = value |
0 | 95 |
return value |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
96 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
97 |
def __generate_value(self, entity, attrname, index, **kwargs): |
0 | 98 |
"""generates a consistent value for 'attrname'""" |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
99 |
eschema = self.eschema |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
100 |
attrtype = str(eschema.destination(attrname)).lower() |
0 | 101 |
# Before calling generate_%s functions, try to find values domain |
102 |
if self.choice_func is not None: |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
103 |
values_domain = self.choice_func(eschema, attrname) |
0 | 104 |
if values_domain is not None: |
105 |
return choice(values_domain) |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
106 |
gen_func = getattr(self, 'generate_%s_%s' % (eschema, attrname), |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
107 |
getattr(self, 'generate_Any_%s' % attrname, None)) |
0 | 108 |
if gen_func is not None: |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
109 |
return gen_func(entity, index, **kwargs) |
0 | 110 |
# If no specific values domain, then generate a dummy value |
111 |
gen_func = getattr(self, 'generate_%s' % (attrtype)) |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
112 |
return gen_func(entity, attrname, index, **kwargs) |
0 | 113 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
114 |
def generate_string(self, entity, attrname, index, format=None): |
0 | 115 |
"""generates a consistent value for 'attrname' if it's a string""" |
116 |
# First try to get choices |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
117 |
choosed = self.get_choice(entity, attrname) |
0 | 118 |
if choosed is not None: |
119 |
return choosed |
|
120 |
# All other case, generate a default string |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
121 |
attrlength = get_max_length(self.eschema, attrname) |
0 | 122 |
num_len = numlen(index) |
123 |
if num_len >= attrlength: |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
124 |
ascii = self.eschema.rdef(attrname).internationalizable |
0 | 125 |
return ('&'+decompose_b26(index, ascii))[:attrlength] |
126 |
# always use plain text when no format is specified |
|
127 |
attrprefix = attrname[:max(attrlength-num_len-1, 0)] |
|
128 |
if format == 'text/html': |
|
129 |
value = u'<span>�%s<b>%d</b></span>' % (attrprefix, index) |
|
130 |
elif format == 'text/rest': |
|
131 |
value = u""" |
|
132 |
title |
|
133 |
----- |
|
134 |
||
135 |
* %s |
|
136 |
* %d |
|
137 |
* �& |
|
138 |
""" % (attrprefix, index) |
|
139 |
else: |
|
140 |
value = u'�&%s%d' % (attrprefix, index) |
|
141 |
return value[:attrlength] |
|
142 |
||
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
143 |
def generate_password(self, entity, attrname, index): |
0 | 144 |
"""generates a consistent value for 'attrname' if it's a password""" |
145 |
return u'toto' |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
146 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
147 |
def generate_integer(self, entity, attrname, index): |
0 | 148 |
"""generates a consistent value for 'attrname' if it's an integer""" |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
149 |
return self._constrained_generate(entity, attrname, 0, 1, index) |
9207
c99e97e9f566
[testing] add missing generate_bigint method to ValueGenerator. Closes #3059327
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8238
diff
changeset
|
150 |
generate_int = generate_bigint = generate_integer |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
151 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
152 |
def generate_float(self, entity, attrname, index): |
0 | 153 |
"""generates a consistent value for 'attrname' if it's a float""" |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
154 |
return self._constrained_generate(entity, attrname, 0.0, 1.0, index) |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
155 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
156 |
def generate_decimal(self, entity, attrname, index): |
0 | 157 |
"""generates a consistent value for 'attrname' if it's a float""" |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
158 |
return Decimal(str(self.generate_float(entity, attrname, index))) |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
159 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
160 |
def generate_datetime(self, entity, attrname, index): |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
161 |
"""generates a random date (format is 'yyyy-mm-dd HH:MM')""" |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
162 |
base = datetime(randint(2000, 2004), randint(1, 12), randint(1, 28), 11, index%60) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
163 |
return self._constrained_generate(entity, attrname, base, timedelta(hours=1), index) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
164 |
|
7166
dde161937d3e
[time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7152
diff
changeset
|
165 |
generate_tzdatetime = generate_datetime # XXX implementation should add a timezone |
dde161937d3e
[time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7152
diff
changeset
|
166 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
167 |
def generate_date(self, entity, attrname, index): |
0 | 168 |
"""generates a random date (format is 'yyyy-mm-dd')""" |
4651
f9cd35dece09
[devtools/fill] fix date generation and its test
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
4513
diff
changeset
|
169 |
base = date(randint(2000, 2010), 1, 1) + timedelta(randint(1, 365)) |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
170 |
return self._constrained_generate(entity, attrname, base, timedelta(days=1), index) |
0 | 171 |
|
6884
6fa712e9dfa5
[testlib] missing method on value generator to generate the Interval type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
172 |
def generate_interval(self, entity, attrname, index): |
6fa712e9dfa5
[testlib] missing method on value generator to generate the Interval type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
173 |
"""generates a random date (format is 'yyyy-mm-dd')""" |
6fa712e9dfa5
[testlib] missing method on value generator to generate the Interval type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
174 |
base = timedelta(randint(1, 365)) |
6fa712e9dfa5
[testlib] missing method on value generator to generate the Interval type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
175 |
return self._constrained_generate(entity, attrname, base, timedelta(days=1), index) |
6fa712e9dfa5
[testlib] missing method on value generator to generate the Interval type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
176 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
177 |
def generate_time(self, entity, attrname, index): |
0 | 178 |
"""generates a random time (format is ' HH:MM')""" |
2118
0012d997b992
should use time here, not timedelta
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
179 |
return time(11, index%60) #'11:%02d' % (index % 60) |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
180 |
|
7166
dde161937d3e
[time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7152
diff
changeset
|
181 |
generate_tztime = generate_time # XXX implementation should add a timezone |
dde161937d3e
[time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7152
diff
changeset
|
182 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
183 |
def generate_bytes(self, entity, attrname, index, format=None): |
10704
73367a56ee41
[py3k] ensure Binary objects are initialized with bytes
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10702
diff
changeset
|
184 |
fakefile = Binary(("%s%s" % (attrname, index)).encode('ascii')) |
3541
63ba0a669c4f
unicode file name
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
185 |
fakefile.filename = u"file_%s" % attrname |
0 | 186 |
return fakefile |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
187 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
188 |
def generate_boolean(self, entity, attrname, index): |
0 | 189 |
"""generates a consistent value for 'attrname' if it's a boolean""" |
190 |
return index % 2 == 0 |
|
191 |
||
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
192 |
def _constrained_generate(self, entity, attrname, base, step, index): |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
193 |
choosed = self.get_choice(entity, attrname) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
194 |
if choosed is not None: |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
195 |
return choosed |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
196 |
# ensure index > 0 |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
197 |
index += 1 |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
198 |
minvalue, maxvalue = self.get_bounds(entity, attrname) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
199 |
if maxvalue is None: |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
200 |
if minvalue is not None: |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
201 |
base = max(minvalue, base) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
202 |
maxvalue = base + index * step |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
203 |
if minvalue is None: |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
204 |
minvalue = maxvalue - (index * step) # i.e. randint(-index, 0) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
205 |
return choice(list(custom_range(minvalue, maxvalue, step))) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
206 |
|
6916
3970185a8f8c
[devtools] fix understanding of boundary constraint on db automatic population
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
207 |
def _actual_boundary(self, entity, attrname, boundary): |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
208 |
if isinstance(boundary, Attribute): |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
209 |
# ensure we've a value for this attribute |
6916
3970185a8f8c
[devtools] fix understanding of boundary constraint on db automatic population
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
210 |
entity[attrname] = None # infinite loop safety belt |
3970185a8f8c
[devtools] fix understanding of boundary constraint on db automatic population
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
211 |
if not boundary.attr in entity: |
3970185a8f8c
[devtools] fix understanding of boundary constraint on db automatic population
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
212 |
self.generate_attribute_value(entity, boundary.attr) |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
213 |
boundary = actual_value(boundary, entity) |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
214 |
return boundary |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
215 |
|
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
216 |
def get_bounds(self, entity, attrname): |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
217 |
minvalue = maxvalue = None |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
218 |
for cst in self.eschema.rdef(attrname).constraints: |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
219 |
if isinstance(cst, IntervalBoundConstraint): |
11080
245f440b814a
[devtools] fix get_bounds
Julien Cristau <julien.cristau@logilab.fr>
parents:
11079
diff
changeset
|
220 |
if cst.minvalue is not None: |
245f440b814a
[devtools] fix get_bounds
Julien Cristau <julien.cristau@logilab.fr>
parents:
11079
diff
changeset
|
221 |
minvalue = self._actual_boundary(entity, attrname, cst.minvalue) |
245f440b814a
[devtools] fix get_bounds
Julien Cristau <julien.cristau@logilab.fr>
parents:
11079
diff
changeset
|
222 |
if cst.maxvalue is not None: |
245f440b814a
[devtools] fix get_bounds
Julien Cristau <julien.cristau@logilab.fr>
parents:
11079
diff
changeset
|
223 |
maxvalue = self._actual_boundary(entity, attrname, cst.maxvalue) |
6916
3970185a8f8c
[devtools] fix understanding of boundary constraint on db automatic population
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
224 |
elif isinstance(cst, BoundaryConstraint): |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
225 |
if cst.operator[0] == '<': |
6916
3970185a8f8c
[devtools] fix understanding of boundary constraint on db automatic population
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
226 |
maxvalue = self._actual_boundary(entity, attrname, cst.boundary) |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
227 |
else: |
6916
3970185a8f8c
[devtools] fix understanding of boundary constraint on db automatic population
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5811
diff
changeset
|
228 |
minvalue = self._actual_boundary(entity, attrname, cst.boundary) |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
229 |
return minvalue, maxvalue |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
230 |
|
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
231 |
def get_choice(self, entity, attrname): |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
232 |
"""generates a consistent value for 'attrname' if it has some static |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
233 |
vocabulary set, else return None. |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
234 |
""" |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
235 |
for cst in self.eschema.rdef(attrname).constraints: |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
236 |
if isinstance(cst, StaticVocabularyConstraint): |
10702
f94c812c3669
[py3k] unicode → six.text_type
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10688
diff
changeset
|
237 |
return text_type(choice(cst.vocabulary())) |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
238 |
return None |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
239 |
|
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
240 |
# XXX nothing to do here |
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
241 |
def generate_Any_data_format(self, entity, index, **kwargs): |
5575
8a531340c3ef
no more Image type w/ cw 3.9
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
242 |
# data_format attribute of File has no vocabulary constraint, we |
0 | 243 |
# need this method else stupid values will be set which make mtconverter |
244 |
# raise exception |
|
245 |
return u'application/octet-stream' |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
246 |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
247 |
def generate_Any_content_format(self, entity, index, **kwargs): |
0 | 248 |
# content_format attribute of EmailPart has no vocabulary constraint, we |
249 |
# need this method else stupid values will be set which make mtconverter |
|
250 |
# raise exception |
|
251 |
return u'text/plain' |
|
252 |
||
7995
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7815
diff
changeset
|
253 |
def generate_CWDataImport_log(self, entity, index, **kwargs): |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7815
diff
changeset
|
254 |
# content_format attribute of EmailPart has no vocabulary constraint, we |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7815
diff
changeset
|
255 |
# need this method else stupid values will be set which make mtconverter |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7815
diff
changeset
|
256 |
# raise exception |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7815
diff
changeset
|
257 |
logs = [u'%s\t%s\t%s\t%s<br/>' % (logging.ERROR, 'http://url.com?arg1=hop&arg2=hip', |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7815
diff
changeset
|
258 |
1, xml_escape('hjoio&oio"'))] |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7815
diff
changeset
|
259 |
return u'<br/>'.join(logs) |
9a9f35ef418c
Record a log of datafeed source imports (closes #2026097)
Julien Cristau <julien.cristau@logilab.fr>
parents:
7815
diff
changeset
|
260 |
|
0 | 261 |
|
262 |
class autoextend(type): |
|
263 |
def __new__(mcs, name, bases, classdict): |
|
264 |
for attrname, attrvalue in classdict.items(): |
|
265 |
if callable(attrvalue): |
|
266 |
if attrname.startswith('generate_') and \ |
|
10714
8a2ec43fcf44
[py3k] use inspect module to get a function's arguments
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10704
diff
changeset
|
267 |
len(inspect.getargspec(attrvalue).args) < 2: |
0 | 268 |
raise TypeError('generate_xxx must accept at least 1 argument') |
269 |
setattr(_ValueGenerator, attrname, attrvalue) |
|
270 |
return type.__new__(mcs, name, bases, classdict) |
|
271 |
||
10688
fa29f3628a1b
[py3k] use six.add_metaclass
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10609
diff
changeset
|
272 |
|
fa29f3628a1b
[py3k] use six.add_metaclass
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10609
diff
changeset
|
273 |
@add_metaclass(autoextend) |
0 | 274 |
class ValueGenerator(_ValueGenerator): |
10688
fa29f3628a1b
[py3k] use six.add_metaclass
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10609
diff
changeset
|
275 |
pass |
0 | 276 |
|
277 |
||
278 |
def _default_choice_func(etype, attrname): |
|
279 |
"""default choice_func for insert_entity_queries""" |
|
280 |
return None |
|
281 |
||
282 |
def insert_entity_queries(etype, schema, vreg, entity_num, |
|
283 |
choice_func=_default_choice_func): |
|
284 |
"""returns a list of 'add entity' queries (couples query, args) |
|
285 |
:type etype: str |
|
286 |
:param etype: the entity's type |
|
287 |
||
288 |
:type schema: cubicweb.schema.Schema |
|
2476
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2118
diff
changeset
|
289 |
:param schema: the instance schema |
0 | 290 |
|
291 |
:type entity_num: int |
|
292 |
:param entity_num: the number of entities to insert |
|
293 |
||
294 |
XXX FIXME: choice_func is here for *historical* reasons, it should |
|
295 |
probably replaced by a nicer way to specify choices |
|
296 |
:type choice_func: function |
|
297 |
:param choice_func: a function that takes an entity type, an attrname and |
|
298 |
returns acceptable values for this attribute |
|
299 |
""" |
|
300 |
queries = [] |
|
10609
e2d8e81bfe68
[py3k] import range using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
301 |
for index in range(entity_num): |
0 | 302 |
restrictions = [] |
303 |
args = {} |
|
304 |
for attrname, value in make_entity(etype, schema, vreg, index, choice_func).items(): |
|
305 |
restrictions.append('X %s %%(%s)s' % (attrname, attrname)) |
|
306 |
args[attrname] = value |
|
307 |
if restrictions: |
|
308 |
queries.append(('INSERT %s X: %s' % (etype, ', '.join(restrictions)), |
|
309 |
args)) |
|
310 |
assert not 'eid' in args, args |
|
311 |
else: |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
312 |
queries.append(('INSERT %s X' % etype, {})) |
0 | 313 |
return queries |
314 |
||
315 |
||
316 |
def make_entity(etype, schema, vreg, index=0, choice_func=_default_choice_func, |
|
317 |
form=False): |
|
318 |
"""generates a random entity and returns it as a dict |
|
319 |
||
320 |
by default, generate an entity to be inserted in the repository |
|
8238
087bb529035c
[spelling] fix dictionnary -> dictionary typo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7995
diff
changeset
|
321 |
elif form, generate an form dictionary to be given to a web controller |
0 | 322 |
""" |
323 |
eschema = schema.eschema(etype) |
|
324 |
valgen = ValueGenerator(eschema, choice_func) |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
325 |
entity = attrdict() |
0 | 326 |
# preprocessing to deal with _format fields |
327 |
attributes = [] |
|
328 |
relatedfields = {} |
|
329 |
for rschema, attrschema in eschema.attribute_definitions(): |
|
330 |
attrname = rschema.type |
|
331 |
if attrname == 'eid': |
|
332 |
# don't specify eids ! |
|
333 |
continue |
|
334 |
if attrname.endswith('_format') and attrname[:-7] in eschema.subject_relations(): |
|
335 |
relatedfields[attrname[:-7]] = attrschema |
|
336 |
else: |
|
337 |
attributes.append((attrname, attrschema)) |
|
338 |
for attrname, attrschema in attributes: |
|
339 |
if attrname in relatedfields: |
|
340 |
# first generate a format and record it |
|
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
341 |
format = valgen.generate_attribute_value(entity, attrname + '_format', index) |
0 | 342 |
# then a value coherent with this format |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
343 |
value = valgen.generate_attribute_value(entity, attrname, index, format=format) |
0 | 344 |
else: |
4337
27ea69e2cfea
refactor attributes'value generator to deal with yams special
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
345 |
value = valgen.generate_attribute_value(entity, attrname, index) |
0 | 346 |
if form: # need to encode values |
347 |
if attrschema.type == 'Bytes': |
|
348 |
# twisted way |
|
349 |
fakefile = value |
|
350 |
filename = value.filename |
|
351 |
value = (filename, u"text/plain", fakefile) |
|
352 |
elif attrschema.type == 'Date': |
|
353 |
value = value.strftime(vreg.property_value('ui.date-format')) |
|
354 |
elif attrschema.type == 'Datetime': |
|
355 |
value = value.strftime(vreg.property_value('ui.datetime-format')) |
|
356 |
elif attrschema.type == 'Time': |
|
357 |
value = value.strftime(vreg.property_value('ui.time-format')) |
|
358 |
elif attrschema.type == 'Float': |
|
359 |
fmt = vreg.property_value('ui.float-format') |
|
360 |
value = fmt % value |
|
361 |
else: |
|
10702
f94c812c3669
[py3k] unicode → six.text_type
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10688
diff
changeset
|
362 |
value = text_type(value) |
0 | 363 |
return entity |
364 |
||
365 |
||
366 |
||
9837
64c8ee99baf7
[devtools/testlib,fill] use the new connection api (for auto_populate)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9521
diff
changeset
|
367 |
def select(constraints, cnx, selectvar='O', objtype=None): |
0 | 368 |
"""returns list of eids matching <constraints> |
369 |
||
370 |
<selectvar> should be either 'O' or 'S' to match schema definitions |
|
371 |
""" |
|
372 |
try: |
|
3742
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
373 |
rql = 'Any %s WHERE %s' % (selectvar, constraints) |
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
374 |
if objtype: |
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
375 |
rql += ', %s is %s' % (selectvar, objtype) |
9837
64c8ee99baf7
[devtools/testlib,fill] use the new connection api (for auto_populate)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9521
diff
changeset
|
376 |
rset = cnx.execute(rql) |
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7445
diff
changeset
|
377 |
except Exception: |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
9837
diff
changeset
|
378 |
print("could restrict eid_list with given constraints (%r)" % constraints) |
0 | 379 |
return [] |
380 |
return set(eid for eid, in rset.rows) |
|
381 |
||
382 |
||
383 |
||
9837
64c8ee99baf7
[devtools/testlib,fill] use the new connection api (for auto_populate)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9521
diff
changeset
|
384 |
def make_relations_queries(schema, edict, cnx, ignored_relations=(), |
0 | 385 |
existingrels=None): |
386 |
"""returns a list of generated RQL queries for relations |
|
2476
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2118
diff
changeset
|
387 |
:param schema: The instance schema |
0 | 388 |
|
389 |
:param e_dict: mapping between etypes and eids |
|
390 |
||
391 |
:param ignored_relations: list of relations to ignore (i.e. don't try |
|
392 |
to generate insert queries for these relations) |
|
393 |
""" |
|
9837
64c8ee99baf7
[devtools/testlib,fill] use the new connection api (for auto_populate)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9521
diff
changeset
|
394 |
gen = RelationsQueriesGenerator(schema, cnx, existingrels) |
0 | 395 |
return gen.compute_queries(edict, ignored_relations) |
396 |
||
3742
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
397 |
def composite_relation(rschema): |
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
398 |
for obj in rschema.objects(): |
9521
9eb810333b0f
[hooks/security, devtools/fill] silence yams 0.38.0 warnings
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9207
diff
changeset
|
399 |
if obj.rdef(rschema, 'object', takefirst=True).composite == 'subject': |
3742
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
400 |
return True |
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
401 |
for obj in rschema.subjects(): |
9521
9eb810333b0f
[hooks/security, devtools/fill] silence yams 0.38.0 warnings
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9207
diff
changeset
|
402 |
if obj.rdef(rschema, 'subject', takefirst=True).composite == 'object': |
3742
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
403 |
return True |
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
404 |
return False |
0 | 405 |
|
406 |
class RelationsQueriesGenerator(object): |
|
407 |
rql_tmpl = 'SET S %s O WHERE S eid %%(subjeid)s, O eid %%(objeid)s' |
|
9837
64c8ee99baf7
[devtools/testlib,fill] use the new connection api (for auto_populate)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9521
diff
changeset
|
408 |
def __init__(self, schema, cnx, existing=None): |
0 | 409 |
self.schema = schema |
9837
64c8ee99baf7
[devtools/testlib,fill] use the new connection api (for auto_populate)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9521
diff
changeset
|
410 |
self.cnx = cnx |
0 | 411 |
self.existingrels = existing or {} |
412 |
||
413 |
def compute_queries(self, edict, ignored_relations): |
|
414 |
queries = [] |
|
415 |
# 1/ skip final relations and explictly ignored relations |
|
3742
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
416 |
rels = sorted([rschema for rschema in self.schema.relations() |
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
417 |
if not (rschema.final or rschema in ignored_relations)], |
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
418 |
key=lambda x:not composite_relation(x)) |
0 | 419 |
# for each relation |
420 |
# 2/ take each possible couple (subj, obj) |
|
421 |
# 3/ analyze cardinality of relation |
|
422 |
# a/ if relation is mandatory, insert one relation |
|
423 |
# b/ else insert N relations where N is the mininum |
|
424 |
# of 20 and the number of existing targetable entities |
|
425 |
for rschema in rels: |
|
426 |
sym = set() |
|
427 |
sedict = deepcopy(edict) |
|
428 |
oedict = deepcopy(edict) |
|
429 |
delayed = [] |
|
430 |
# for each couple (subjschema, objschema), insert relations |
|
4053
7cc66b1d9183
more api update
Sandrine Ribeau <sandrine.ribeau@logilab.fr>
parents:
3749
diff
changeset
|
431 |
for subj, obj in rschema.rdefs: |
0 | 432 |
sym.add( (subj, obj) ) |
4467
0e73d299730a
fix long-waiting symetric typo: should be spelled symmetric. Add auto database migration on schema deserialization
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4337
diff
changeset
|
433 |
if rschema.symmetric and (obj, subj) in sym: |
0 | 434 |
continue |
4053
7cc66b1d9183
more api update
Sandrine Ribeau <sandrine.ribeau@logilab.fr>
parents:
3749
diff
changeset
|
435 |
subjcard, objcard = rschema.rdef(subj, obj).cardinality |
0 | 436 |
# process mandatory relations first |
3742
20f429eb5f46
kill separate attribute client-side handling #473636
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3712
diff
changeset
|
437 |
if subjcard in '1+' or objcard in '1+' or composite_relation(rschema): |
3749
a84e798b99f6
turn make_relations_query into a true generator (this allows interleaving of actual entities linking and query generation)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3747
diff
changeset
|
438 |
for query, args in self.make_relation_queries(sedict, oedict, |
a84e798b99f6
turn make_relations_query into a true generator (this allows interleaving of actual entities linking and query generation)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3747
diff
changeset
|
439 |
rschema, subj, obj): |
a84e798b99f6
turn make_relations_query into a true generator (this allows interleaving of actual entities linking and query generation)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3747
diff
changeset
|
440 |
yield query, args |
0 | 441 |
else: |
442 |
delayed.append( (subj, obj) ) |
|
443 |
for subj, obj in delayed: |
|
3749
a84e798b99f6
turn make_relations_query into a true generator (this allows interleaving of actual entities linking and query generation)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3747
diff
changeset
|
444 |
for query, args in self.make_relation_queries(sedict, oedict, rschema, |
a84e798b99f6
turn make_relations_query into a true generator (this allows interleaving of actual entities linking and query generation)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3747
diff
changeset
|
445 |
subj, obj): |
a84e798b99f6
turn make_relations_query into a true generator (this allows interleaving of actual entities linking and query generation)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3747
diff
changeset
|
446 |
yield query, args |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
447 |
|
0 | 448 |
def qargs(self, subjeids, objeids, subjcard, objcard, subjeid, objeid): |
4513
8abf464d2ffe
fix and begin to document autofill algorithm
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4467
diff
changeset
|
449 |
if subjcard in '?1+': |
0 | 450 |
subjeids.remove(subjeid) |
4513
8abf464d2ffe
fix and begin to document autofill algorithm
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4467
diff
changeset
|
451 |
if objcard in '?1+': |
0 | 452 |
objeids.remove(objeid) |
453 |
return {'subjeid' : subjeid, 'objeid' : objeid} |
|
454 |
||
455 |
def make_relation_queries(self, sedict, oedict, rschema, subj, obj): |
|
4053
7cc66b1d9183
more api update
Sandrine Ribeau <sandrine.ribeau@logilab.fr>
parents:
3749
diff
changeset
|
456 |
rdef = rschema.rdef(subj, obj) |
7cc66b1d9183
more api update
Sandrine Ribeau <sandrine.ribeau@logilab.fr>
parents:
3749
diff
changeset
|
457 |
subjcard, objcard = rdef.cardinality |
0 | 458 |
subjeids = sedict.get(subj, frozenset()) |
459 |
used = self.existingrels[rschema.type] |
|
460 |
preexisting_subjrels = set(subj for subj, obj in used) |
|
461 |
preexisting_objrels = set(obj for subj, obj in used) |
|
462 |
# if there are constraints, only select appropriate objeids |
|
463 |
q = self.rql_tmpl % rschema.type |
|
4053
7cc66b1d9183
more api update
Sandrine Ribeau <sandrine.ribeau@logilab.fr>
parents:
3749
diff
changeset
|
464 |
constraints = [c for c in rdef.constraints |
0 | 465 |
if isinstance(c, RQLConstraint)] |
466 |
if constraints: |
|
7152
39c1ffc7d93f
[schema, refactoring] use RQLExpression as base class for RQL constraint classes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6931
diff
changeset
|
467 |
restrictions = ', '.join(c.expression for c in constraints) |
0 | 468 |
q += ', %s' % restrictions |
469 |
# restrict object eids if possible |
|
3711
486a29d66f2a
work around a problem in constraint application
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
3689
diff
changeset
|
470 |
# XXX the attempt to restrict below in completely wrong |
486a29d66f2a
work around a problem in constraint application
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
3689
diff
changeset
|
471 |
# disabling it for now |
9837
64c8ee99baf7
[devtools/testlib,fill] use the new connection api (for auto_populate)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9521
diff
changeset
|
472 |
objeids = select(restrictions, self.cnx, objtype=obj) |
3712
4b8d7838d74d
revert change, but there's something fishy here
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
3711
diff
changeset
|
473 |
else: |
4b8d7838d74d
revert change, but there's something fishy here
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
3711
diff
changeset
|
474 |
objeids = oedict.get(obj, frozenset()) |
0 | 475 |
if subjcard in '?1' or objcard in '?1': |
476 |
for subjeid, objeid in used: |
|
477 |
if subjcard in '?1' and subjeid in subjeids: |
|
478 |
subjeids.remove(subjeid) |
|
3673
9342e6783bd2
[testlib] hummm
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3541
diff
changeset
|
479 |
# XXX why? |
9342e6783bd2
[testlib] hummm
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3541
diff
changeset
|
480 |
#if objeid in objeids: |
9342e6783bd2
[testlib] hummm
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3541
diff
changeset
|
481 |
# objeids.remove(objeid) |
0 | 482 |
if objcard in '?1' and objeid in objeids: |
483 |
objeids.remove(objeid) |
|
3673
9342e6783bd2
[testlib] hummm
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3541
diff
changeset
|
484 |
# XXX why? |
9342e6783bd2
[testlib] hummm
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3541
diff
changeset
|
485 |
#if subjeid in subjeids: |
9342e6783bd2
[testlib] hummm
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3541
diff
changeset
|
486 |
# subjeids.remove(subjeid) |
0 | 487 |
if not subjeids: |
488 |
check_card_satisfied(objcard, objeids, subj, rschema, obj) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
489 |
return |
0 | 490 |
if not objeids: |
491 |
check_card_satisfied(subjcard, subjeids, subj, rschema, obj) |
|
492 |
return |
|
493 |
if subjcard in '?1+': |
|
494 |
for subjeid in tuple(subjeids): |
|
495 |
# do not insert relation if this entity already has a relation |
|
496 |
if subjeid in preexisting_subjrels: |
|
497 |
continue |
|
498 |
objeid = choose_eid(objeids, subjeid) |
|
499 |
if objeid is None or (subjeid, objeid) in used: |
|
500 |
continue |
|
501 |
yield q, self.qargs(subjeids, objeids, subjcard, objcard, |
|
502 |
subjeid, objeid) |
|
503 |
used.add( (subjeid, objeid) ) |
|
504 |
if not objeids: |
|
505 |
check_card_satisfied(subjcard, subjeids, subj, rschema, obj) |
|
506 |
break |
|
507 |
elif objcard in '?1+': |
|
508 |
for objeid in tuple(objeids): |
|
509 |
# do not insert relation if this entity already has a relation |
|
510 |
if objeid in preexisting_objrels: |
|
511 |
continue |
|
512 |
subjeid = choose_eid(subjeids, objeid) |
|
513 |
if subjeid is None or (subjeid, objeid) in used: |
|
514 |
continue |
|
515 |
yield q, self.qargs(subjeids, objeids, subjcard, objcard, |
|
516 |
subjeid, objeid) |
|
517 |
used.add( (subjeid, objeid) ) |
|
518 |
if not subjeids: |
|
519 |
check_card_satisfied(objcard, objeids, subj, rschema, obj) |
|
520 |
break |
|
521 |
else: |
|
522 |
# FIXME: 20 should be read from config |
|
10609
e2d8e81bfe68
[py3k] import range using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
523 |
subjeidsiter = [choice(tuple(subjeids)) for i in range(min(len(subjeids), 20))] |
e2d8e81bfe68
[py3k] import range using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
524 |
objeidsiter = [choice(tuple(objeids)) for i in range(min(len(objeids), 20))] |
0 | 525 |
for subjeid, objeid in zip(subjeidsiter, objeidsiter): |
526 |
if subjeid != objeid and not (subjeid, objeid) in used: |
|
527 |
used.add( (subjeid, objeid) ) |
|
528 |
yield q, self.qargs(subjeids, objeids, subjcard, objcard, |
|
529 |
subjeid, objeid) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
530 |
|
0 | 531 |
def check_card_satisfied(card, remaining, subj, rschema, obj): |
532 |
if card in '1+' and remaining: |
|
3749
a84e798b99f6
turn make_relations_query into a true generator (this allows interleaving of actual entities linking and query generation)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3747
diff
changeset
|
533 |
raise Exception("can't satisfy cardinality %s for relation %s %s %s" % |
a84e798b99f6
turn make_relations_query into a true generator (this allows interleaving of actual entities linking and query generation)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3747
diff
changeset
|
534 |
(card, subj, rschema, obj)) |
a84e798b99f6
turn make_relations_query into a true generator (this allows interleaving of actual entities linking and query generation)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
3747
diff
changeset
|
535 |
|
0 | 536 |
|
537 |
def choose_eid(values, avoid): |
|
538 |
values = tuple(values) |
|
539 |
if len(values) == 1 and values[0] == avoid: |
|
540 |
return None |
|
541 |
objeid = choice(values) |
|
542 |
while objeid == avoid: # avoid infinite recursion like in X comment X |
|
543 |
objeid = choice(values) |
|
544 |
return objeid |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
545 |
|
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
546 |
|
0 | 547 |
|
548 |
# UTILITIES FUNCS ############################################################## |
|
549 |
def make_tel(num_tel): |
|
550 |
"""takes an integer, converts is as a string and inserts |
|
551 |
white spaces each 2 chars (french notation) |
|
552 |
""" |
|
553 |
num_list = list(str(num_tel)) |
|
554 |
for index in (6, 4, 2): |
|
555 |
num_list.insert(index, ' ') |
|
556 |
||
557 |
return ''.join(num_list) |
|
558 |
||
559 |
||
560 |
def numlen(number): |
|
561 |
"""returns the number's length""" |
|
562 |
return len(str(number)) |