author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Tue, 16 Feb 2010 09:23:35 +0100 | |
changeset 4591 | 47acae3cb778 |
parent 4475 | 37c413a07216 |
child 5421 | 8167de96c523 |
permissions | -rw-r--r-- |
2476
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
1 |
""" Usage: %s [OPTIONS] <instance id> <queries file> |
0 | 2 |
|
3 |
Stress test a CubicWeb repository |
|
4 |
||
5 |
OPTIONS: |
|
6 |
-h / --help |
|
7 |
Display this help message and exit. |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
8 |
|
0 | 9 |
-u / --user <user> |
10 |
Connect as <user> instead of being prompted to give it. |
|
11 |
-p / --password <password> |
|
12 |
Automatically give <password> for authentication instead of being prompted |
|
13 |
to give it. |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
14 |
|
0 | 15 |
-n / --nb-times <num> |
16 |
Repeat queries <num> times. |
|
17 |
-t / --nb-threads <num> |
|
18 |
Execute queries in <num> parallel threads. |
|
19 |
-P / --profile <prof_file> |
|
20 |
dumps profile results (hotshot) in <prof_file> |
|
21 |
-o / --report-output <filename> |
|
22 |
Write profiler report into <filename> rather than on stdout |
|
23 |
||
4212
ab6573088b4a
update copyright: welcome 2010
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
2476
diff
changeset
|
24 |
Copyright (c) 2003-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. |
0 | 25 |
http://www.logilab.fr/ -- mailto:contact@logilab.fr |
1977
606923dff11b
big bunch of copyright / docstring update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
26 |
:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses |
0 | 27 |
""" |
28 |
||
29 |
import os |
|
30 |
import sys |
|
31 |
import threading |
|
32 |
import getopt |
|
33 |
import traceback |
|
34 |
from getpass import getpass |
|
35 |
from os.path import basename |
|
36 |
from time import clock |
|
37 |
||
38 |
from logilab.common.fileutils import lines |
|
39 |
from logilab.common.ureports import Table, TextWriter |
|
40 |
from cubicweb.server.repository import Repository |
|
41 |
from cubicweb.dbapi import Connection |
|
42 |
||
43 |
TB_LOCK = threading.Lock() |
|
44 |
||
45 |
class QueryExecutor: |
|
46 |
def __init__(self, cursor, times, queries, reporter = None): |
|
47 |
self._cursor = cursor |
|
48 |
self._times = times |
|
49 |
self._queries = queries |
|
50 |
self._reporter = reporter |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
51 |
|
0 | 52 |
def run(self): |
53 |
cursor = self._cursor |
|
54 |
times = self._times |
|
55 |
while times: |
|
56 |
for index, query in enumerate(self._queries): |
|
57 |
start = clock() |
|
58 |
try: |
|
59 |
cursor.execute(query) |
|
60 |
except KeyboardInterrupt: |
|
61 |
raise |
|
62 |
except: |
|
63 |
TB_LOCK.acquire() |
|
64 |
traceback.print_exc() |
|
65 |
TB_LOCK.release() |
|
66 |
return |
|
67 |
if self._reporter is not None: |
|
68 |
self._reporter.add_proftime(clock() - start, index) |
|
69 |
times -= 1 |
|
70 |
||
71 |
def usage(status=0): |
|
72 |
"""print usage string and exit""" |
|
73 |
print __doc__ % basename(sys.argv[0]) |
|
74 |
sys.exit(status) |
|
75 |
||
76 |
||
77 |
class ProfileReporter: |
|
78 |
"""a profile reporter gathers all profile informations from several |
|
79 |
threads and can write a report that summarizes all profile informations |
|
80 |
""" |
|
81 |
profiler_lock = threading.Lock() |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
82 |
|
0 | 83 |
def __init__(self, queries): |
84 |
self._queries = tuple(queries) |
|
85 |
self._profile_results = [(0., 0)] * len(self._queries) |
|
86 |
# self._table_report = Table(3, rheaders = True) |
|
87 |
len_max = max([len(query) for query in self._queries]) + 5 |
|
88 |
self._query_fmt = '%%%ds' % len_max |
|
89 |
||
90 |
def add_proftime(self, elapsed_time, query_index): |
|
91 |
"""add a new time measure for query""" |
|
92 |
ProfileReporter.profiler_lock.acquire() |
|
93 |
cumul_time, times = self._profile_results[query_index] |
|
94 |
cumul_time += elapsed_time |
|
95 |
times += 1. |
|
96 |
self._profile_results[query_index] = (cumul_time, times) |
|
97 |
ProfileReporter.profiler_lock.release() |
|
98 |
||
99 |
def dump_report(self, output = sys.stdout): |
|
100 |
"""dump report in 'output'""" |
|
101 |
table_elems = ['RQL Query', 'Times', 'Avg Time'] |
|
102 |
total_time = 0. |
|
103 |
for query, (cumul_time, times) in zip(self._queries, self._profile_results): |
|
104 |
avg_time = cumul_time / float(times) |
|
105 |
table_elems += [str(query), '%f' % times, '%f' % avg_time ] |
|
106 |
total_time += cumul_time |
|
107 |
table_elems.append('Total time :') |
|
108 |
table_elems.append(str(total_time)) |
|
109 |
table_elems.append(' ') |
|
110 |
table_layout = Table(3, rheaders = True, children = table_elems) |
|
111 |
TextWriter().format(table_layout, output) |
|
112 |
# output.write('\n'.join(tmp_output)) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
113 |
|
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
114 |
|
0 | 115 |
def run(args): |
116 |
"""run the command line tool""" |
|
117 |
try: |
|
118 |
opts, args = getopt.getopt(args, 'hn:t:u:p:P:o:', ['help', 'user=', 'password=', |
|
119 |
'nb-times=', 'nb-threads=', |
|
120 |
'profile', 'report-output=',]) |
|
121 |
except Exception, ex: |
|
122 |
print ex |
|
123 |
usage(1) |
|
124 |
repeat = 100 |
|
125 |
threads = 1 |
|
126 |
user = os.environ.get('USER', os.environ.get('LOGNAME')) |
|
127 |
password = None |
|
128 |
report_output = sys.stdout |
|
129 |
prof_file = None |
|
130 |
for opt, val in opts: |
|
131 |
if opt in ('-h', '--help'): |
|
132 |
usage() |
|
133 |
if opt in ('-u', '--user'): |
|
134 |
user = val |
|
135 |
elif opt in ('-p', '--password'): |
|
136 |
password = val |
|
137 |
elif opt in ('-n', '--nb-times'): |
|
138 |
repeat = int(val) |
|
139 |
elif opt in ('-t', '--nb-threads'): |
|
140 |
threads = int(val) |
|
141 |
elif opt in ('-P', '--profile'): |
|
142 |
prof_file = val |
|
143 |
elif opt in ('-o', '--report-output'): |
|
144 |
report_output = file(val, 'w') |
|
145 |
if len(args) != 2: |
|
146 |
usage(1) |
|
147 |
queries = [query for query in lines(args[1]) if not query.startswith('#')] |
|
148 |
if user is None: |
|
149 |
user = raw_input('login: ') |
|
150 |
if password is None: |
|
151 |
password = getpass('password: ') |
|
2476
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
152 |
from cubicweb.cwconfig import instance_configuration |
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
153 |
config = instance_configuration(args[0]) |
0 | 154 |
# get local access to the repository |
155 |
print "Creating repo", prof_file |
|
156 |
repo = Repository(config, prof_file) |
|
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
157 |
cnxid = repo.connect(user, password=password) |
0 | 158 |
# connection to the CubicWeb repository |
159 |
repo_cnx = Connection(repo, cnxid) |
|
160 |
repo_cursor = repo_cnx.cursor() |
|
161 |
reporter = ProfileReporter(queries) |
|
162 |
if threads > 1: |
|
163 |
executors = [] |
|
164 |
while threads: |
|
165 |
qe = QueryExecutor(repo_cursor, repeat, queries, reporter = reporter) |
|
166 |
executors.append(qe) |
|
167 |
thread = threading.Thread(target=qe.run) |
|
168 |
qe.thread = thread |
|
169 |
thread.start() |
|
170 |
threads -= 1 |
|
171 |
for qe in executors: |
|
172 |
qe.thread.join() |
|
173 |
## for qe in executors: |
|
174 |
## print qe.thread, repeat - qe._times, 'times' |
|
175 |
else: |
|
176 |
QueryExecutor(repo_cursor, repeat, queries, reporter = reporter).run() |
|
177 |
reporter.dump_report(report_output) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
178 |
|
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
179 |
|
0 | 180 |
if __name__ == '__main__': |
181 |
run(sys.argv[1:]) |