author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Thu, 03 Apr 2014 19:55:11 +0200 | |
changeset 9656 | 19c423d9d215 |
parent 8695 | 358d8bed9626 |
child 10589 | 7c23b7de2b8d |
permissions | -rw-r--r-- |
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
1 |
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
3 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
4 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
5 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
9 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
10 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
14 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
15 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4475
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
2476
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
18 |
""" Usage: %s [OPTIONS] <instance id> <queries file> |
0 | 19 |
|
20 |
Stress test a CubicWeb repository |
|
21 |
||
22 |
OPTIONS: |
|
23 |
-h / --help |
|
24 |
Display this help message and exit. |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
25 |
|
0 | 26 |
-u / --user <user> |
27 |
Connect as <user> instead of being prompted to give it. |
|
28 |
-p / --password <password> |
|
29 |
Automatically give <password> for authentication instead of being prompted |
|
30 |
to give it. |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
31 |
|
0 | 32 |
-n / --nb-times <num> |
33 |
Repeat queries <num> times. |
|
34 |
-t / --nb-threads <num> |
|
35 |
Execute queries in <num> parallel threads. |
|
36 |
-P / --profile <prof_file> |
|
37 |
dumps profile results (hotshot) in <prof_file> |
|
38 |
-o / --report-output <filename> |
|
39 |
Write profiler report into <filename> rather than on stdout |
|
40 |
||
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
41 |
Copyright (c) 2003-2011 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. |
0 | 42 |
http://www.logilab.fr/ -- mailto:contact@logilab.fr |
43 |
""" |
|
44 |
||
45 |
import os |
|
46 |
import sys |
|
47 |
import threading |
|
48 |
import getopt |
|
49 |
import traceback |
|
50 |
from getpass import getpass |
|
51 |
from os.path import basename |
|
52 |
from time import clock |
|
53 |
||
54 |
from logilab.common.fileutils import lines |
|
55 |
from logilab.common.ureports import Table, TextWriter |
|
56 |
from cubicweb.server.repository import Repository |
|
57 |
from cubicweb.dbapi import Connection |
|
58 |
||
59 |
TB_LOCK = threading.Lock() |
|
60 |
||
61 |
class QueryExecutor: |
|
62 |
def __init__(self, cursor, times, queries, reporter = None): |
|
63 |
self._cursor = cursor |
|
64 |
self._times = times |
|
65 |
self._queries = queries |
|
66 |
self._reporter = reporter |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
67 |
|
0 | 68 |
def run(self): |
69 |
cursor = self._cursor |
|
70 |
times = self._times |
|
71 |
while times: |
|
72 |
for index, query in enumerate(self._queries): |
|
73 |
start = clock() |
|
74 |
try: |
|
75 |
cursor.execute(query) |
|
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
76 |
except Exception: |
0 | 77 |
TB_LOCK.acquire() |
78 |
traceback.print_exc() |
|
79 |
TB_LOCK.release() |
|
80 |
return |
|
81 |
if self._reporter is not None: |
|
82 |
self._reporter.add_proftime(clock() - start, index) |
|
83 |
times -= 1 |
|
84 |
||
85 |
def usage(status=0): |
|
86 |
"""print usage string and exit""" |
|
87 |
print __doc__ % basename(sys.argv[0]) |
|
88 |
sys.exit(status) |
|
89 |
||
90 |
||
91 |
class ProfileReporter: |
|
92 |
"""a profile reporter gathers all profile informations from several |
|
93 |
threads and can write a report that summarizes all profile informations |
|
94 |
""" |
|
95 |
profiler_lock = threading.Lock() |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
96 |
|
0 | 97 |
def __init__(self, queries): |
98 |
self._queries = tuple(queries) |
|
99 |
self._profile_results = [(0., 0)] * len(self._queries) |
|
100 |
# self._table_report = Table(3, rheaders = True) |
|
101 |
len_max = max([len(query) for query in self._queries]) + 5 |
|
102 |
self._query_fmt = '%%%ds' % len_max |
|
103 |
||
104 |
def add_proftime(self, elapsed_time, query_index): |
|
105 |
"""add a new time measure for query""" |
|
106 |
ProfileReporter.profiler_lock.acquire() |
|
107 |
cumul_time, times = self._profile_results[query_index] |
|
108 |
cumul_time += elapsed_time |
|
109 |
times += 1. |
|
110 |
self._profile_results[query_index] = (cumul_time, times) |
|
111 |
ProfileReporter.profiler_lock.release() |
|
112 |
||
113 |
def dump_report(self, output = sys.stdout): |
|
114 |
"""dump report in 'output'""" |
|
115 |
table_elems = ['RQL Query', 'Times', 'Avg Time'] |
|
116 |
total_time = 0. |
|
117 |
for query, (cumul_time, times) in zip(self._queries, self._profile_results): |
|
118 |
avg_time = cumul_time / float(times) |
|
119 |
table_elems += [str(query), '%f' % times, '%f' % avg_time ] |
|
120 |
total_time += cumul_time |
|
121 |
table_elems.append('Total time :') |
|
122 |
table_elems.append(str(total_time)) |
|
123 |
table_elems.append(' ') |
|
124 |
table_layout = Table(3, rheaders = True, children = table_elems) |
|
125 |
TextWriter().format(table_layout, output) |
|
126 |
# output.write('\n'.join(tmp_output)) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
127 |
|
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
128 |
|
0 | 129 |
def run(args): |
130 |
"""run the command line tool""" |
|
131 |
try: |
|
132 |
opts, args = getopt.getopt(args, 'hn:t:u:p:P:o:', ['help', 'user=', 'password=', |
|
133 |
'nb-times=', 'nb-threads=', |
|
134 |
'profile', 'report-output=',]) |
|
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7815
diff
changeset
|
135 |
except Exception as ex: |
0 | 136 |
print ex |
137 |
usage(1) |
|
138 |
repeat = 100 |
|
139 |
threads = 1 |
|
140 |
user = os.environ.get('USER', os.environ.get('LOGNAME')) |
|
141 |
password = None |
|
142 |
report_output = sys.stdout |
|
143 |
prof_file = None |
|
144 |
for opt, val in opts: |
|
145 |
if opt in ('-h', '--help'): |
|
146 |
usage() |
|
147 |
if opt in ('-u', '--user'): |
|
148 |
user = val |
|
149 |
elif opt in ('-p', '--password'): |
|
150 |
password = val |
|
151 |
elif opt in ('-n', '--nb-times'): |
|
152 |
repeat = int(val) |
|
153 |
elif opt in ('-t', '--nb-threads'): |
|
154 |
threads = int(val) |
|
155 |
elif opt in ('-P', '--profile'): |
|
156 |
prof_file = val |
|
157 |
elif opt in ('-o', '--report-output'): |
|
158 |
report_output = file(val, 'w') |
|
159 |
if len(args) != 2: |
|
160 |
usage(1) |
|
161 |
queries = [query for query in lines(args[1]) if not query.startswith('#')] |
|
162 |
if user is None: |
|
163 |
user = raw_input('login: ') |
|
164 |
if password is None: |
|
165 |
password = getpass('password: ') |
|
2476
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
166 |
from cubicweb.cwconfig import instance_configuration |
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
167 |
config = instance_configuration(args[0]) |
0 | 168 |
# get local access to the repository |
169 |
print "Creating repo", prof_file |
|
170 |
repo = Repository(config, prof_file) |
|
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
171 |
cnxid = repo.connect(user, password=password) |
0 | 172 |
# connection to the CubicWeb repository |
173 |
repo_cnx = Connection(repo, cnxid) |
|
174 |
repo_cursor = repo_cnx.cursor() |
|
175 |
reporter = ProfileReporter(queries) |
|
176 |
if threads > 1: |
|
177 |
executors = [] |
|
178 |
while threads: |
|
179 |
qe = QueryExecutor(repo_cursor, repeat, queries, reporter = reporter) |
|
180 |
executors.append(qe) |
|
181 |
thread = threading.Thread(target=qe.run) |
|
182 |
qe.thread = thread |
|
183 |
thread.start() |
|
184 |
threads -= 1 |
|
185 |
for qe in executors: |
|
186 |
qe.thread.join() |
|
187 |
## for qe in executors: |
|
188 |
## print qe.thread, repeat - qe._times, 'times' |
|
189 |
else: |
|
190 |
QueryExecutor(repo_cursor, repeat, queries, reporter = reporter).run() |
|
191 |
reporter.dump_report(report_output) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
192 |
|
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
0
diff
changeset
|
193 |
|
0 | 194 |
if __name__ == '__main__': |
195 |
run(sys.argv[1:]) |