|
1 # copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
|
2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
|
3 # |
|
4 # This file is part of CubicWeb. |
|
5 # |
|
6 # CubicWeb is free software: you can redistribute it and/or modify it under the |
|
7 # terms of the GNU Lesser General Public License as published by the Free |
|
8 # Software Foundation, either version 2.1 of the License, or (at your option) |
|
9 # any later version. |
|
10 # |
|
11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
|
12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
|
13 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
|
14 # details. |
|
15 # |
|
16 # You should have received a copy of the GNU Lesser General Public License along |
|
17 # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
|
18 """ Usage: %s [OPTIONS] <instance id> <queries file> |
|
19 |
|
20 Stress test a CubicWeb repository |
|
21 |
|
22 OPTIONS: |
|
23 -h / --help |
|
24 Display this help message and exit. |
|
25 |
|
26 -u / --user <user> |
|
27 Connect as <user> instead of being prompted to give it. |
|
28 -p / --password <password> |
|
29 Automatically give <password> for authentication instead of being prompted |
|
30 to give it. |
|
31 |
|
32 -n / --nb-times <num> |
|
33 Repeat queries <num> times. |
|
34 -t / --nb-threads <num> |
|
35 Execute queries in <num> parallel threads. |
|
36 -P / --profile <prof_file> |
|
37 dumps profile results (hotshot) in <prof_file> |
|
38 -o / --report-output <filename> |
|
39 Write profiler report into <filename> rather than on stdout |
|
40 |
|
41 Copyright (c) 2003-2011 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. |
|
42 http://www.logilab.fr/ -- mailto:contact@logilab.fr |
|
43 """ |
|
44 from __future__ import print_function |
|
45 |
|
46 import os |
|
47 import sys |
|
48 import threading |
|
49 import getopt |
|
50 import traceback |
|
51 from getpass import getpass |
|
52 from os.path import basename |
|
53 from time import clock |
|
54 |
|
55 from logilab.common.fileutils import lines |
|
56 from logilab.common.ureports import Table, TextWriter |
|
57 from cubicweb.server.repository import Repository |
|
58 from cubicweb.dbapi import Connection |
|
59 |
|
60 TB_LOCK = threading.Lock() |
|
61 |
|
62 class QueryExecutor: |
|
63 def __init__(self, cursor, times, queries, reporter = None): |
|
64 self._cursor = cursor |
|
65 self._times = times |
|
66 self._queries = queries |
|
67 self._reporter = reporter |
|
68 |
|
69 def run(self): |
|
70 cursor = self._cursor |
|
71 times = self._times |
|
72 while times: |
|
73 for index, query in enumerate(self._queries): |
|
74 start = clock() |
|
75 try: |
|
76 cursor.execute(query) |
|
77 except Exception: |
|
78 TB_LOCK.acquire() |
|
79 traceback.print_exc() |
|
80 TB_LOCK.release() |
|
81 return |
|
82 if self._reporter is not None: |
|
83 self._reporter.add_proftime(clock() - start, index) |
|
84 times -= 1 |
|
85 |
|
86 def usage(status=0): |
|
87 """print usage string and exit""" |
|
88 print(__doc__ % basename(sys.argv[0])) |
|
89 sys.exit(status) |
|
90 |
|
91 |
|
92 class ProfileReporter: |
|
93 """a profile reporter gathers all profile informations from several |
|
94 threads and can write a report that summarizes all profile informations |
|
95 """ |
|
96 profiler_lock = threading.Lock() |
|
97 |
|
98 def __init__(self, queries): |
|
99 self._queries = tuple(queries) |
|
100 self._profile_results = [(0., 0)] * len(self._queries) |
|
101 # self._table_report = Table(3, rheaders = True) |
|
102 len_max = max([len(query) for query in self._queries]) + 5 |
|
103 self._query_fmt = '%%%ds' % len_max |
|
104 |
|
105 def add_proftime(self, elapsed_time, query_index): |
|
106 """add a new time measure for query""" |
|
107 ProfileReporter.profiler_lock.acquire() |
|
108 cumul_time, times = self._profile_results[query_index] |
|
109 cumul_time += elapsed_time |
|
110 times += 1. |
|
111 self._profile_results[query_index] = (cumul_time, times) |
|
112 ProfileReporter.profiler_lock.release() |
|
113 |
|
114 def dump_report(self, output = sys.stdout): |
|
115 """dump report in 'output'""" |
|
116 table_elems = ['RQL Query', 'Times', 'Avg Time'] |
|
117 total_time = 0. |
|
118 for query, (cumul_time, times) in zip(self._queries, self._profile_results): |
|
119 avg_time = cumul_time / float(times) |
|
120 table_elems += [str(query), '%f' % times, '%f' % avg_time ] |
|
121 total_time += cumul_time |
|
122 table_elems.append('Total time :') |
|
123 table_elems.append(str(total_time)) |
|
124 table_elems.append(' ') |
|
125 table_layout = Table(3, rheaders = True, children = table_elems) |
|
126 TextWriter().format(table_layout, output) |
|
127 # output.write('\n'.join(tmp_output)) |
|
128 |
|
129 |
|
130 def run(args): |
|
131 """run the command line tool""" |
|
132 try: |
|
133 opts, args = getopt.getopt(args, 'hn:t:u:p:P:o:', ['help', 'user=', 'password=', |
|
134 'nb-times=', 'nb-threads=', |
|
135 'profile', 'report-output=',]) |
|
136 except Exception as ex: |
|
137 print(ex) |
|
138 usage(1) |
|
139 repeat = 100 |
|
140 threads = 1 |
|
141 user = os.environ.get('USER', os.environ.get('LOGNAME')) |
|
142 password = None |
|
143 report_output = sys.stdout |
|
144 prof_file = None |
|
145 for opt, val in opts: |
|
146 if opt in ('-h', '--help'): |
|
147 usage() |
|
148 if opt in ('-u', '--user'): |
|
149 user = val |
|
150 elif opt in ('-p', '--password'): |
|
151 password = val |
|
152 elif opt in ('-n', '--nb-times'): |
|
153 repeat = int(val) |
|
154 elif opt in ('-t', '--nb-threads'): |
|
155 threads = int(val) |
|
156 elif opt in ('-P', '--profile'): |
|
157 prof_file = val |
|
158 elif opt in ('-o', '--report-output'): |
|
159 report_output = open(val, 'w') |
|
160 if len(args) != 2: |
|
161 usage(1) |
|
162 queries = [query for query in lines(args[1]) if not query.startswith('#')] |
|
163 if user is None: |
|
164 user = raw_input('login: ') |
|
165 if password is None: |
|
166 password = getpass('password: ') |
|
167 from cubicweb.cwconfig import instance_configuration |
|
168 config = instance_configuration(args[0]) |
|
169 # get local access to the repository |
|
170 print("Creating repo", prof_file) |
|
171 repo = Repository(config, prof_file) |
|
172 cnxid = repo.connect(user, password=password) |
|
173 # connection to the CubicWeb repository |
|
174 repo_cnx = Connection(repo, cnxid) |
|
175 repo_cursor = repo_cnx.cursor() |
|
176 reporter = ProfileReporter(queries) |
|
177 if threads > 1: |
|
178 executors = [] |
|
179 while threads: |
|
180 qe = QueryExecutor(repo_cursor, repeat, queries, reporter = reporter) |
|
181 executors.append(qe) |
|
182 thread = threading.Thread(target=qe.run) |
|
183 qe.thread = thread |
|
184 thread.start() |
|
185 threads -= 1 |
|
186 for qe in executors: |
|
187 qe.thread.join() |
|
188 ## for qe in executors: |
|
189 ## print qe.thread, repeat - qe._times, 'times' |
|
190 else: |
|
191 QueryExecutor(repo_cursor, repeat, queries, reporter = reporter).run() |
|
192 reporter.dump_report(report_output) |
|
193 |
|
194 |
|
195 if __name__ == '__main__': |
|
196 run(sys.argv[1:]) |