|
1 """ Usage: %s [OPTIONS] <application id> <queries file> |
|
2 |
|
3 Stress test a CubicWeb repository |
|
4 |
|
5 OPTIONS: |
|
6 -h / --help |
|
7 Display this help message and exit. |
|
8 |
|
9 -u / --user <user> |
|
10 Connect as <user> instead of being prompted to give it. |
|
11 -p / --password <password> |
|
12 Automatically give <password> for authentication instead of being prompted |
|
13 to give it. |
|
14 |
|
15 -n / --nb-times <num> |
|
16 Repeat queries <num> times. |
|
17 -t / --nb-threads <num> |
|
18 Execute queries in <num> parallel threads. |
|
19 -P / --profile <prof_file> |
|
20 dumps profile results (hotshot) in <prof_file> |
|
21 -o / --report-output <filename> |
|
22 Write profiler report into <filename> rather than on stdout |
|
23 |
|
24 Copyright (c) 2003-2006 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
|
25 http://www.logilab.fr/ -- mailto:contact@logilab.fr |
|
26 """ |
|
27 |
|
28 __revision__ = "$Id: stresstester.py,v 1.3 2006-03-05 14:35:27 syt Exp $" |
|
29 |
|
30 import os |
|
31 import sys |
|
32 import threading |
|
33 import getopt |
|
34 import traceback |
|
35 from getpass import getpass |
|
36 from os.path import basename |
|
37 from time import clock |
|
38 |
|
39 from logilab.common.fileutils import lines |
|
40 from logilab.common.ureports import Table, TextWriter |
|
41 from cubicweb.server.repository import Repository |
|
42 from cubicweb.dbapi import Connection |
|
43 |
|
44 TB_LOCK = threading.Lock() |
|
45 |
|
46 class QueryExecutor: |
|
47 def __init__(self, cursor, times, queries, reporter = None): |
|
48 self._cursor = cursor |
|
49 self._times = times |
|
50 self._queries = queries |
|
51 self._reporter = reporter |
|
52 |
|
53 def run(self): |
|
54 cursor = self._cursor |
|
55 times = self._times |
|
56 while times: |
|
57 for index, query in enumerate(self._queries): |
|
58 start = clock() |
|
59 try: |
|
60 cursor.execute(query) |
|
61 except KeyboardInterrupt: |
|
62 raise |
|
63 except: |
|
64 TB_LOCK.acquire() |
|
65 traceback.print_exc() |
|
66 TB_LOCK.release() |
|
67 return |
|
68 if self._reporter is not None: |
|
69 self._reporter.add_proftime(clock() - start, index) |
|
70 times -= 1 |
|
71 |
|
72 def usage(status=0): |
|
73 """print usage string and exit""" |
|
74 print __doc__ % basename(sys.argv[0]) |
|
75 sys.exit(status) |
|
76 |
|
77 |
|
78 class ProfileReporter: |
|
79 """a profile reporter gathers all profile informations from several |
|
80 threads and can write a report that summarizes all profile informations |
|
81 """ |
|
82 profiler_lock = threading.Lock() |
|
83 |
|
84 def __init__(self, queries): |
|
85 self._queries = tuple(queries) |
|
86 self._profile_results = [(0., 0)] * len(self._queries) |
|
87 # self._table_report = Table(3, rheaders = True) |
|
88 len_max = max([len(query) for query in self._queries]) + 5 |
|
89 self._query_fmt = '%%%ds' % len_max |
|
90 |
|
91 def add_proftime(self, elapsed_time, query_index): |
|
92 """add a new time measure for query""" |
|
93 ProfileReporter.profiler_lock.acquire() |
|
94 cumul_time, times = self._profile_results[query_index] |
|
95 cumul_time += elapsed_time |
|
96 times += 1. |
|
97 self._profile_results[query_index] = (cumul_time, times) |
|
98 ProfileReporter.profiler_lock.release() |
|
99 |
|
100 def dump_report(self, output = sys.stdout): |
|
101 """dump report in 'output'""" |
|
102 table_elems = ['RQL Query', 'Times', 'Avg Time'] |
|
103 total_time = 0. |
|
104 for query, (cumul_time, times) in zip(self._queries, self._profile_results): |
|
105 avg_time = cumul_time / float(times) |
|
106 table_elems += [str(query), '%f' % times, '%f' % avg_time ] |
|
107 total_time += cumul_time |
|
108 table_elems.append('Total time :') |
|
109 table_elems.append(str(total_time)) |
|
110 table_elems.append(' ') |
|
111 table_layout = Table(3, rheaders = True, children = table_elems) |
|
112 TextWriter().format(table_layout, output) |
|
113 # output.write('\n'.join(tmp_output)) |
|
114 |
|
115 |
|
116 def run(args): |
|
117 """run the command line tool""" |
|
118 try: |
|
119 opts, args = getopt.getopt(args, 'hn:t:u:p:P:o:', ['help', 'user=', 'password=', |
|
120 'nb-times=', 'nb-threads=', |
|
121 'profile', 'report-output=',]) |
|
122 except Exception, ex: |
|
123 print ex |
|
124 usage(1) |
|
125 repeat = 100 |
|
126 threads = 1 |
|
127 user = os.environ.get('USER', os.environ.get('LOGNAME')) |
|
128 password = None |
|
129 report_output = sys.stdout |
|
130 prof_file = None |
|
131 for opt, val in opts: |
|
132 if opt in ('-h', '--help'): |
|
133 usage() |
|
134 if opt in ('-u', '--user'): |
|
135 user = val |
|
136 elif opt in ('-p', '--password'): |
|
137 password = val |
|
138 elif opt in ('-n', '--nb-times'): |
|
139 repeat = int(val) |
|
140 elif opt in ('-t', '--nb-threads'): |
|
141 threads = int(val) |
|
142 elif opt in ('-P', '--profile'): |
|
143 prof_file = val |
|
144 elif opt in ('-o', '--report-output'): |
|
145 report_output = file(val, 'w') |
|
146 if len(args) != 2: |
|
147 usage(1) |
|
148 queries = [query for query in lines(args[1]) if not query.startswith('#')] |
|
149 if user is None: |
|
150 user = raw_input('login: ') |
|
151 if password is None: |
|
152 password = getpass('password: ') |
|
153 from cubicweb.cwconfig import application_configuration |
|
154 config = application_configuration(args[0]) |
|
155 # get local access to the repository |
|
156 print "Creating repo", prof_file |
|
157 repo = Repository(config, prof_file) |
|
158 cnxid = repo.connect(user, password) |
|
159 # connection to the CubicWeb repository |
|
160 repo_cnx = Connection(repo, cnxid) |
|
161 repo_cursor = repo_cnx.cursor() |
|
162 reporter = ProfileReporter(queries) |
|
163 if threads > 1: |
|
164 executors = [] |
|
165 while threads: |
|
166 qe = QueryExecutor(repo_cursor, repeat, queries, reporter = reporter) |
|
167 executors.append(qe) |
|
168 thread = threading.Thread(target=qe.run) |
|
169 qe.thread = thread |
|
170 thread.start() |
|
171 threads -= 1 |
|
172 for qe in executors: |
|
173 qe.thread.join() |
|
174 ## for qe in executors: |
|
175 ## print qe.thread, repeat - qe._times, 'times' |
|
176 else: |
|
177 QueryExecutor(repo_cursor, repeat, queries, reporter = reporter).run() |
|
178 reporter.dump_report(report_output) |
|
179 |
|
180 |
|
181 if __name__ == '__main__': |
|
182 run(sys.argv[1:]) |