equal
deleted
inserted
replaced
25 from time import asctime |
25 from time import asctime |
26 from datetime import date, datetime, time |
26 from datetime import date, datetime, time |
27 from collections import defaultdict |
27 from collections import defaultdict |
28 from base64 import b64encode |
28 from base64 import b64encode |
29 |
29 |
30 from six.moves import cPickle as pickle |
30 from six.moves import cPickle as pickle, range |
31 |
31 |
32 from cubicweb.utils import make_uid |
32 from cubicweb.utils import make_uid |
33 from cubicweb.server.sqlutils import SQL_PREFIX |
33 from cubicweb.server.sqlutils import SQL_PREFIX |
34 from cubicweb.dataimport.stores import NoHookRQLObjectStore |
34 from cubicweb.dataimport.stores import NoHookRQLObjectStore |
35 |
35 |
40 Import a bunch of sql statements, using different threads. |
40 Import a bunch of sql statements, using different threads. |
41 """ |
41 """ |
42 try: |
42 try: |
43 chunksize = (len(statements) / nb_threads) + 1 |
43 chunksize = (len(statements) / nb_threads) + 1 |
44 threads = [] |
44 threads = [] |
45 for i in xrange(nb_threads): |
45 for i in range(nb_threads): |
46 chunks = statements[i*chunksize:(i+1)*chunksize] |
46 chunks = statements[i*chunksize:(i+1)*chunksize] |
47 thread = threading.Thread(target=_execmany_thread, |
47 thread = threading.Thread(target=_execmany_thread, |
48 args=(sql_connect, chunks, |
48 args=(sql_connect, chunks, |
49 dump_output_dir, |
49 dump_output_dir, |
50 support_copy_from, |
50 support_copy_from, |
184 # Create a list rather than directly create a StringIO |
184 # Create a list rather than directly create a StringIO |
185 # to correctly write lines separated by '\n' in a single step |
185 # to correctly write lines separated by '\n' in a single step |
186 rows = [] |
186 rows = [] |
187 if columns is None: |
187 if columns is None: |
188 if isinstance(data[0], (tuple, list)): |
188 if isinstance(data[0], (tuple, list)): |
189 columns = range(len(data[0])) |
189 columns = list(range(len(data[0]))) |
190 elif isinstance(data[0], dict): |
190 elif isinstance(data[0], dict): |
191 columns = data[0].keys() |
191 columns = data[0].keys() |
192 else: |
192 else: |
193 raise ValueError('Could not get columns: you must provide columns.') |
193 raise ValueError('Could not get columns: you must provide columns.') |
194 for row in data: |
194 for row in data: |