50 thread.start() |
51 thread.start() |
51 threads.append(thread) |
52 threads.append(thread) |
52 for t in threads: |
53 for t in threads: |
53 t.join() |
54 t.join() |
54 except Exception: |
55 except Exception: |
55 print 'Error in import statements' |
56 print('Error in import statements') |
56 |
57 |
57 def _execmany_thread_not_copy_from(cu, statement, data, table=None, |
58 def _execmany_thread_not_copy_from(cu, statement, data, table=None, |
58 columns=None, encoding='utf-8'): |
59 columns=None, encoding='utf-8'): |
59 """ Execute thread without copy from |
60 """ Execute thread without copy from |
60 """ |
61 """ |
98 columns = None |
99 columns = None |
99 else: |
100 else: |
100 columns = list(data[0]) |
101 columns = list(data[0]) |
101 execmany_func(cu, statement, data, table, columns, encoding) |
102 execmany_func(cu, statement, data, table, columns, encoding) |
102 except Exception: |
103 except Exception: |
103 print 'unable to copy data into table %s' % table |
104 print('unable to copy data into table %s' % table) |
104 # Error in import statement, save data in dump_output_dir |
105 # Error in import statement, save data in dump_output_dir |
105 if dump_output_dir is not None: |
106 if dump_output_dir is not None: |
106 pdata = {'data': data, 'statement': statement, |
107 pdata = {'data': data, 'statement': statement, |
107 'time': asctime(), 'columns': columns} |
108 'time': asctime(), 'columns': columns} |
108 filename = make_uid() |
109 filename = make_uid() |
109 try: |
110 try: |
110 with open(osp.join(dump_output_dir, |
111 with open(osp.join(dump_output_dir, |
111 '%s.pickle' % filename), 'w') as fobj: |
112 '%s.pickle' % filename), 'w') as fobj: |
112 fobj.write(cPickle.dumps(pdata)) |
113 fobj.write(cPickle.dumps(pdata)) |
113 except IOError: |
114 except IOError: |
114 print 'ERROR while pickling in', dump_output_dir, filename+'.pickle' |
115 print('ERROR while pickling in', dump_output_dir, filename+'.pickle') |
115 pass |
116 pass |
116 cnx.rollback() |
117 cnx.rollback() |
117 raise |
118 raise |
118 finally: |
119 finally: |
119 cnx.commit() |
120 cnx.commit() |
333 self._sql.inlined_relations = {} |
334 self._sql.inlined_relations = {} |
334 # keep track, for each eid of the corresponding data dict |
335 # keep track, for each eid of the corresponding data dict |
335 self._sql.eid_insertdicts = {} |
336 self._sql.eid_insertdicts = {} |
336 |
337 |
337 def flush(self): |
338 def flush(self): |
338 print 'starting flush' |
339 print('starting flush') |
339 _entities_sql = self._sql.entities |
340 _entities_sql = self._sql.entities |
340 _relations_sql = self._sql.relations |
341 _relations_sql = self._sql.relations |
341 _inlined_relations_sql = self._sql.inlined_relations |
342 _inlined_relations_sql = self._sql.inlined_relations |
342 _insertdicts = self._sql.eid_insertdicts |
343 _insertdicts = self._sql.eid_insertdicts |
343 try: |
344 try: |