dataimport/massive_store.py
changeset 10873 0611466ce367
parent 10872 ff4f94cfa2fb
child 10874 538e17174769
equal deleted inserted replaced
10872:ff4f94cfa2fb 10873:0611466ce367
    83     """
    83     """
    84     # size of eid range reserved by the store for each batch
    84     # size of eid range reserved by the store for each batch
    85     eids_seq_range = 10000
    85     eids_seq_range = 10000
    86     # initial eid (None means use the value in the db)
    86     # initial eid (None means use the value in the db)
    87     eids_seq_start = None
    87     eids_seq_start = None
       
    88     # max size of the iid, used to create the iid_eid conversion table
       
    89     iid_maxsize = 1024
    88 
    90 
    89     def __init__(self, cnx, autoflush_metadata=True,
    91     def __init__(self, cnx, autoflush_metadata=True,
    90                  commit_at_flush=True,
    92                  commit_at_flush=True,
    91                  iid_maxsize=1024, uri_param_name='rdf:about',
    93                  uri_param_name='rdf:about',
    92                  on_commit_callback=None, on_rollback_callback=None,
    94                  on_commit_callback=None, on_rollback_callback=None,
    93                  slave_mode=False,
    95                  slave_mode=False,
    94                  source=None):
    96                  source=None):
    95         """ Create a MassiveObject store, with the following attributes:
    97         """ Create a MassiveObject store, with the following attributes:
    96 
    98 
    97         - cnx: CubicWeb cnx
    99         - cnx: CubicWeb cnx
    98         - autoflush_metadata: Boolean.
   100         - autoflush_metadata: Boolean.
    99                               Automatically flush the metadata after
   101                               Automatically flush the metadata after
   100                               each flush()
   102                               each flush()
   101         - commit_at_flush: Boolean. Commit after each flush().
   103         - commit_at_flush: Boolean. Commit after each flush().
   102         - iid_maxsize: Int. Max size of the iid, used to create the
       
   103                     iid_eid convertion table.
       
   104         - uri_param_name: String. If given, will use this parameter to get cw_uri
   104         - uri_param_name: String. If given, will use this parameter to get cw_uri
   105                           for entities.
   105                           for entities.
   106         """
   106         """
   107         super(MassiveObjectStore, self).__init__(cnx)
   107         super(MassiveObjectStore, self).__init__(cnx)
   108         self.logger = logging.getLogger('dataio.relationmixin')
   108         self.logger = logging.getLogger('dataio.relationmixin')
   109         self._cnx = cnx
   109         self._cnx = cnx
   110         self.sql = cnx.system_sql
   110         self.sql = cnx.system_sql
   111         self.iid_maxsize = iid_maxsize
       
   112         self.commit_at_flush = commit_at_flush
   111         self.commit_at_flush = commit_at_flush
   113         self._data_uri_relations = defaultdict(list)
   112         self._data_uri_relations = defaultdict(list)
   114         self._initialized = {'init_uri_eid': set(),
   113         self._initialized = {'init_uri_eid': set(),
   115                              'uri_eid_inserted': set(),
   114                              'uri_eid_inserted': set(),
   116                              'uri_rtypes': set(),
   115                              'uri_rtypes': set(),