from collections import defaultdict

todelete = defaultdict(list)

extids = set()

duplicates = []

    etype = entity.__regid__

    if not source.support_entity(etype):

        print "source doesn't support %s, delete %s" % (etype, entity.eid)

        todelete[etype].append(entity)

        continue

    try:

        entity.complete()

    except Exception:

        print '%s %s much probably deleted, delete it (extid %s)' % (

            etype, entity.eid, entity.cw_metainformation()['extid'])

        todelete[etype].append(entity)

        continue

    print 'get back', etype, entity.eid

    entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache)

    if not entity.creation_date:

        entity.cw_edited['creation_date'] = datetime.now()

    if not entity.modification_date:

        entity.cw_edited['modification_date'] = datetime.now()

    if not entity.upassword:

        entity.cw_edited['upassword'] = u''

    extid = entity.cw_metainformation()['extid']

    if not entity.cwuri:

        entity.cw_edited['cwuri'] = '%s/?dn=%s' % (

            source.urls[0], extid.decode('utf-8', 'ignore'))

    print entity.cw_edited

    if extid in extids:

        duplicates.append(extid)

        continue

    extids.add(extid)

    system_source.add_entity(session, entity)

    sql("UPDATE entities SET source='system' "

        "WHERE eid=%(eid)s", {'eid': entity.eid})

if duplicates:

    print 'found %s duplicate entries' % len(duplicates)

    from pprint import pprint

    pprint(duplicates)

print len(todelete), 'entities will be deleted'

for etype, entities in todelete.values():

    print 'deleting', etype, [e.login for e in entities]

if raw_input('Commit ?') in 'yY':

    print 'committing'

    commit()

else:

    rollback()

    print 'rollbacked'