server/sqlutils.py
author Julien Cristau <julien.cristau@logilab.fr>
Mon, 05 Oct 2015 14:51:24 +0200
changeset 10766 d730f91251af
parent 10760 c34bbdb18745
child 10824 fbefdaa56d2b
permissions -rw-r--r--
[ldapfeed] port to ldap3 python-ldap has no python3 support. XXX I only checked that unittest_ldapfeed passes.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
9466
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
     1
# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
     3
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
     4
# This file is part of CubicWeb.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
     5
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
     9
# any later version.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
    10
#
5424
8ecbcbff9777 replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5421
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
    14
# details.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
    15
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5214
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
5768
1e73a466aa69 [fti] support for fti ranking: has_text query results sorted by relevance, and provides a way to control weight per entity / entity's attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5625
diff changeset
    18
"""SQL utilities functions and classes."""
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10588
diff changeset
    19
from __future__ import print_function
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    20
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    21
__docformat__ = "restructuredtext en"
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    22
9466
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
    23
import sys
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    24
import re
4298
2ca56131079e enable cubicweb-ctl db-dump and db-restore on Windows with SQL Server
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 4212
diff changeset
    25
import subprocess
9466
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
    26
from os.path import abspath
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
    27
from logging import getLogger
10643
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
    28
from datetime import time, datetime
1016
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    29
10760
c34bbdb18745 [server] unicode → six.text_type
Julien Cristau <julien.cristau@logilab.fr>
parents: 10662
diff changeset
    30
from six import string_types, text_type
10607
6519ae8cca0c [py3k] import filter using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10589
diff changeset
    31
from six.moves import filter
6519ae8cca0c [py3k] import filter using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10589
diff changeset
    32
4849
3827b9ee77ac missing rename
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4848
diff changeset
    33
from logilab import database as db, common as lgc
10375
28ec01db78b3 [server] Do not use progress bar when stdout is not a TTY
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 10125
diff changeset
    34
from logilab.common.shellutils import ProgressBar, DummyProgressBar
9466
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
    35
from logilab.common.deprecation import deprecated
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
    36
from logilab.common.logging_ext import set_log_methods
10643
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
    37
from logilab.common.date import utctime, utcdatetime
4848
41f84eea63c9 rename logilab.db into logilab.database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4831
diff changeset
    38
from logilab.database.sqlgen import SQLGenerator
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    39
9466
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
    40
from cubicweb import Binary, ConfigurationError
4023
eae23c40627a drop common subpackage
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3689
diff changeset
    41
from cubicweb.uilib import remove_html_tags
2596
d02eed70937f [R repo, schema] use VIRTUAL_RTYPES const
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2512
diff changeset
    42
from cubicweb.schema import PURE_VIRTUAL_RTYPES
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    43
from cubicweb.server import SQL_CONNECT_HOOKS
1132
96752791c2b6 pylint cleanup
sylvain.thenault@logilab.fr
parents: 1026
diff changeset
    44
from cubicweb.server.utils import crypt_password
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    45
1783
b81f9761907c mx flag has moved
sylvain.thenault@logilab.fr
parents: 1619
diff changeset
    46
lgc.USE_MX_DATETIME = False
1251
af40e615dc89 introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents: 0
diff changeset
    47
SQL_PREFIX = 'cw_'
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    48
4353
7db69db4913c command may now officially be either a string or a list, don't make think it's for backward compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4342
diff changeset
    49
def _run_command(cmd):
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10588
diff changeset
    50
    print(' '.join(cmd))
4342
b4e186da08f2 handle lgc.adbh api changes within regards of backup/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4322
diff changeset
    51
    return subprocess.call(cmd)
b4e186da08f2 handle lgc.adbh api changes within regards of backup/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4322
diff changeset
    52
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    53
9891
3386fd89c914 remove references to global environment variable APYCOT_ROOT
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9724
diff changeset
    54
def sqlexec(sqlstmts, cursor_or_execute, withpb=True,
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    55
            pbtitle='', delimiter=';', cnx=None):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    56
    """execute sql statements ignoring DROP/ CREATE GROUP or USER statements
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    57
    error.
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    58
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    59
    :sqlstmts_as_string: a string or a list of sql statements.
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    60
    :cursor_or_execute: sql cursor or a callback used to execute statements
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    61
    :cnx: if given, commit/rollback at each statement.
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    62
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    63
    :withpb: if True, display a progresse bar
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    64
    :pbtitle: a string displayed as the progress bar title (if `withpb=True`)
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    65
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    66
    :delimiter: a string used to split sqlstmts (if it is a string)
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    67
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    68
    Return the failed statements (same type as sqlstmts)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    69
    """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    70
    if hasattr(cursor_or_execute, 'execute'):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    71
        execute = cursor_or_execute.execute
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    72
    else:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    73
        execute = cursor_or_execute
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    74
    sqlstmts_as_string = False
10612
84468b90e9c1 [py3k] basestring → six.string_types
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10607
diff changeset
    75
    if isinstance(sqlstmts, string_types):
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    76
        sqlstmts_as_string = True
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    77
        sqlstmts = sqlstmts.split(delimiter)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    78
    if withpb:
10375
28ec01db78b3 [server] Do not use progress bar when stdout is not a TTY
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 10125
diff changeset
    79
        if sys.stdout.isatty():
28ec01db78b3 [server] Do not use progress bar when stdout is not a TTY
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 10125
diff changeset
    80
            pb = ProgressBar(len(sqlstmts), title=pbtitle)
28ec01db78b3 [server] Do not use progress bar when stdout is not a TTY
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 10125
diff changeset
    81
        else:
28ec01db78b3 [server] Do not use progress bar when stdout is not a TTY
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 10125
diff changeset
    82
            pb = DummyProgressBar()
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    83
    failed = []
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    84
    for sql in sqlstmts:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    85
        sql = sql.strip()
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    86
        if withpb:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    87
            pb.update()
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    88
        if not sql:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    89
            continue
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    90
        try:
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    91
            # some dbapi modules doesn't accept unicode for sql string
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    92
            execute(str(sql))
10588
fdaa0e4b7eaf [py3k] except as
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10411
diff changeset
    93
        except Exception as err:
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    94
            if cnx:
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    95
                cnx.rollback()
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    96
            failed.append(sql)
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    97
        else:
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    98
            if cnx:
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
    99
                cnx.commit()
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   100
    if withpb:
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10588
diff changeset
   101
        print()
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   102
    if sqlstmts_as_string:
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   103
        failed = delimiter.join(failed)
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   104
    return failed
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   105
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   106
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   107
def sqlgrants(schema, driver, user,
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   108
              text_index=True, set_owner=True,
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   109
              skip_relations=(), skip_entities=()):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   110
    """return sql to give all access privileges to the given user on the system
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   111
    schema
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   112
    """
10200
cceb2c7c02f4 Use our version of schema2sql
Julien Cristau <julien.cristau@logilab.fr>
parents: 10125
diff changeset
   113
    from cubicweb.server.schema2sql import grant_schema
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   114
    from cubicweb.server.sources import native
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   115
    output = []
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   116
    w = output.append
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   117
    w(native.grant_schema(user, set_owner))
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   118
    w('')
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   119
    if text_index:
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   120
        dbhelper = db.get_db_helper(driver)
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   121
        w(dbhelper.sql_grant_user_on_fti(user))
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   122
        w('')
1251
af40e615dc89 introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents: 0
diff changeset
   123
    w(grant_schema(schema, user, set_owner, skip_entities=skip_entities, prefix=SQL_PREFIX))
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   124
    return '\n'.join(output)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   125
1619
e4845b54a704 force proper date/datetime according to type (necessary for sqlite at least)
sylvain.thenault@logilab.fr
parents: 1408
diff changeset
   126
e4845b54a704 force proper date/datetime according to type (necessary for sqlite at least)
sylvain.thenault@logilab.fr
parents: 1408
diff changeset
   127
def sqlschema(schema, driver, text_index=True,
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   128
              user=None, set_owner=False,
2596
d02eed70937f [R repo, schema] use VIRTUAL_RTYPES const
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2512
diff changeset
   129
              skip_relations=PURE_VIRTUAL_RTYPES, skip_entities=()):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   130
    """return the system sql schema, according to the given parameters"""
10200
cceb2c7c02f4 Use our version of schema2sql
Julien Cristau <julien.cristau@logilab.fr>
parents: 10125
diff changeset
   131
    from cubicweb.server.schema2sql import schema2sql
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   132
    from cubicweb.server.sources import native
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   133
    if set_owner:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   134
        assert user, 'user is argument required when set_owner is true'
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   135
    output = []
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   136
    w = output.append
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   137
    w(native.sql_schema(driver))
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   138
    w('')
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   139
    dbhelper = db.get_db_helper(driver)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   140
    if text_index:
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4899
diff changeset
   141
        w(dbhelper.sql_init_fti().replace(';', ';;'))
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   142
        w('')
1619
e4845b54a704 force proper date/datetime according to type (necessary for sqlite at least)
sylvain.thenault@logilab.fr
parents: 1408
diff changeset
   143
    w(schema2sql(dbhelper, schema, prefix=SQL_PREFIX,
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4899
diff changeset
   144
                 skip_entities=skip_entities,
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4899
diff changeset
   145
                 skip_relations=skip_relations).replace(';', ';;'))
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   146
    if dbhelper.users_support and user:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   147
        w('')
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   148
        w(sqlgrants(schema, driver, user, text_index, set_owner,
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4899
diff changeset
   149
                    skip_relations, skip_entities).replace(';', ';;'))
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   150
    return '\n'.join(output)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   151
1619
e4845b54a704 force proper date/datetime according to type (necessary for sqlite at least)
sylvain.thenault@logilab.fr
parents: 1408
diff changeset
   152
e4845b54a704 force proper date/datetime according to type (necessary for sqlite at least)
sylvain.thenault@logilab.fr
parents: 1408
diff changeset
   153
def sqldropschema(schema, driver, text_index=True,
2596
d02eed70937f [R repo, schema] use VIRTUAL_RTYPES const
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2512
diff changeset
   154
                  skip_relations=PURE_VIRTUAL_RTYPES, skip_entities=()):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   155
    """return the sql to drop the schema, according to the given parameters"""
10200
cceb2c7c02f4 Use our version of schema2sql
Julien Cristau <julien.cristau@logilab.fr>
parents: 10125
diff changeset
   156
    from cubicweb.server.schema2sql import dropschema2sql
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   157
    from cubicweb.server.sources import native
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   158
    output = []
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   159
    w = output.append
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   160
    if text_index:
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   161
        dbhelper = db.get_db_helper(driver)
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   162
        w(dbhelper.sql_drop_fti())
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   163
        w('')
7906
203d574c8a1d repaire cctl db-init -d on sqlserver (closes #1979670)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7341
diff changeset
   164
    w(dropschema2sql(dbhelper, schema, prefix=SQL_PREFIX,
1954
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1953
diff changeset
   165
                     skip_entities=skip_entities,
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1953
diff changeset
   166
                     skip_relations=skip_relations))
5214
3285b6e3b930 fix cwctl db-init -d on SQL Server
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5013
diff changeset
   167
    w('')
3285b6e3b930 fix cwctl db-init -d on SQL Server
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5013
diff changeset
   168
    w(native.sql_drop_schema(driver))
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   169
    return '\n'.join(output)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   170
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   171
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   172
_SQL_DROP_ALL_USER_TABLES_FILTER_FUNCTION = re.compile('^(?!(sql|pg)_)').match
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   173
def sql_drop_all_user_tables(driver_or_helper, sqlcursor):
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   174
    """Return ths sql to drop all tables found in the database system."""
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   175
    if not getattr(driver_or_helper, 'list_tables', None):
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   176
        dbhelper = db.get_db_helper(driver_or_helper)
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   177
    else:
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   178
        dbhelper = driver_or_helper
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   179
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   180
    cmds = [dbhelper.sql_drop_sequence('entities_id_seq')]
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   181
    # for mssql, we need to drop views before tables
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   182
    if hasattr(dbhelper, 'list_views'):
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   183
        cmds += ['DROP VIEW %s;' % name
10607
6519ae8cca0c [py3k] import filter using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10589
diff changeset
   184
                 for name in filter(_SQL_DROP_ALL_USER_TABLES_FILTER_FUNCTION, dbhelper.list_views(sqlcursor))]
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   185
    cmds += ['DROP TABLE %s;' % name
10607
6519ae8cca0c [py3k] import filter using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10589
diff changeset
   186
             for name in filter(_SQL_DROP_ALL_USER_TABLES_FILTER_FUNCTION, dbhelper.list_tables(sqlcursor))]
8755
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   187
    return '\n'.join(cmds)
1f3757ef3762 [server] *init_repository* lookup the database instead of the schema to drop tables (closes #810743)
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 8139
diff changeset
   188
9447
0636c4960259 [multi-sources-removal] Drop cw.server.sources.extlite
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9364
diff changeset
   189
9466
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   190
class ConnectionWrapper(object):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   191
    """handle connection to the system source, at some point associated to a
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   192
    :class:`Session`
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   193
    """
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   194
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   195
    # since 3.19, we only have to manage the system source connection
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   196
    def __init__(self, system_source):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   197
        # dictionary of (source, connection), indexed by sources'uri
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   198
        self._source = system_source
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   199
        self.cnx = system_source.get_connection()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   200
        self.cu = self.cnx.cursor()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   201
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   202
    def commit(self):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   203
        """commit the current transaction for this user"""
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   204
        # let exception propagates
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   205
        self.cnx.commit()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   206
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   207
    def rollback(self):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   208
        """rollback the current transaction for this user"""
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   209
        # catch exceptions, rollback other sources anyway
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   210
        try:
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   211
            self.cnx.rollback()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   212
        except Exception:
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   213
            self._source.critical('rollback error', exc_info=sys.exc_info())
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   214
            # error on rollback, the connection is much probably in a really
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   215
            # bad state. Replace it by a new one.
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   216
            self.reconnect()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   217
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   218
    def close(self, i_know_what_i_do=False):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   219
        """close all connections in the set"""
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   220
        if i_know_what_i_do is not True: # unexpected closing safety belt
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   221
            raise RuntimeError('connections set shouldn\'t be closed')
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   222
        try:
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   223
            self.cu.close()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   224
            self.cu = None
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   225
        except Exception:
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   226
            pass
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   227
        try:
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   228
            self.cnx.close()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   229
            self.cnx = None
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   230
        except Exception:
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   231
            pass
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   232
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   233
    # internals ###############################################################
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   234
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   235
    def cnxset_freed(self):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   236
        """connections set is being freed from a session"""
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   237
        pass # no nothing by default
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   238
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   239
    def reconnect(self):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   240
        """reopen a connection for this source or all sources if none specified
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   241
        """
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   242
        try:
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   243
            # properly close existing connection if any
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   244
            self.cnx.close()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   245
        except Exception:
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   246
            pass
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   247
        self._source.info('trying to reconnect')
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   248
        self.cnx = self._source.get_connection()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   249
        self.cu = self.cnx.cursor()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   250
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   251
    @deprecated('[3.19] use .cu instead')
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   252
    def __getitem__(self, uri):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   253
        assert uri == 'system'
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   254
        return self.cu
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   255
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   256
    @deprecated('[3.19] use repo.system_source instead')
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   257
    def source(self, uid):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   258
        assert uid == 'system'
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   259
        return self._source
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   260
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   261
    @deprecated('[3.19] use .cnx instead')
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   262
    def connection(self, uid):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   263
        assert uid == 'system'
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   264
        return self.cnx
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   265
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   266
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   267
class SqliteConnectionWrapper(ConnectionWrapper):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   268
    """Sqlite specific connection wrapper: close the connection each time it's
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   269
    freed (and reopen it later when needed)
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   270
    """
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   271
    def __init__(self, system_source):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   272
        # don't call parent's __init__, we don't want to initiate the connection
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   273
        self._source = system_source
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   274
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   275
    _cnx = None
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   276
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   277
    def cnxset_freed(self):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   278
        self.cu.close()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   279
        self.cnx.close()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   280
        self.cnx = self.cu = None
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   281
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   282
    @property
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   283
    def cnx(self):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   284
        if self._cnx is None:
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   285
            self._cnx = self._source.get_connection()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   286
            self._cu = self._cnx.cursor()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   287
        return self._cnx
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   288
    @cnx.setter
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   289
    def cnx(self, value):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   290
        self._cnx = value
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   291
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   292
    @property
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   293
    def cu(self):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   294
        if self._cnx is None:
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   295
            self._cnx = self._source.get_connection()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   296
            self._cu = self._cnx.cursor()
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   297
        return self._cu
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   298
    @cu.setter
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   299
    def cu(self, value):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   300
        self._cu = value
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   301
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   302
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   303
class SQLAdapterMixIn(object):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   304
    """Mixin for SQL data sources, getting a connection from a configuration
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   305
    dictionary and handling connection locking
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   306
    """
9466
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   307
    cnx_wrap = ConnectionWrapper
1619
e4845b54a704 force proper date/datetime according to type (necessary for sqlite at least)
sylvain.thenault@logilab.fr
parents: 1408
diff changeset
   308
9724
e45bf9baa7b7 Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents: 9466
diff changeset
   309
    def __init__(self, source_config, repairing=False):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   310
        try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   311
            self.dbdriver = source_config['db-driver'].lower()
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   312
            dbname = source_config['db-name']
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   313
        except KeyError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   314
            raise ConfigurationError('missing some expected entries in sources file')
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   315
        dbhost = source_config.get('db-host')
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   316
        port = source_config.get('db-port')
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   317
        dbport = port and int(port) or None
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   318
        dbuser = source_config.get('db-user')
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   319
        dbpassword = source_config.get('db-password')
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   320
        dbencoding = source_config.get('db-encoding', 'UTF-8')
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   321
        dbextraargs = source_config.get('db-extra-arguments')
10125
bc6461a7d2da [server] add a db-namespace option in source definition (closes #1631339)
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 10086
diff changeset
   322
        dbnamespace = source_config.get('db-namespace')
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   323
        self.dbhelper = db.get_db_helper(self.dbdriver)
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   324
        self.dbhelper.record_connection_info(dbname, dbhost, dbport, dbuser,
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   325
                                             dbpassword, dbextraargs,
10125
bc6461a7d2da [server] add a db-namespace option in source definition (closes #1631339)
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 10086
diff changeset
   326
                                             dbencoding, dbnamespace)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   327
        self.sqlgen = SQLGenerator()
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   328
        # copy back some commonly accessed attributes
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   329
        dbapi_module = self.dbhelper.dbapi_module
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   330
        self.OperationalError = dbapi_module.OperationalError
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   331
        self.InterfaceError = dbapi_module.InterfaceError
5605
2604545d7dd9 [win32 SQLServer] connection lost detection
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5424
diff changeset
   332
        self.DbapiError = dbapi_module.Error
6379
3f67f7ea5632 [R] use dbhelper.binary_value to process passwords and other Bytes fields
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6333
diff changeset
   333
        self._binary = self.dbhelper.binary_value
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   334
        self._process_value = dbapi_module.process_value
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   335
        self._dbencoding = dbencoding
9466
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   336
        if self.dbdriver == 'sqlite':
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   337
            self.cnx_wrap = SqliteConnectionWrapper
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   338
            self.dbhelper.dbname = abspath(self.dbhelper.dbname)
9724
e45bf9baa7b7 Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents: 9466
diff changeset
   339
        if not repairing:
e45bf9baa7b7 Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents: 9466
diff changeset
   340
            statement_timeout = int(source_config.get('db-statement-timeout', 0))
e45bf9baa7b7 Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents: 9466
diff changeset
   341
            if statement_timeout > 0:
e45bf9baa7b7 Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents: 9466
diff changeset
   342
                def set_postgres_timeout(cnx):
e45bf9baa7b7 Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents: 9466
diff changeset
   343
                    cnx.cursor().execute('SET statement_timeout to %d' % statement_timeout)
e45bf9baa7b7 Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents: 9466
diff changeset
   344
                    cnx.commit()
e45bf9baa7b7 Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents: 9466
diff changeset
   345
                postgres_hooks = SQL_CONNECT_HOOKS['postgres']
e45bf9baa7b7 Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents: 9466
diff changeset
   346
                postgres_hooks.append(set_postgres_timeout)
9466
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   347
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   348
    def wrapped_connection(self):
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   349
        """open and return a connection to the database, wrapped into a class
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   350
        handling reconnection and all
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   351
        """
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   352
        return self.cnx_wrap(self)
1619
e4845b54a704 force proper date/datetime according to type (necessary for sqlite at least)
sylvain.thenault@logilab.fr
parents: 1408
diff changeset
   353
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   354
    def get_connection(self):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   355
        """open and return a connection to the database"""
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   356
        return self.dbhelper.get_connection()
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   357
4893
15ae9a33a7f2 [db backup] fix name error in backup_to_file: we've to pass .confirm all along the chain as for restore
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4854
diff changeset
   358
    def backup_to_file(self, backupfile, confirm):
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   359
        for cmd in self.dbhelper.backup_commands(backupfile,
4342
b4e186da08f2 handle lgc.adbh api changes within regards of backup/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4322
diff changeset
   360
                                                 keepownership=False):
4353
7db69db4913c command may now officially be either a string or a list, don't make think it's for backward compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4342
diff changeset
   361
            if _run_command(cmd):
7db69db4913c command may now officially be either a string or a list, don't make think it's for backward compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4342
diff changeset
   362
                if not confirm('   [Failed] Continue anyway?', default='n'):
4342
b4e186da08f2 handle lgc.adbh api changes within regards of backup/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4322
diff changeset
   363
                    raise Exception('Failed command: %s' % cmd)
2493
9806571ea790 major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2396
diff changeset
   364
9806571ea790 major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2396
diff changeset
   365
    def restore_from_file(self, backupfile, confirm, drop=True):
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   366
        for cmd in self.dbhelper.restore_commands(backupfile,
4854
b06d2a3b27d9 logilab.db compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   367
                                                  keepownership=False,
2493
9806571ea790 major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2396
diff changeset
   368
                                                  drop=drop):
4353
7db69db4913c command may now officially be either a string or a list, don't make think it's for backward compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4342
diff changeset
   369
            if _run_command(cmd):
7db69db4913c command may now officially be either a string or a list, don't make think it's for backward compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4342
diff changeset
   370
                if not confirm('   [Failed] Continue anyway?', default='n'):
4195
86dcaf6bb92f closes #601987
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4179
diff changeset
   371
                    raise Exception('Failed command: %s' % cmd)
2493
9806571ea790 major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2396
diff changeset
   372
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   373
    def merge_args(self, args, query_args):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   374
        if args is not None:
4474
55fe19813bb7 kill mx compat code (dropped since 3.2), more efficient merge_args implementation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4466
diff changeset
   375
            newargs = {}
10662
10942ed172de [py3k] dict.iteritems → dict.items
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10651
diff changeset
   376
            for key, val in args.items():
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   377
                # convert cubicweb binary into db binary
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   378
                if isinstance(val, Binary):
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   379
                    val = self._binary(val.getvalue())
10643
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
   380
                # convert timestamp to utc.
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
   381
                # expect SET TiME ZONE to UTC at connection opening time.
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
   382
                # This shouldn't change anything for datetime without TZ.
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
   383
                elif isinstance(val, datetime) and val.tzinfo is not None:
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
   384
                    val = utcdatetime(val)
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
   385
                elif isinstance(val, time) and val.tzinfo is not None:
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
   386
                    val = utctime(val)
4474
55fe19813bb7 kill mx compat code (dropped since 3.2), more efficient merge_args implementation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4466
diff changeset
   387
                newargs[key] = val
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   388
            # should not collide
10643
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
   389
            assert not (frozenset(newargs) & frozenset(query_args)), \
cfded6d0da11 fix bad-caching of datetime with tz info at sql generation time
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10411
diff changeset
   390
                'unexpected collision: %s' % (frozenset(newargs) & frozenset(query_args))
4474
55fe19813bb7 kill mx compat code (dropped since 3.2), more efficient merge_args implementation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4466
diff changeset
   391
            newargs.update(query_args)
55fe19813bb7 kill mx compat code (dropped since 3.2), more efficient merge_args implementation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4466
diff changeset
   392
            return newargs
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   393
        return query_args
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   394
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9891
diff changeset
   395
    def process_result(self, cursor, cnx=None, column_callbacks=None):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   396
        """return a list of CubicWeb compliant values from data in the given cursor
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   397
        """
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9891
diff changeset
   398
        return list(self.iter_process_result(cursor, cnx, column_callbacks))
7341
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   399
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9891
diff changeset
   400
    def iter_process_result(self, cursor, cnx, column_callbacks=None):
7341
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   401
        """return a iterator on tuples of CubicWeb compliant values from data
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   402
        in the given cursor
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   403
        """
5013
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   404
        # use two different implementations to avoid paying the price of
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   405
        # callback lookup for each *cell* in results when there is nothing to
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   406
        # lookup
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   407
        if not column_callbacks:
8139
f9ebb6d1abc3 [server] use lgd.process_cursor to optimize processing of large resultsets
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 7907
diff changeset
   408
            return self.dbhelper.dbapi_module.process_cursor(cursor, self._dbencoding,
f9ebb6d1abc3 [server] use lgd.process_cursor to optimize processing of large resultsets
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 7907
diff changeset
   409
                                                             Binary)
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9891
diff changeset
   410
        assert cnx
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9891
diff changeset
   411
        return self._cb_process_result(cursor, column_callbacks, cnx)
5013
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   412
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9891
diff changeset
   413
    def _cb_process_result(self, cursor, column_callbacks, cnx):
5013
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   414
        # begin bind to locals for optimization
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   415
        descr = cursor.description
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   416
        encoding = self._dbencoding
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   417
        process_value = self._process_value
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   418
        binary = Binary
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   419
        # /end
7341
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   420
        cursor.arraysize = 100
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   421
        while True:
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   422
            results = cursor.fetchmany()
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   423
            if not results:
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   424
                break
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   425
            for line in results:
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   426
                result = []
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   427
                for col, value in enumerate(line):
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   428
                    if value is None:
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   429
                        result.append(value)
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   430
                        continue
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   431
                    cbstack = column_callbacks.get(col, None)
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   432
                    if cbstack is None:
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   433
                        value = process_value(value, descr[col], encoding, binary)
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   434
                    else:
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   435
                        for cb in cbstack:
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9891
diff changeset
   436
                            value = cb(self, cnx, value)
5013
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   437
                    result.append(value)
7341
c419c2d0d13e add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7229
diff changeset
   438
                yield result
5013
ad91f93bbb93 [source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4965
diff changeset
   439
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   440
    def preprocess_entity(self, entity):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   441
        """return a dictionary to use as extra argument to cursor.execute
1251
af40e615dc89 introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents: 0
diff changeset
   442
        to insert/update an entity into a SQL database
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   443
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   444
        attrs = {}
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   445
        eschema = entity.e_schema
8944
b167f039b6cb [sql] preprocess_entity uses lgdb helper's SQL converters.
Vincent Michel <vincent.michel@logilab.fr>
parents: 8755
diff changeset
   446
        converters = getattr(self.dbhelper, 'TYPE_CONVERTERS', {})
10662
10942ed172de [py3k] dict.iteritems → dict.items
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10651
diff changeset
   447
        for attr, value in entity.cw_edited.items():
6284
c35b2ebeb3c9 [repo/sql] don't do anything when value is None. This will avoid None to be turned into False in the case of a boolean for instance.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5768
diff changeset
   448
            if value is not None and eschema.subjrels[attr].final:
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   449
                atype = str(entity.e_schema.destination(attr))
8944
b167f039b6cb [sql] preprocess_entity uses lgdb helper's SQL converters.
Vincent Michel <vincent.michel@logilab.fr>
parents: 8755
diff changeset
   450
                if atype in converters:
b167f039b6cb [sql] preprocess_entity uses lgdb helper's SQL converters.
Vincent Michel <vincent.michel@logilab.fr>
parents: 8755
diff changeset
   451
                    # It is easier to modify preprocess_entity rather
b167f039b6cb [sql] preprocess_entity uses lgdb helper's SQL converters.
Vincent Michel <vincent.michel@logilab.fr>
parents: 8755
diff changeset
   452
                    # than add_entity (native) as this behavior
b167f039b6cb [sql] preprocess_entity uses lgdb helper's SQL converters.
Vincent Michel <vincent.michel@logilab.fr>
parents: 8755
diff changeset
   453
                    # may also be used for update.
b167f039b6cb [sql] preprocess_entity uses lgdb helper's SQL converters.
Vincent Michel <vincent.michel@logilab.fr>
parents: 8755
diff changeset
   454
                    value = converters[atype](value)
b167f039b6cb [sql] preprocess_entity uses lgdb helper's SQL converters.
Vincent Michel <vincent.michel@logilab.fr>
parents: 8755
diff changeset
   455
                elif atype == 'Password': # XXX could be done using a TYPE_CONVERTERS callback
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   456
                    # if value is a Binary instance, this mean we got it
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   457
                    # from a query result and so it is already encrypted
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   458
                    if isinstance(value, Binary):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   459
                        value = value.getvalue()
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   460
                    else:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   461
                        value = crypt_password(value)
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   462
                    value = self._binary(value)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   463
                elif isinstance(value, Binary):
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   464
                    value = self._binary(value.getvalue())
1251
af40e615dc89 introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents: 0
diff changeset
   465
            attrs[SQL_PREFIX+str(attr)] = value
4965
04543ed0bbdc [source] only consider edited_attributes in source.preprocess_entity()
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 4913
diff changeset
   466
        attrs[SQL_PREFIX+'eid'] = entity.eid
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   467
        return attrs
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   468
7083
b8e35cde46e9 help pylint by explicitely defining some attributes
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6379
diff changeset
   469
    # these are overridden by set_log_methods below
b8e35cde46e9 help pylint by explicitely defining some attributes
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6379
diff changeset
   470
    # only defining here to prevent pylint from complaining
b8e35cde46e9 help pylint by explicitely defining some attributes
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6379
diff changeset
   471
    info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   472
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   473
set_log_methods(SQLAdapterMixIn, getLogger('cubicweb.sqladapter'))
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   474
9447
0636c4960259 [multi-sources-removal] Drop cw.server.sources.extlite
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9364
diff changeset
   475
9466
c3a5f4507f12 [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9447
diff changeset
   476
# connection initialization functions ##########################################
9447
0636c4960259 [multi-sources-removal] Drop cw.server.sources.extlite
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9364
diff changeset
   477
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   478
def init_sqlite_connexion(cnx):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   479
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   480
    class group_concat(object):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   481
        def __init__(self):
9364
73bd5012336f Make the GROUP_CONCAT aggregate function not repeat values (closes #3223975)
Julien Cristau <julien.cristau@logilab.fr>
parents: 9340
diff changeset
   482
            self.values = set()
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   483
        def step(self, value):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   484
            if value is not None:
9364
73bd5012336f Make the GROUP_CONCAT aggregate function not repeat values (closes #3223975)
Julien Cristau <julien.cristau@logilab.fr>
parents: 9340
diff changeset
   485
                self.values.add(value)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   486
        def finalize(self):
10760
c34bbdb18745 [server] unicode → six.text_type
Julien Cristau <julien.cristau@logilab.fr>
parents: 10662
diff changeset
   487
            return ', '.join(text_type(v) for v in self.values)
9334
ea12401c0a68 [sqlutils] avoid a crash with sqlite when using group_concat (closes #3331906)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9267
diff changeset
   488
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   489
    cnx.create_aggregate("GROUP_CONCAT", 1, group_concat)
1619
e4845b54a704 force proper date/datetime according to type (necessary for sqlite at least)
sylvain.thenault@logilab.fr
parents: 1408
diff changeset
   490
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   491
    def _limit_size(text, maxsize, format='text/plain'):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   492
        if len(text) < maxsize:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   493
            return text
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   494
        if format in ('text/html', 'text/xhtml', 'text/xml'):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   495
            text = remove_html_tags(text)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   496
        if len(text) > maxsize:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   497
            text = text[:maxsize] + '...'
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   498
        return text
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   499
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   500
    def limit_size3(text, format, maxsize):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   501
        return _limit_size(text, maxsize, format)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   502
    cnx.create_function("LIMIT_SIZE", 3, limit_size3)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   503
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   504
    def limit_size2(text, maxsize):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   505
        return _limit_size(text, maxsize)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   506
    cnx.create_function("TEXT_LIMIT_SIZE", 2, limit_size2)
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   507
7904
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   508
    from logilab.common.date import strptime
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   509
    def weekday(ustr):
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   510
        try:
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   511
            dt = strptime(ustr, '%Y-%m-%d %H:%M:%S')
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   512
        except:
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   513
            dt =  strptime(ustr, '%Y-%m-%d')
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   514
        # expect sunday to be 1, saturday 7 while weekday method return 0 for
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   515
        # monday
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   516
        return (dt.weekday() + 1) % 7
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   517
    cnx.create_function("WEEKDAY", 1, weekday)
f41bb38dda7c [rql, sql] support for weekday function introduced in lgdp 1.7 (closes #1979717)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7341
diff changeset
   518
10205
737271ffe407 Ask sqlite to check foreign keys
Julien Cristau <julien.cristau@logilab.fr>
parents: 10200
diff changeset
   519
    cnx.cursor().execute("pragma foreign_keys = on")
737271ffe407 Ask sqlite to check foreign keys
Julien Cristau <julien.cristau@logilab.fr>
parents: 10200
diff changeset
   520
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   521
    import yams.constraints
4831
c5aec27c1bf7 [repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4719
diff changeset
   522
    yams.constraints.patch_sqlite_decimal()
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   523
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   524
sqlite_hooks = SQL_CONNECT_HOOKS.setdefault('sqlite', [])
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   525
sqlite_hooks.append(init_sqlite_connexion)
7166
dde161937d3e [time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7083
diff changeset
   526
dde161937d3e [time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7083
diff changeset
   527
dde161937d3e [time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7083
diff changeset
   528
def init_postgres_connexion(cnx):
dde161937d3e [time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7083
diff changeset
   529
    cnx.cursor().execute('SET TIME ZONE UTC')
7229
a60522259c2c [tz postgres support] we've to commit the connection once time-zone is set, else we may loose the setting
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7221
diff changeset
   530
    # commit is needed, else setting are lost if the connection is first
9267
24d9b86dfa54 spelling: rollbacked -> rolled back
Rémi Cardona <remi.cardona@logilab.fr>
parents: 8944
diff changeset
   531
    # rolled back
7229
a60522259c2c [tz postgres support] we've to commit the connection once time-zone is set, else we may loose the setting
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7221
diff changeset
   532
    cnx.commit()
7166
dde161937d3e [time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7083
diff changeset
   533
dde161937d3e [time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7083
diff changeset
   534
postgres_hooks = SQL_CONNECT_HOOKS.setdefault('postgres', [])
dde161937d3e [time zone] support for TZDatetime and TZTime data type
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7083
diff changeset
   535
postgres_hooks.append(init_postgres_connexion)