diff --git a/addons/source-python/packages/site-packages/pymysql/__init__.py b/addons/source-python/packages/site-packages/pymysql/__init__.py index d51392648..6ffb2ae60 100644 --- a/addons/source-python/packages/site-packages/pymysql/__init__.py +++ b/addons/source-python/packages/site-packages/pymysql/__init__.py @@ -26,14 +26,20 @@ from ._compat import PY2 from .constants import FIELD_TYPE from .converters import escape_dict, escape_sequence, escape_string -from .err import Warning, Error, InterfaceError, DataError, \ - DatabaseError, OperationalError, IntegrityError, InternalError, \ - NotSupportedError, ProgrammingError, MySQLError -from .times import Date, Time, Timestamp, \ - DateFromTicks, TimeFromTicks, TimestampFromTicks - - -VERSION = (0, 7, 5, None) +from .err import ( + Warning, Error, InterfaceError, DataError, + DatabaseError, OperationalError, IntegrityError, InternalError, + NotSupportedError, ProgrammingError, MySQLError) +from .times import ( + Date, Time, Timestamp, + DateFromTicks, TimeFromTicks, TimestampFromTicks) + + +VERSION = (0, 9, 3, None) +if VERSION[3] is not None: + VERSION_STRING = "%d.%d.%d_%s" % VERSION +else: + VERSION_STRING = "%d.%d.%d" % VERSION[:3] threadsafety = 1 apilevel = "2.0" paramstyle = "pyformat" @@ -87,19 +93,22 @@ def Connect(*args, **kwargs): from .connections import Connection return Connection(*args, **kwargs) -from pymysql import connections as _orig_conn +from . import connections as _orig_conn if _orig_conn.Connection.__init__.__doc__ is not None: Connect.__doc__ = _orig_conn.Connection.__init__.__doc__ del _orig_conn def get_client_info(): # for MySQLdb compatibility - return '.'.join(map(str, VERSION)) + version = VERSION + if VERSION[3] is None: + version = VERSION[:3] + return '.'.join(map(str, version)) connect = Connection = Connect # we include a doctored version_info here for MySQLdb compatibility -version_info = (1,2,6,"final",0) +version_info = (1, 3, 13, "final", 0) NULL = "NULL" @@ -111,7 +120,7 @@ def thread_safe(): def install_as_MySQLdb(): """ After this function is called, any application that imports MySQLdb or - _mysql will unwittingly actually use + _mysql will unwittingly actually use pymysql. """ sys.modules["MySQLdb"] = sys.modules["_mysql"] = sys.modules["pymysql"] diff --git a/addons/source-python/packages/site-packages/pymysql/_auth.py b/addons/source-python/packages/site-packages/pymysql/_auth.py new file mode 100644 index 000000000..a7fdaa48e --- /dev/null +++ b/addons/source-python/packages/site-packages/pymysql/_auth.py @@ -0,0 +1,264 @@ +""" +Implements auth methods +""" +from ._compat import PY2 +from .err import OperationalError +from .util import byte2int, int2byte + + +try: + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import serialization, hashes + from cryptography.hazmat.primitives.asymmetric import padding + _have_cryptography = True +except ImportError: + _have_cryptography = False + +from functools import partial +import hashlib +import io +import struct +import warnings + + +DEBUG = False +SCRAMBLE_LENGTH = 20 +sha1_new = partial(hashlib.new, 'sha1') + + +# mysql_native_password +# https://dev.mysql.com/doc/internals/en/secure-password-authentication.html#packet-Authentication::Native41 + + +def scramble_native_password(password, message): + """Scramble used for mysql_native_password""" + if not password: + return b'' + + stage1 = sha1_new(password).digest() + stage2 = sha1_new(stage1).digest() + s = sha1_new() + s.update(message[:SCRAMBLE_LENGTH]) + s.update(stage2) + result = s.digest() + return _my_crypt(result, stage1) + + +def _my_crypt(message1, message2): + result = bytearray(message1) + if PY2: + message2 = bytearray(message2) + + for i in range(len(result)): + result[i] ^= message2[i] + + return bytes(result) + + +# old_passwords support ported from libmysql/password.c +# https://dev.mysql.com/doc/internals/en/old-password-authentication.html + +SCRAMBLE_LENGTH_323 = 8 + + +class RandStruct_323(object): + + def __init__(self, seed1, seed2): + self.max_value = 0x3FFFFFFF + self.seed1 = seed1 % self.max_value + self.seed2 = seed2 % self.max_value + + def my_rnd(self): + self.seed1 = (self.seed1 * 3 + self.seed2) % self.max_value + self.seed2 = (self.seed1 + self.seed2 + 33) % self.max_value + return float(self.seed1) / float(self.max_value) + + +def scramble_old_password(password, message): + """Scramble for old_password""" + warnings.warn("old password (for MySQL <4.1) is used. Upgrade your password with newer auth method.\n" + "old password support will be removed in future PyMySQL version") + hash_pass = _hash_password_323(password) + hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) + hash_pass_n = struct.unpack(">LL", hash_pass) + hash_message_n = struct.unpack(">LL", hash_message) + + rand_st = RandStruct_323( + hash_pass_n[0] ^ hash_message_n[0], hash_pass_n[1] ^ hash_message_n[1] + ) + outbuf = io.BytesIO() + for _ in range(min(SCRAMBLE_LENGTH_323, len(message))): + outbuf.write(int2byte(int(rand_st.my_rnd() * 31) + 64)) + extra = int2byte(int(rand_st.my_rnd() * 31)) + out = outbuf.getvalue() + outbuf = io.BytesIO() + for c in out: + outbuf.write(int2byte(byte2int(c) ^ byte2int(extra))) + return outbuf.getvalue() + + +def _hash_password_323(password): + nr = 1345345333 + add = 7 + nr2 = 0x12345671 + + # x in py3 is numbers, p27 is chars + for c in [byte2int(x) for x in password if x not in (' ', '\t', 32, 9)]: + nr ^= (((nr & 63) + add) * c) + (nr << 8) & 0xFFFFFFFF + nr2 = (nr2 + ((nr2 << 8) ^ nr)) & 0xFFFFFFFF + add = (add + c) & 0xFFFFFFFF + + r1 = nr & ((1 << 31) - 1) # kill sign bits + r2 = nr2 & ((1 << 31) - 1) + return struct.pack(">LL", r1, r2) + + +# sha256_password + + +def _roundtrip(conn, send_data): + conn.write_packet(send_data) + pkt = conn._read_packet() + pkt.check_error() + return pkt + + +def _xor_password(password, salt): + password_bytes = bytearray(password) + salt = bytearray(salt) # for PY2 compat. + salt_len = len(salt) + for i in range(len(password_bytes)): + password_bytes[i] ^= salt[i % salt_len] + return bytes(password_bytes) + + +def sha2_rsa_encrypt(password, salt, public_key): + """Encrypt password with salt and public_key. + + Used for sha256_password and caching_sha2_password. + """ + if not _have_cryptography: + raise RuntimeError("'cryptography' package is required for sha256_password or caching_sha2_password auth methods") + message = _xor_password(password + b'\0', salt) + rsa_key = serialization.load_pem_public_key(public_key, default_backend()) + return rsa_key.encrypt( + message, + padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA1()), + algorithm=hashes.SHA1(), + label=None, + ), + ) + + +def sha256_password_auth(conn, pkt): + if conn._secure: + if DEBUG: + print("sha256: Sending plain password") + data = conn.password + b'\0' + return _roundtrip(conn, data) + + if pkt.is_auth_switch_request(): + conn.salt = pkt.read_all() + if not conn.server_public_key and conn.password: + # Request server public key + if DEBUG: + print("sha256: Requesting server public key") + pkt = _roundtrip(conn, b'\1') + + if pkt.is_extra_auth_data(): + conn.server_public_key = pkt._data[1:] + if DEBUG: + print("Received public key:\n", conn.server_public_key.decode('ascii')) + + if conn.password: + if not conn.server_public_key: + raise OperationalError("Couldn't receive server's public key") + + data = sha2_rsa_encrypt(conn.password, conn.salt, conn.server_public_key) + else: + data = b'' + + return _roundtrip(conn, data) + + +def scramble_caching_sha2(password, nonce): + # (bytes, bytes) -> bytes + """Scramble algorithm used in cached_sha2_password fast path. + + XOR(SHA256(password), SHA256(SHA256(SHA256(password)), nonce)) + """ + if not password: + return b'' + + p1 = hashlib.sha256(password).digest() + p2 = hashlib.sha256(p1).digest() + p3 = hashlib.sha256(p2 + nonce).digest() + + res = bytearray(p1) + if PY2: + p3 = bytearray(p3) + for i in range(len(p3)): + res[i] ^= p3[i] + + return bytes(res) + + +def caching_sha2_password_auth(conn, pkt): + # No password fast path + if not conn.password: + return _roundtrip(conn, b'') + + if pkt.is_auth_switch_request(): + # Try from fast auth + if DEBUG: + print("caching sha2: Trying fast path") + conn.salt = pkt.read_all() + scrambled = scramble_caching_sha2(conn.password, conn.salt) + pkt = _roundtrip(conn, scrambled) + # else: fast auth is tried in initial handshake + + if not pkt.is_extra_auth_data(): + raise OperationalError( + "caching sha2: Unknown packet for fast auth: %s" % pkt._data[:1] + ) + + # magic numbers: + # 2 - request public key + # 3 - fast auth succeeded + # 4 - need full auth + + pkt.advance(1) + n = pkt.read_uint8() + + if n == 3: + if DEBUG: + print("caching sha2: succeeded by fast path.") + pkt = conn._read_packet() + pkt.check_error() # pkt must be OK packet + return pkt + + if n != 4: + raise OperationalError("caching sha2: Unknwon result for fast auth: %s" % n) + + if DEBUG: + print("caching sha2: Trying full auth...") + + if conn._secure: + if DEBUG: + print("caching sha2: Sending plain password via secure connection") + return _roundtrip(conn, conn.password + b'\0') + + if not conn.server_public_key: + pkt = _roundtrip(conn, b'\x02') # Request public key + if not pkt.is_extra_auth_data(): + raise OperationalError( + "caching sha2: Unknown packet for public key: %s" % pkt._data[:1] + ) + + conn.server_public_key = pkt._data[1:] + if DEBUG: + print(conn.server_public_key.decode('ascii')) + + data = sha2_rsa_encrypt(conn.password, conn.salt, conn.server_public_key) + pkt = _roundtrip(conn, data) diff --git a/addons/source-python/packages/site-packages/pymysql/charset.py b/addons/source-python/packages/site-packages/pymysql/charset.py index 1cf7d9131..d3ced67ca 100644 --- a/addons/source-python/packages/site-packages/pymysql/charset.py +++ b/addons/source-python/packages/site-packages/pymysql/charset.py @@ -11,11 +11,21 @@ def __init__(self, id, name, collation, is_default): self.id, self.name, self.collation = id, name, collation self.is_default = is_default == 'Yes' + def __repr__(self): + return "Charset(id=%s, name=%r, collation=%r)" % ( + self.id, self.name, self.collation) + @property def encoding(self): name = self.name - if name == 'utf8mb4': + if name in ('utf8mb4', 'utf8mb3'): return 'utf8' + if name == 'latin1': + return 'cp1252' + if name == 'koi8r': + return 'koi8_r' + if name == 'koi8u': + return 'koi8_u' return name @property @@ -26,18 +36,18 @@ def is_binary(self): class Charsets: def __init__(self): self._by_id = {} + self._by_name = {} def add(self, c): self._by_id[c.id] = c + if c.is_default: + self._by_name[c.name] = c def by_id(self, id): return self._by_id[id] def by_name(self, name): - name = name.lower() - for c in self._by_id.values(): - if c.name == name and c.is_default: - return c + return self._by_name.get(name.lower()) _charsets = Charsets() """ @@ -85,7 +95,6 @@ def by_name(self, name): _charsets.add(Charset(32, 'armscii8', 'armscii8_general_ci', 'Yes')) _charsets.add(Charset(33, 'utf8', 'utf8_general_ci', 'Yes')) _charsets.add(Charset(34, 'cp1250', 'cp1250_czech_cs', '')) -_charsets.add(Charset(35, 'ucs2', 'ucs2_general_ci', 'Yes')) _charsets.add(Charset(36, 'cp866', 'cp866_general_ci', 'Yes')) _charsets.add(Charset(37, 'keybcs2', 'keybcs2_general_ci', 'Yes')) _charsets.add(Charset(38, 'macce', 'macce_general_ci', 'Yes')) @@ -104,13 +113,9 @@ def by_name(self, name): _charsets.add(Charset(51, 'cp1251', 'cp1251_general_ci', 'Yes')) _charsets.add(Charset(52, 'cp1251', 'cp1251_general_cs', '')) _charsets.add(Charset(53, 'macroman', 'macroman_bin', '')) -_charsets.add(Charset(54, 'utf16', 'utf16_general_ci', 'Yes')) -_charsets.add(Charset(55, 'utf16', 'utf16_bin', '')) _charsets.add(Charset(57, 'cp1256', 'cp1256_general_ci', 'Yes')) _charsets.add(Charset(58, 'cp1257', 'cp1257_bin', '')) _charsets.add(Charset(59, 'cp1257', 'cp1257_general_ci', 'Yes')) -_charsets.add(Charset(60, 'utf32', 'utf32_general_ci', 'Yes')) -_charsets.add(Charset(61, 'utf32', 'utf32_bin', '')) _charsets.add(Charset(63, 'binary', 'binary', 'Yes')) _charsets.add(Charset(64, 'armscii8', 'armscii8_bin', '')) _charsets.add(Charset(65, 'ascii', 'ascii_bin', '')) @@ -124,6 +129,7 @@ def by_name(self, name): _charsets.add(Charset(73, 'keybcs2', 'keybcs2_bin', '')) _charsets.add(Charset(74, 'koi8r', 'koi8r_bin', '')) _charsets.add(Charset(75, 'koi8u', 'koi8u_bin', '')) +_charsets.add(Charset(76, 'utf8', 'utf8_tolower_ci', '')) _charsets.add(Charset(77, 'latin2', 'latin2_bin', '')) _charsets.add(Charset(78, 'latin5', 'latin5_bin', '')) _charsets.add(Charset(79, 'latin7', 'latin7_bin', '')) @@ -137,7 +143,6 @@ def by_name(self, name): _charsets.add(Charset(87, 'gbk', 'gbk_bin', '')) _charsets.add(Charset(88, 'sjis', 'sjis_bin', '')) _charsets.add(Charset(89, 'tis620', 'tis620_bin', '')) -_charsets.add(Charset(90, 'ucs2', 'ucs2_bin', '')) _charsets.add(Charset(91, 'ujis', 'ujis_bin', '')) _charsets.add(Charset(92, 'geostd8', 'geostd8_general_ci', 'Yes')) _charsets.add(Charset(93, 'geostd8', 'geostd8_bin', '')) @@ -147,67 +152,6 @@ def by_name(self, name): _charsets.add(Charset(97, 'eucjpms', 'eucjpms_japanese_ci', 'Yes')) _charsets.add(Charset(98, 'eucjpms', 'eucjpms_bin', '')) _charsets.add(Charset(99, 'cp1250', 'cp1250_polish_ci', '')) -_charsets.add(Charset(101, 'utf16', 'utf16_unicode_ci', '')) -_charsets.add(Charset(102, 'utf16', 'utf16_icelandic_ci', '')) -_charsets.add(Charset(103, 'utf16', 'utf16_latvian_ci', '')) -_charsets.add(Charset(104, 'utf16', 'utf16_romanian_ci', '')) -_charsets.add(Charset(105, 'utf16', 'utf16_slovenian_ci', '')) -_charsets.add(Charset(106, 'utf16', 'utf16_polish_ci', '')) -_charsets.add(Charset(107, 'utf16', 'utf16_estonian_ci', '')) -_charsets.add(Charset(108, 'utf16', 'utf16_spanish_ci', '')) -_charsets.add(Charset(109, 'utf16', 'utf16_swedish_ci', '')) -_charsets.add(Charset(110, 'utf16', 'utf16_turkish_ci', '')) -_charsets.add(Charset(111, 'utf16', 'utf16_czech_ci', '')) -_charsets.add(Charset(112, 'utf16', 'utf16_danish_ci', '')) -_charsets.add(Charset(113, 'utf16', 'utf16_lithuanian_ci', '')) -_charsets.add(Charset(114, 'utf16', 'utf16_slovak_ci', '')) -_charsets.add(Charset(115, 'utf16', 'utf16_spanish2_ci', '')) -_charsets.add(Charset(116, 'utf16', 'utf16_roman_ci', '')) -_charsets.add(Charset(117, 'utf16', 'utf16_persian_ci', '')) -_charsets.add(Charset(118, 'utf16', 'utf16_esperanto_ci', '')) -_charsets.add(Charset(119, 'utf16', 'utf16_hungarian_ci', '')) -_charsets.add(Charset(120, 'utf16', 'utf16_sinhala_ci', '')) -_charsets.add(Charset(128, 'ucs2', 'ucs2_unicode_ci', '')) -_charsets.add(Charset(129, 'ucs2', 'ucs2_icelandic_ci', '')) -_charsets.add(Charset(130, 'ucs2', 'ucs2_latvian_ci', '')) -_charsets.add(Charset(131, 'ucs2', 'ucs2_romanian_ci', '')) -_charsets.add(Charset(132, 'ucs2', 'ucs2_slovenian_ci', '')) -_charsets.add(Charset(133, 'ucs2', 'ucs2_polish_ci', '')) -_charsets.add(Charset(134, 'ucs2', 'ucs2_estonian_ci', '')) -_charsets.add(Charset(135, 'ucs2', 'ucs2_spanish_ci', '')) -_charsets.add(Charset(136, 'ucs2', 'ucs2_swedish_ci', '')) -_charsets.add(Charset(137, 'ucs2', 'ucs2_turkish_ci', '')) -_charsets.add(Charset(138, 'ucs2', 'ucs2_czech_ci', '')) -_charsets.add(Charset(139, 'ucs2', 'ucs2_danish_ci', '')) -_charsets.add(Charset(140, 'ucs2', 'ucs2_lithuanian_ci', '')) -_charsets.add(Charset(141, 'ucs2', 'ucs2_slovak_ci', '')) -_charsets.add(Charset(142, 'ucs2', 'ucs2_spanish2_ci', '')) -_charsets.add(Charset(143, 'ucs2', 'ucs2_roman_ci', '')) -_charsets.add(Charset(144, 'ucs2', 'ucs2_persian_ci', '')) -_charsets.add(Charset(145, 'ucs2', 'ucs2_esperanto_ci', '')) -_charsets.add(Charset(146, 'ucs2', 'ucs2_hungarian_ci', '')) -_charsets.add(Charset(147, 'ucs2', 'ucs2_sinhala_ci', '')) -_charsets.add(Charset(159, 'ucs2', 'ucs2_general_mysql500_ci', '')) -_charsets.add(Charset(160, 'utf32', 'utf32_unicode_ci', '')) -_charsets.add(Charset(161, 'utf32', 'utf32_icelandic_ci', '')) -_charsets.add(Charset(162, 'utf32', 'utf32_latvian_ci', '')) -_charsets.add(Charset(163, 'utf32', 'utf32_romanian_ci', '')) -_charsets.add(Charset(164, 'utf32', 'utf32_slovenian_ci', '')) -_charsets.add(Charset(165, 'utf32', 'utf32_polish_ci', '')) -_charsets.add(Charset(166, 'utf32', 'utf32_estonian_ci', '')) -_charsets.add(Charset(167, 'utf32', 'utf32_spanish_ci', '')) -_charsets.add(Charset(168, 'utf32', 'utf32_swedish_ci', '')) -_charsets.add(Charset(169, 'utf32', 'utf32_turkish_ci', '')) -_charsets.add(Charset(170, 'utf32', 'utf32_czech_ci', '')) -_charsets.add(Charset(171, 'utf32', 'utf32_danish_ci', '')) -_charsets.add(Charset(172, 'utf32', 'utf32_lithuanian_ci', '')) -_charsets.add(Charset(173, 'utf32', 'utf32_slovak_ci', '')) -_charsets.add(Charset(174, 'utf32', 'utf32_spanish2_ci', '')) -_charsets.add(Charset(175, 'utf32', 'utf32_roman_ci', '')) -_charsets.add(Charset(176, 'utf32', 'utf32_persian_ci', '')) -_charsets.add(Charset(177, 'utf32', 'utf32_esperanto_ci', '')) -_charsets.add(Charset(178, 'utf32', 'utf32_hungarian_ci', '')) -_charsets.add(Charset(179, 'utf32', 'utf32_sinhala_ci', '')) _charsets.add(Charset(192, 'utf8', 'utf8_unicode_ci', '')) _charsets.add(Charset(193, 'utf8', 'utf8_icelandic_ci', '')) _charsets.add(Charset(194, 'utf8', 'utf8_latvian_ci', '')) @@ -228,6 +172,10 @@ def by_name(self, name): _charsets.add(Charset(209, 'utf8', 'utf8_esperanto_ci', '')) _charsets.add(Charset(210, 'utf8', 'utf8_hungarian_ci', '')) _charsets.add(Charset(211, 'utf8', 'utf8_sinhala_ci', '')) +_charsets.add(Charset(212, 'utf8', 'utf8_german2_ci', '')) +_charsets.add(Charset(213, 'utf8', 'utf8_croatian_ci', '')) +_charsets.add(Charset(214, 'utf8', 'utf8_unicode_520_ci', '')) +_charsets.add(Charset(215, 'utf8', 'utf8_vietnamese_ci', '')) _charsets.add(Charset(223, 'utf8', 'utf8_general_mysql500_ci', '')) _charsets.add(Charset(224, 'utf8mb4', 'utf8mb4_unicode_ci', '')) _charsets.add(Charset(225, 'utf8mb4', 'utf8mb4_icelandic_ci', '')) @@ -249,14 +197,14 @@ def by_name(self, name): _charsets.add(Charset(241, 'utf8mb4', 'utf8mb4_esperanto_ci', '')) _charsets.add(Charset(242, 'utf8mb4', 'utf8mb4_hungarian_ci', '')) _charsets.add(Charset(243, 'utf8mb4', 'utf8mb4_sinhala_ci', '')) - +_charsets.add(Charset(244, 'utf8mb4', 'utf8mb4_german2_ci', '')) +_charsets.add(Charset(245, 'utf8mb4', 'utf8mb4_croatian_ci', '')) +_charsets.add(Charset(246, 'utf8mb4', 'utf8mb4_unicode_520_ci', '')) +_charsets.add(Charset(247, 'utf8mb4', 'utf8mb4_vietnamese_ci', '')) +_charsets.add(Charset(248, 'gb18030', 'gb18030_chinese_ci', 'Yes')) +_charsets.add(Charset(249, 'gb18030', 'gb18030_bin', '')) +_charsets.add(Charset(250, 'gb18030', 'gb18030_unicode_520_ci', '')) +_charsets.add(Charset(255, 'utf8mb4', 'utf8mb4_0900_ai_ci', '')) charset_by_name = _charsets.by_name charset_by_id = _charsets.by_id - - -def charset_to_encoding(name): - """Convert MySQL's charset name to Python's codec name""" - if name == 'utf8mb4': - return 'utf8' - return name diff --git a/addons/source-python/packages/site-packages/pymysql/connections.py b/addons/source-python/packages/site-packages/pymysql/connections.py index 52f13f05f..a1cd8c25e 100644 --- a/addons/source-python/packages/site-packages/pymysql/connections.py +++ b/addons/source-python/packages/site-packages/pymysql/connections.py @@ -1,13 +1,11 @@ # Python implementation of the MySQL client-server protocol # http://dev.mysql.com/doc/internals/en/client-server-protocol.html # Error codes: -# http://dev.mysql.com/doc/refman/5.5/en/error-messages-client.html +# https://dev.mysql.com/doc/refman/5.5/en/error-handling.html from __future__ import print_function from ._compat import PY2, range_type, text_type, str_type, JYTHON, IRONPYTHON import errno -from functools import partial -import hashlib import io import os import socket @@ -16,13 +14,19 @@ import traceback import warnings -from .charset import MBLENGTH, charset_by_name, charset_by_id -from .constants import CLIENT, COMMAND, FIELD_TYPE, SERVER_STATUS -from .converters import escape_item, escape_string, through, conversions as _conv +from . import _auth + +from .charset import charset_by_name, charset_by_id +from .constants import CLIENT, COMMAND, CR, FIELD_TYPE, SERVER_STATUS +from . import converters from .cursors import Cursor from .optionfile import Parser +from .protocol import ( + dump_packet, MysqlPacket, FieldDescriptorPacket, OKPacketWrapper, + EOFPacketWrapper, LoadLocalPacketWrapper +) from .util import byte2int, int2byte -from . import err +from . import err, VERSION_STRING try: import ssl @@ -39,178 +43,60 @@ # KeyError occurs when there's no entry in OS database for a current user. DEFAULT_USER = None - DEBUG = False _py_version = sys.version_info[:2] +if PY2: + pass +elif _py_version < (3, 6): + # See http://bugs.python.org/issue24870 + _surrogateescape_table = [chr(i) if i < 0x80 else chr(i + 0xdc00) for i in range(256)] + + def _fast_surrogateescape(s): + return s.decode('latin1').translate(_surrogateescape_table) +else: + def _fast_surrogateescape(s): + return s.decode('ascii', 'surrogateescape') # socket.makefile() in Python 2 is not usable because very inefficient and # bad behavior about timeout. # XXX: ._socketio doesn't work under IronPython. -if _py_version == (2, 7) and not IRONPYTHON: +if PY2 and not IRONPYTHON: # read method of file-like returned by sock.makefile() is very slow. # So we copy io-based one from Python 3. from ._socketio import SocketIO def _makefile(sock, mode): return io.BufferedReader(SocketIO(sock, mode)) -elif _py_version == (2, 6): - # Python 2.6 doesn't have fast io module. - # So we make original one. - class SockFile(object): - def __init__(self, sock): - self._sock = sock - - def read(self, n): - read = self._sock.recv(n) - if len(read) == n: - return read - while True: - data = self._sock.recv(n-len(read)) - if not data: - return read - read += data - if len(read) == n: - return read - - def _makefile(sock, mode): - assert mode == 'rb' - return SockFile(sock) else: # socket.makefile in Python 3 is nice. def _makefile(sock, mode): return sock.makefile(mode) -TEXT_TYPES = set([ +TEXT_TYPES = { FIELD_TYPE.BIT, FIELD_TYPE.BLOB, FIELD_TYPE.LONG_BLOB, FIELD_TYPE.MEDIUM_BLOB, - FIELD_TYPE.JSON, FIELD_TYPE.STRING, FIELD_TYPE.TINY_BLOB, FIELD_TYPE.VAR_STRING, FIELD_TYPE.VARCHAR, - FIELD_TYPE.GEOMETRY]) - -sha_new = partial(hashlib.new, 'sha1') + FIELD_TYPE.GEOMETRY, +} -NULL_COLUMN = 251 -UNSIGNED_CHAR_COLUMN = 251 -UNSIGNED_SHORT_COLUMN = 252 -UNSIGNED_INT24_COLUMN = 253 -UNSIGNED_INT64_COLUMN = 254 -DEFAULT_CHARSET = 'latin1' +DEFAULT_CHARSET = 'utf8mb4' MAX_PACKET_LEN = 2**24-1 -def dump_packet(data): # pragma: no cover - def is_ascii(data): - if 65 <= byte2int(data) <= 122: - if isinstance(data, int): - return chr(data) - return data - return '.' - - try: - print("packet length:", len(data)) - print("method call[1]:", sys._getframe(1).f_code.co_name) - print("method call[2]:", sys._getframe(2).f_code.co_name) - print("method call[3]:", sys._getframe(3).f_code.co_name) - print("method call[4]:", sys._getframe(4).f_code.co_name) - print("method call[5]:", sys._getframe(5).f_code.co_name) - print("-" * 88) - except ValueError: - pass - dump_data = [data[i:i+16] for i in range_type(0, min(len(data), 256), 16)] - for d in dump_data: - print(' '.join(map(lambda x: "{:02X}".format(byte2int(x)), d)) + - ' ' * (16 - len(d)) + ' ' * 2 + - ' '.join(map(lambda x: "{}".format(is_ascii(x)), d))) - print("-" * 88) - print() - - -def _scramble(password, message): - if not password: - return b'' - if DEBUG: print('password=' + str(password)) - stage1 = sha_new(password).digest() - stage2 = sha_new(stage1).digest() - s = sha_new() - s.update(message) - s.update(stage2) - result = s.digest() - return _my_crypt(result, stage1) - - -def _my_crypt(message1, message2): - length = len(message1) - result = b'' - for i in range_type(length): - x = (struct.unpack('B', message1[i:i+1])[0] ^ - struct.unpack('B', message2[i:i+1])[0]) - result += struct.pack('B', x) - return result - -# old_passwords support ported from libmysql/password.c -SCRAMBLE_LENGTH_323 = 8 - - -class RandStruct_323(object): - def __init__(self, seed1, seed2): - self.max_value = 0x3FFFFFFF - self.seed1 = seed1 % self.max_value - self.seed2 = seed2 % self.max_value - - def my_rnd(self): - self.seed1 = (self.seed1 * 3 + self.seed2) % self.max_value - self.seed2 = (self.seed1 + self.seed2 + 33) % self.max_value - return float(self.seed1) / float(self.max_value) - - -def _scramble_323(password, message): - hash_pass = _hash_password_323(password) - hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) - hash_pass_n = struct.unpack(">LL", hash_pass) - hash_message_n = struct.unpack(">LL", hash_message) - - rand_st = RandStruct_323(hash_pass_n[0] ^ hash_message_n[0], - hash_pass_n[1] ^ hash_message_n[1]) - outbuf = io.BytesIO() - for _ in range_type(min(SCRAMBLE_LENGTH_323, len(message))): - outbuf.write(int2byte(int(rand_st.my_rnd() * 31) + 64)) - extra = int2byte(int(rand_st.my_rnd() * 31)) - out = outbuf.getvalue() - outbuf = io.BytesIO() - for c in out: - outbuf.write(int2byte(byte2int(c) ^ byte2int(extra))) - return outbuf.getvalue() - - -def _hash_password_323(password): - nr = 1345345333 - add = 7 - nr2 = 0x12345671 - - # x in py3 is numbers, p27 is chars - for c in [byte2int(x) for x in password if x not in (' ', '\t', 32, 9)]: - nr ^= (((nr & 63) + add) * c) + (nr << 8) & 0xFFFFFFFF - nr2 = (nr2 + ((nr2 << 8) ^ nr)) & 0xFFFFFFFF - add = (add + c) & 0xFFFFFFFF - - r1 = nr & ((1 << 31) - 1) # kill sign bits - r2 = nr2 & ((1 << 31) - 1) - return struct.pack(">LL", r1, r2) - - def pack_int24(n): return struct.pack(' len(self._data): - raise Exception('Invalid advance amount (%s) for cursor. ' - 'Position=%s' % (length, new_position)) - self._position = new_position - - def rewind(self, position=0): - """Set the position of the data buffer cursor to 'position'.""" - if position < 0 or position > len(self._data): - raise Exception("Invalid position to rewind cursor to: %s." % position) - self._position = position - - def get_bytes(self, position, length=1): - """Get 'length' bytes starting at 'position'. - - Position is start of payload (first four packet header bytes are not - included) starting at index '0'. - - No error checking is done. If requesting outside end of buffer - an empty string (or string shorter than 'length') may be returned! - """ - return self._data[position:(position+length)] - - if PY2: - def read_uint8(self): - result = ord(self._data[self._position]) - self._position += 1 - return result - else: - def read_uint8(self): - result = self._data[self._position] - self._position += 1 - return result - - def read_uint16(self): - result = struct.unpack_from('`_ in the + specification. """ _sock = None _auth_plugin_name = '' + _closed = False + _secure = False def __init__(self, host=None, user=None, password="", database=None, port=0, unix_socket=None, charset='', sql_mode=None, read_default_file=None, conv=None, use_unicode=None, client_flag=0, cursorclass=Cursor, init_command=None, - connect_timeout=None, ssl=None, read_default_group=None, - compress=None, named_pipe=None, no_delay=None, + connect_timeout=10, ssl=None, read_default_group=None, + compress=None, named_pipe=None, autocommit=False, db=None, passwd=None, local_infile=False, max_allowed_packet=16*1024*1024, defer_connect=False, - auth_plugin_map={}, read_timeout=None, write_timeout=None): - """ - Establish a connection to the MySQL database. Accepts several - arguments: - - host: Host where the database server is located - user: Username to log in as - password: Password to use. - database: Database to use, None to not use a particular one. - port: MySQL port to use, default is usually OK. (default: 3306) - unix_socket: Optionally, you can use a unix socket rather than TCP/IP. - charset: Charset you want to use. - sql_mode: Default SQL_MODE to use. - read_default_file: - Specifies my.cnf file to read these parameters from under the [client] section. - conv: - Conversion dictionary to use instead of the default one. - This is used to provide custom marshalling and unmarshaling of types. - See converters. - use_unicode: - Whether or not to default to unicode strings. - This option defaults to true for Py3k. - client_flag: Custom flags to send to MySQL. Find potential values in constants.CLIENT. - cursorclass: Custom cursor class to use. - init_command: Initial SQL statement to run when connection is established. - connect_timeout: Timeout before throwing an exception when connecting. - ssl: - A dict of arguments similar to mysql_ssl_set()'s parameters. - For now the capath and cipher arguments are not supported. - read_default_group: Group to read from in the configuration file. - compress; Not supported - named_pipe: Not supported - autocommit: Autocommit mode. None means use server default. (default: False) - local_infile: Boolean to enable the use of LOAD DATA LOCAL command. (default: False) - max_allowed_packet: Max size of packet sent to server in bytes. (default: 16MB) - Only used to limit size of "LOAD LOCAL INFILE" data packet smaller than default (16KB). - defer_connect: Don't explicitly connect on contruction - wait for connect call. - (default: False) - auth_plugin_map: A dict of plugin names to a class that processes that plugin. - The class will take the Connection object as the argument to the constructor. - The class needs an authenticate method taking an authentication packet as - an argument. For the dialog plugin, a prompt(echo, prompt) method can be used - (if no authenticate method) for returning a string from the user. (experimental) - db: Alias for database. (for compatibility to MySQLdb) - passwd: Alias for password. (for compatibility to MySQLdb) - """ - if no_delay is not None: - warnings.warn("no_delay option is deprecated", DeprecationWarning) - + auth_plugin_map=None, read_timeout=None, write_timeout=None, + bind_address=None, binary_prefix=False, program_name=None, + server_public_key=None): if use_unicode is None and sys.version_info[0] > 2: use_unicode = True @@ -596,17 +203,10 @@ def __init__(self, host=None, user=None, password="", if compress or named_pipe: raise NotImplementedError("compress and named_pipe arguments are not supported") - if local_infile: + self._local_infile = bool(local_infile) + if self._local_infile: client_flag |= CLIENT.LOCAL_FILES - self.ssl = False - if ssl: - if not SSL_ENABLED: - raise NotImplementedError("ssl module not found") - self.ssl = True - client_flag |= CLIENT.SSL - self.ctx = self._create_ssl_ctx(ssl) - if read_default_group and not read_default_file: if sys.platform.startswith("win"): read_default_file = "c:\\my.ini" @@ -634,19 +234,43 @@ def _config(key, arg): database = _config("database", database) unix_socket = _config("socket", unix_socket) port = int(_config("port", port)) + bind_address = _config("bind-address", bind_address) charset = _config("default-character-set", charset) + if not ssl: + ssl = {} + if isinstance(ssl, dict): + for key in ["ca", "capath", "cert", "key", "cipher"]: + value = _config("ssl-" + key, ssl.get(key)) + if value: + ssl[key] = value + + self.ssl = False + if ssl: + if not SSL_ENABLED: + raise NotImplementedError("ssl module not found") + self.ssl = True + client_flag |= CLIENT.SSL + self.ctx = self._create_ssl_ctx(ssl) self.host = host or "localhost" self.port = port or 3306 + if type(self.port) is not int: + raise ValueError("port should be of type int") self.user = user or DEFAULT_USER - self.password = password or "" + self.password = password or b"" + if isinstance(self.password, text_type): + self.password = self.password.encode('latin1') self.db = database self.unix_socket = unix_socket + self.bind_address = bind_address + if not (0 < connect_timeout <= 31536000): + raise ValueError("connect_timeout should be >0 and <=31536000") + self.connect_timeout = connect_timeout or None if read_timeout is not None and read_timeout <= 0: - raise ValueError("read_timeout should be >= 0") + raise ValueError("read_timeout should be > 0") self._read_timeout = read_timeout if write_timeout is not None and write_timeout <= 0: - raise ValueError("write_timeout should be >= 0") + raise ValueError("write_timeout should be > 0") self._write_timeout = write_timeout if charset: self.charset = charset @@ -660,30 +284,43 @@ def _config(key, arg): self.encoding = charset_by_name(self.charset).encoding - client_flag |= CLIENT.CAPABILITIES | CLIENT.MULTI_STATEMENTS + client_flag |= CLIENT.CAPABILITIES if self.db: client_flag |= CLIENT.CONNECT_WITH_DB + self.client_flag = client_flag self.cursorclass = cursorclass - self.connect_timeout = connect_timeout self._result = None self._affected_rows = 0 self.host_info = "Not connected" - #: specified autocommit mode. None means use server default. + # specified autocommit mode. None means use server default. self.autocommit_mode = autocommit if conv is None: - conv = _conv + conv = converters.conversions + # Need for MySQLdb compatibility. - self.encoders = dict([(k, v) for (k, v) in conv.items() if type(k) is not int]) - self.decoders = dict([(k, v) for (k, v) in conv.items() if type(k) is int]) + self.encoders = {k: v for (k, v) in conv.items() if type(k) is not int} + self.decoders = {k: v for (k, v) in conv.items() if type(k) is int} self.sql_mode = sql_mode self.init_command = init_command self.max_allowed_packet = max_allowed_packet - self._auth_plugin_map = auth_plugin_map + self._auth_plugin_map = auth_plugin_map or {} + self._binary_prefix = binary_prefix + self.server_public_key = server_public_key + + self._connect_attrs = { + '_client_name': 'pymysql', + '_pid': str(os.getpid()), + '_client_version': VERSION_STRING, + } + + if program_name: + self._connect_attrs["program_name"] = program_name + if defer_connect: self._sock = None else: @@ -707,33 +344,44 @@ def _create_ssl_ctx(self, sslp): return ctx def close(self): - """Send the quit message and close the socket""" - if self._sock is None: + """ + Send the quit message and close the socket. + + See `Connection.close() `_ + in the specification. + + :raise Error: If the connection is already closed. + """ + if self._closed: raise err.Error("Already closed") + self._closed = True + if self._sock is None: + return send_data = struct.pack('`_ + in the specification. + """ self._execute_command(COMMAND.COM_QUERY, "COMMIT") self._read_ok_packet() def rollback(self): - """Roll back the current transaction""" + """ + Roll back the current transaction. + + See `Connection.rollback() `_ + in the specification. + """ self._execute_command(COMMAND.COM_QUERY, "ROLLBACK") self._read_ok_packet() def show_warnings(self): - """SHOW WARNINGS""" + """Send the "SHOW WARNINGS" SQL command.""" self._execute_command(COMMAND.COM_QUERY, "SHOW WARNINGS") result = MySQLResult(self) result.read() return result.rows def select_db(self, db): - """Set current db""" + """ + Set current db. + + :param db: The name of the db. + """ self._execute_command(COMMAND.COM_INIT_DB, db) self._read_ok_packet() def escape(self, obj, mapping=None): """Escape whatever value you pass to it. - + Non-standard, for internal use; do not use this in your applications. """ if isinstance(obj, str_type): return "'" + self.escape_string(obj) + "'" - return escape_item(obj, self.charset, mapping=mapping) + if isinstance(obj, (bytes, bytearray)): + ret = self._quote_bytes(obj) + if self._binary_prefix: + ret = "_binary" + ret + return ret + return converters.escape_item(obj, self.charset, mapping=mapping) def literal(self, obj): """Alias for escape() - + Non-standard, for internal use; do not use this in your applications. """ return self.escape(obj, self.encoders) @@ -805,25 +472,26 @@ def escape_string(self, s): if (self.server_status & SERVER_STATUS.SERVER_STATUS_NO_BACKSLASH_ESCAPES): return s.replace("'", "''") - return escape_string(s) + return converters.escape_string(s) + + def _quote_bytes(self, s): + if (self.server_status & + SERVER_STATUS.SERVER_STATUS_NO_BACKSLASH_ESCAPES): + return "'%s'" % (_fast_surrogateescape(s.replace(b"'", b"''")),) + return converters.escape_bytes(s) def cursor(self, cursor=None): - """Create a new cursor to execute queries with""" + """ + Create a new cursor to execute queries with. + + :param cursor: The type of cursor to create; one of :py:class:`Cursor`, + :py:class:`SSCursor`, :py:class:`DictCursor`, or :py:class:`SSDictCursor`. + None means use Cursor. + """ if cursor: return cursor(self) return self.cursorclass(self) - def __enter__(self): - """Context manager that returns a Cursor""" - return self.cursor() - - def __exit__(self, exc, value, traceback): - """On successful exit, commit. On exception, rollback""" - if exc: - self.rollback() - else: - self.commit() - # The following methods are INTERNAL USE ONLY (called from Cursor) def query(self, sql, unbuffered=False): # if DEBUG: @@ -850,7 +518,12 @@ def kill(self, thread_id): return self._read_ok_packet() def ping(self, reconnect=True): - """Check if the server is alive""" + """ + Check if the server is alive. + + :param reconnect: If the connection is closed, reconnect. + :raise Error: If the connection is closed and reconnect=False. + """ if self._sock is None: if reconnect: self.connect() @@ -859,11 +532,11 @@ def ping(self, reconnect=True): raise err.Error("Already closed") try: self._execute_command(COMMAND.COM_PING, "") - return self._read_ok_packet() + self._read_ok_packet() except Exception: if reconnect: self.connect() - return self.ping(False) + self.ping(False) else: raise @@ -877,19 +550,25 @@ def set_charset(self, charset): self.encoding = encoding def connect(self, sock=None): + self._closed = False try: if sock is None: - if self.unix_socket and self.host in ('localhost', '127.0.0.1'): + if self.unix_socket: sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.settimeout(self.connect_timeout) sock.connect(self.unix_socket) self.host_info = "Localhost via UNIX socket" + self._secure = True if DEBUG: print('connected using unix_socket') else: + kwargs = {} + if self.bind_address is not None: + kwargs['source_address'] = (self.bind_address, 0) while True: try: sock = socket.create_connection( - (self.host, self.port), self.connect_timeout) + (self.host, self.port), self.connect_timeout, + **kwargs) break except (OSError, IOError) as e: if e.errno == errno.EINTR: @@ -898,8 +577,9 @@ def connect(self, sock=None): self.host_info = "socket %s:%d" % (self.host, self.port) if DEBUG: print('connected using socket') sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.settimeout(None) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + self._sock = sock self._rfile = _makefile(sock, 'rb') self._next_seq_id = 0 @@ -924,7 +604,7 @@ def connect(self, sock=None): if sock is not None: try: sock.close() - except: + except: # noqa pass if isinstance(e, (OSError, IOError, socket.error)): @@ -945,9 +625,9 @@ def connect(self, sock=None): def write_packet(self, payload): """Writes an entire "mysql packet" in its entirety to the network - addings its length and sequence number. + adding its length and sequence number. """ - # Internal note: when you build packet manualy and calls _write_bytes() + # Internal note: when you build packet manually and calls _write_bytes() # directly, you should set self._next_seq_id properly. data = pack_int24(len(payload)) + int2byte(self._next_seq_id) + payload if DEBUG: dump_packet(data) @@ -957,17 +637,27 @@ def write_packet(self, payload): def _read_packet(self, packet_type=MysqlPacket): """Read an entire "mysql packet" in its entirety from the network and return a MysqlPacket type that represents the results. + + :raise OperationalError: If the connection to the MySQL server is lost. + :raise InternalError: If the packet sequence number is wrong. """ - buff = b'' + buff = bytearray() while True: packet_header = self._read_bytes(4) - if DEBUG: dump_packet(packet_header) + #if DEBUG: dump_packet(packet_header) btrl, btrh, packet_number = struct.unpack('= i + 6: lang, stat, cap_h, salt_len = struct.unpack('= 4.1 returns TIMESTAMP in the same format as DATETIME: - - >>> mysql_timestamp_converter('2007-02-25 22:32:17') - datetime.datetime(2007, 2, 25, 22, 32, 17) - - MySQL < 4.1 uses a big string of numbers: - - >>> mysql_timestamp_converter('20070225223217') - datetime.datetime(2007, 2, 25, 22, 32, 17) - - Illegal values are returned as None: - - >>> mysql_timestamp_converter('2007-02-31 22:32:17') is None - True - >>> mysql_timestamp_converter('00000000000000') is None - True - - """ - if not PY2 and isinstance(timestamp, (bytes, bytearray)): - timestamp = timestamp.decode('ascii') - if timestamp[4] == '-': - return convert_datetime(timestamp) - timestamp += "0"*(14-len(timestamp)) # padding - year, month, day, hour, minute, second = \ - int(timestamp[:4]), int(timestamp[4:6]), int(timestamp[6:8]), \ - int(timestamp[8:10]), int(timestamp[10:12]), int(timestamp[12:14]) - try: - return datetime.datetime(year, month, day, hour, minute, second) - except ValueError: - return None - -def convert_set(s): - if isinstance(s, (bytes, bytearray)): - return set(s.split(b",")) - return set(s.split(",")) + return obj def through(x): @@ -324,27 +317,12 @@ def through(x): #def convert_bit(b): # b = "\x00" * (8 - len(b)) + b # pad w/ zeroes # return struct.unpack(">Q", b)[0] -# +# # the snippet above is right, but MySQLdb doesn't process bits, # so we shouldn't either convert_bit = through -def convert_characters(connection, field, data): - field_charset = charset_by_id(field.charsetnr).name - encoding = charset_to_encoding(field_charset) - if field.flags & FLAG.SET: - return convert_set(data.decode(encoding)) - if field.flags & FLAG.BINARY: - return data - - if connection.use_unicode: - data = data.decode(encoding) - elif connection.charset != field_charset: - data = data.decode(encoding) - data = data.encode(connection.encoding) - return data - encoders = { bool: escape_bool, int: escape_int, @@ -357,14 +335,13 @@ def convert_characters(connection, field, data): set: escape_sequence, frozenset: escape_sequence, dict: escape_dict, - bytearray: escape_bytes, type(None): escape_None, datetime.date: escape_date, datetime.datetime: escape_datetime, datetime.timedelta: escape_timedelta, datetime.time: escape_time, time.struct_time: escape_struct_time, - Decimal: escape_object, + Decimal: Decimal2Literal, } if not PY2 or JYTHON or IRONPYTHON: @@ -380,11 +357,10 @@ def convert_characters(connection, field, data): FIELD_TYPE.LONGLONG: int, FIELD_TYPE.INT24: int, FIELD_TYPE.YEAR: int, - FIELD_TYPE.TIMESTAMP: convert_mysql_timestamp, + FIELD_TYPE.TIMESTAMP: convert_datetime, FIELD_TYPE.DATETIME: convert_datetime, FIELD_TYPE.TIME: convert_timedelta, FIELD_TYPE.DATE: convert_date, - FIELD_TYPE.SET: convert_set, FIELD_TYPE.BLOB: through, FIELD_TYPE.TINY_BLOB: through, FIELD_TYPE.MEDIUM_BLOB: through, diff --git a/addons/source-python/packages/site-packages/pymysql/cursors.py b/addons/source-python/packages/site-packages/pymysql/cursors.py index 0aae4ef5f..033b5e7ff 100644 --- a/addons/source-python/packages/site-packages/pymysql/cursors.py +++ b/addons/source-python/packages/site-packages/pymysql/cursors.py @@ -2,39 +2,39 @@ from __future__ import print_function, absolute_import from functools import partial import re -import warnings from ._compat import range_type, text_type, PY2 - from . import err #: Regular expression for :meth:`Cursor.executemany`. -#: executemany only suports simple bulk insert. +#: executemany only supports simple bulk insert. #: You can use it to load large dataset. RE_INSERT_VALUES = re.compile( - r"\s*((?:INSERT|REPLACE)\s.+\sVALUES?\s+)" + + r"\s*((?:INSERT|REPLACE)\b.+\bVALUES?\s*)" + r"(\(\s*(?:%s|%\(.+\)s)\s*(?:,\s*(?:%s|%\(.+\)s)\s*)*\))" + - r"(\s*(?:ON DUPLICATE.*)?)\Z", + r"(\s*(?:ON DUPLICATE.*)?);?\s*\Z", re.IGNORECASE | re.DOTALL) class Cursor(object): - ''' + """ This is the object you use to interact with the database. - ''' - #: Max stetement size which :meth:`executemany` generates. + Do not create an instance of a Cursor yourself. Call + connections.Connection.cursor(). + + See `Cursor `_ in + the specification. + """ + + #: Max statement size which :meth:`executemany` generates. #: #: Max size of allowed statement is max_allowed_packet - packet_header_size. #: Default value of max_allowed_packet is 1048576. max_stmt_length = 1024000 def __init__(self, connection): - ''' - Do not create an instance of a Cursor yourself. Call - connections.Connection.cursor(). - ''' self.connection = connection self.description = None self.rownumber = 0 @@ -45,9 +45,9 @@ def __init__(self, connection): self._rows = None def close(self): - ''' + """ Closing a cursor just exhausts all remaining data. - ''' + """ conn = self.connection if conn is None: return @@ -90,6 +90,8 @@ def _nextset(self, unbuffered=False): return None if not current_result.has_next: return None + self._result = None + self._clear_result() conn.next_result(unbuffered=unbuffered) self._do_get_result() return True @@ -110,12 +112,12 @@ def _escape_args(self, args, conn): if isinstance(args, (tuple, list)): if PY2: args = tuple(map(ensure_bytes, args)) - return tuple(conn.escape(arg) for arg in args) + return tuple(conn.literal(arg) for arg in args) elif isinstance(args, dict): if PY2: - args = dict((ensure_bytes(key), ensure_bytes(val)) for - (key, val) in args.items()) - return dict((key, conn.escape(val)) for (key, val) in args.items()) + args = {ensure_bytes(key): ensure_bytes(val) for + (key, val) in args.items()} + return {key: conn.literal(val) for (key, val) in args.items()} else: # If it's not a dictionary let's try escaping it anyways. # Worst case it will throw a Value error @@ -255,9 +257,10 @@ def callproc(self, procname, args=()): disconnected. """ conn = self._get_db() - for index, arg in enumerate(args): - q = "SET @_%s_%d=%s" % (procname, index, conn.escape(arg)) - self._query(q) + if args: + fmt = '@_{0}_%d=%s'.format(procname) + self._query('SET %s' % ','.join(fmt % (index, conn.escape(arg)) + for index, arg in enumerate(args))) self.nextset() q = "CALL %s(%s)" % (procname, @@ -268,7 +271,7 @@ def callproc(self, procname, args=()): return args def fetchone(self): - ''' Fetch the next row ''' + """Fetch the next row""" self._check_executed() if self._rows is None or self.rownumber >= len(self._rows): return None @@ -277,7 +280,7 @@ def fetchone(self): return result def fetchmany(self, size=None): - ''' Fetch several rows ''' + """Fetch several rows""" self._check_executed() if self._rows is None: return () @@ -287,7 +290,7 @@ def fetchmany(self, size=None): return result def fetchall(self): - ''' Fetch all the rows ''' + """Fetch all the rows""" self._check_executed() if self._rows is None: return () @@ -314,14 +317,23 @@ def scroll(self, value, mode='relative'): def _query(self, q): conn = self._get_db() self._last_executed = q + self._clear_result() conn.query(q) self._do_get_result() return self.rowcount + def _clear_result(self): + self.rownumber = 0 + self._result = None + + self.rowcount = 0 + self.description = None + self.lastrowid = None + self._rows = None + def _do_get_result(self): conn = self._get_db() - self.rownumber = 0 self._result = result = conn._result self.rowcount = result.affected_rows @@ -329,22 +341,6 @@ def _do_get_result(self): self.lastrowid = result.insert_id self._rows = result.rows - if result.warning_count > 0: - self._show_warnings(conn) - - def _show_warnings(self, conn): - if self._result and self._result.has_next: - return - ws = conn.show_warnings() - if ws is None: - return - for w in ws: - msg = w[-1] - if PY2: - if isinstance(msg, unicode): - msg = msg.encode('utf-8', 'replace') - warnings.warn(str(msg), err.Warning, 4) - def __iter__(self): return iter(self.fetchone, None) @@ -394,8 +390,8 @@ class SSCursor(Cursor): or for connections to remote servers over a slow network. Instead of copying every row of data into a buffer, this will fetch - rows as needed. The upside of this, is the client uses much less memory, - and rows are returned much faster when traveling over a slow network, + rows as needed. The upside of this is the client uses much less memory, + and rows are returned much faster when traveling over a slow network or if the result set is very big. There are limitations, though. The MySQL protocol doesn't support @@ -421,9 +417,12 @@ def close(self): finally: self.connection = None + __del__ = close + def _query(self, q): conn = self._get_db() self._last_executed = q + self._clear_result() conn.query(q, unbuffered=True) self._do_get_result() return self.rowcount @@ -432,11 +431,11 @@ def nextset(self): return self._nextset(unbuffered=True) def read_next(self): - """ Read next row """ + """Read next row""" return self._conv_row(self._result._read_rowdata_packet_unbuffered()) def fetchone(self): - """ Fetch next row """ + """Fetch next row""" self._check_executed() row = self.read_next() if row is None: @@ -464,7 +463,7 @@ def __iter__(self): return self.fetchall_unbuffered() def fetchmany(self, size=None): - """ Fetch many """ + """Fetch many""" self._check_executed() if size is None: size = self.arraysize @@ -503,4 +502,4 @@ def scroll(self, value, mode='relative'): class SSDictCursor(DictCursorMixin, SSCursor): - """ An unbuffered cursor, which returns results as a dictionary """ + """An unbuffered cursor, which returns results as a dictionary""" diff --git a/addons/source-python/packages/site-packages/pymysql/err.py b/addons/source-python/packages/site-packages/pymysql/err.py index 9b6f24e0d..8ca23655c 100644 --- a/addons/source-python/packages/site-packages/pymysql/err.py +++ b/addons/source-python/packages/site-packages/pymysql/err.py @@ -68,15 +68,19 @@ class NotSupportedError(DatabaseError): error_map = {} + def _map_error(exc, *errors): for error in errors: error_map[error] = exc + _map_error(ProgrammingError, ER.DB_CREATE_EXISTS, ER.SYNTAX_ERROR, ER.PARSE_ERROR, ER.NO_SUCH_TABLE, ER.WRONG_DB_NAME, ER.WRONG_TABLE_NAME, ER.FIELD_SPECIFIED_TWICE, ER.INVALID_GROUP_FUNC_USE, ER.UNSUPPORTED_EXTENSION, - ER.TABLE_MUST_HAVE_COLUMNS, ER.CANT_DO_THIS_DURING_AN_TRANSACTION) + ER.TABLE_MUST_HAVE_COLUMNS, ER.CANT_DO_THIS_DURING_AN_TRANSACTION, + ER.WRONG_DB_NAME, ER.WRONG_COLUMN_NAME, + ) _map_error(DataError, ER.WARN_DATA_TRUNCATED, ER.WARN_NULL_TO_NOTNULL, ER.WARN_DATA_OUT_OF_RANGE, ER.NO_DEFAULT, ER.PRIMARY_CANT_HAVE_NULL, ER.DATA_TOO_LONG, ER.DATETIME_FUNCTION_OVERFLOW) @@ -87,34 +91,16 @@ def _map_error(exc, *errors): ER.NOT_SUPPORTED_YET, ER.FEATURE_DISABLED, ER.UNKNOWN_STORAGE_ENGINE) _map_error(OperationalError, ER.DBACCESS_DENIED_ERROR, ER.ACCESS_DENIED_ERROR, ER.CON_COUNT_ERROR, ER.TABLEACCESS_DENIED_ERROR, - ER.COLUMNACCESS_DENIED_ERROR) - -del _map_error, ER - + ER.COLUMNACCESS_DENIED_ERROR, ER.CONSTRAINT_FAILED, ER.LOCK_DEADLOCK) -def _get_error_info(data): - errno = struct.unpack(' len(self._data): + raise Exception('Invalid advance amount (%s) for cursor. ' + 'Position=%s' % (length, new_position)) + self._position = new_position + + def rewind(self, position=0): + """Set the position of the data buffer cursor to 'position'.""" + if position < 0 or position > len(self._data): + raise Exception("Invalid position to rewind cursor to: %s." % position) + self._position = position + + def get_bytes(self, position, length=1): + """Get 'length' bytes starting at 'position'. + + Position is start of payload (first four packet header bytes are not + included) starting at index '0'. + + No error checking is done. If requesting outside end of buffer + an empty string (or string shorter than 'length') may be returned! + """ + return self._data[position:(position+length)] + + if PY2: + def read_uint8(self): + result = ord(self._data[self._position]) + self._position += 1 + return result + else: + def read_uint8(self): + result = self._data[self._position] + self._position += 1 + return result + + def read_uint16(self): + result = struct.unpack_from('= 7 + + def is_eof_packet(self): + # http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-EOF_Packet + # Caution: \xFE may be LengthEncodedInteger. + # If \xFE is LengthEncodedInteger header, 8bytes followed. + return self._data[0:1] == b'\xfe' and len(self._data) < 9 + + def is_auth_switch_request(self): + # http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest + return self._data[0:1] == b'\xfe' + + def is_extra_auth_data(self): + # https://dev.mysql.com/doc/internals/en/successful-authentication.html + return self._data[0:1] == b'\x01' + + def is_resultset_packet(self): + field_count = ord(self._data[0:1]) + return 1 <= field_count <= 250 + + def is_load_local_packet(self): + return self._data[0:1] == b'\xfb' + + def is_error_packet(self): + return self._data[0:1] == b'\xff' + + def check_error(self): + if self.is_error_packet(): + self.raise_for_error() + + def raise_for_error(self): + self.rewind() + self.advance(1) # field_count == error (we already know that) + errno = self.read_uint16() + if DEBUG: print("errno =", errno) + err.raise_mysql_exception(self._data) + + def dump(self): + dump_packet(self._data) + + +class FieldDescriptorPacket(MysqlPacket): + """A MysqlPacket that represents a specific column's metadata in the result. + + Parsing is automatically done and the results are exported via public + attributes on the class such as: db, table_name, name, length, type_code. + """ + + def __init__(self, data, encoding): + MysqlPacket.__init__(self, data, encoding) + self._parse_field_descriptor(encoding) + + def _parse_field_descriptor(self, encoding): + """Parse the 'Field Descriptor' (Metadata) packet. + + This is compatible with MySQL 4.1+ (not compatible with MySQL 4.0). + """ + self.catalog = self.read_length_coded_string() + self.db = self.read_length_coded_string() + self.table_name = self.read_length_coded_string().decode(encoding) + self.org_table = self.read_length_coded_string().decode(encoding) + self.name = self.read_length_coded_string().decode(encoding) + self.org_name = self.read_length_coded_string().decode(encoding) + self.charsetnr, self.length, self.type_code, self.flags, self.scale = ( + self.read_struct('