From 516cc30231f9982206980b2ed8f5d75d10f34f2a Mon Sep 17 00:00:00 2001 From: Hai Zhu <35182391+cocolato@users.noreply.github.com> Date: Fri, 19 Apr 2024 00:18:16 +0800 Subject: [PATCH 01/20] Support using ipv6 in make_client/make_server method (#261) * ipv6 support * Update thriftpy2/rpc.py Co-authored-by: AN Long --------- Co-authored-by: AN Long --- tests/test_aio.py | 65 ++++++++++++++++++++++++--------- tests/test_rpc.py | 38 ++++++++++++++++++-- thriftpy2/contrib/aio/rpc.py | 42 +++++++++++++++------- thriftpy2/rpc.py | 70 +++++++++++++++++++++++------------- 4 files changed, 158 insertions(+), 57 deletions(-) diff --git a/tests/test_aio.py b/tests/test_aio.py index b9de45b..c1d6d85 100644 --- a/tests/test_aio.py +++ b/tests/test_aio.py @@ -1,31 +1,31 @@ # -*- coding: utf-8 -*- +import asyncio import os +import random +import socket import sys -import asyncio # import uvloop import threading -import random +import time from unittest.mock import patch +import pytest + +import thriftpy2 +from thriftpy2.contrib.aio.protocol import (TAsyncBinaryProtocolFactory, + TAsyncCompactProtocolFactory) +from thriftpy2.contrib.aio.transport import (TAsyncBufferedTransportFactory, + TAsyncFramedTransportFactory) +from thriftpy2.rpc import make_aio_client, make_aio_server +from thriftpy2.thrift import TApplicationException +from thriftpy2.transport import TTransportException + # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) -import time -import pytest -import thriftpy2 -from thriftpy2.contrib.aio.transport import ( - TAsyncBufferedTransportFactory, - TAsyncFramedTransportFactory, -) -from thriftpy2.contrib.aio.protocol import ( - TAsyncBinaryProtocolFactory, - TAsyncCompactProtocolFactory, -) -from thriftpy2.rpc import make_aio_server, make_aio_client -from thriftpy2.transport import TTransportException -from thriftpy2.thrift import TApplicationException + if sys.platform == "win32": @@ -113,6 +113,7 @@ class _TestAIO: @classmethod def setup_class(cls): cls._start_server() + cls._start_ipv6_server() cls.person = _create_person() @classmethod @@ -139,6 +140,22 @@ def _start_server(cls): st.start() time.sleep(0.1) + @classmethod + def _start_ipv6_server(cls): + cls.server = make_aio_server( + addressbook.AddressBookService, + Dispatcher(), + trans_factory=cls.TRANSPORT_FACTORY, + proto_factory=cls.PROTOCOL_FACTORY, + loop=asyncio.new_event_loop(), + socket_family=socket.AF_INET6, + **cls.server_kwargs(), + ) + st = threading.Thread(target=cls.server.serve) + st.daemon = True + st.start() + time.sleep(0.1) + @classmethod def server_kwargs(cls): name = cls.__name__.lower() @@ -157,12 +174,28 @@ async def client(self, timeout: int = 3000000): **self.client_kwargs(), ) + async def ipv6_client(self, timeout: int = 3000000): + return await make_aio_client( + addressbook.AddressBookService, + trans_factory=self.TRANSPORT_FACTORY, + proto_factory=self.PROTOCOL_FACTORY, + timeout=timeout, + socket_family=socket.AF_INET6, + **self.client_kwargs(), + ) + @pytest.mark.asyncio async def test_void_api(self): c = await self.client() assert await c.ping() is None c.close() + @pytest.mark.asyncio + async def test_api_ipv6(self): + c = await self.ipv6_client() + assert await c.ping() is None + c.close() + @pytest.mark.asyncio async def test_string_api(self): c = await self.client() diff --git a/tests/test_rpc.py b/tests/test_rpc.py index e7b2283..3864500 100644 --- a/tests/test_rpc.py +++ b/tests/test_rpc.py @@ -15,10 +15,9 @@ thriftpy2.install_import_hook() -from thriftpy2.rpc import make_server, client_context # noqa -from thriftpy2.transport import TTransportException # noqa +from thriftpy2.rpc import client_context, make_server # noqa from thriftpy2.thrift import TApplicationException # noqa - +from thriftpy2.transport import TTransportException # noqa if sys.platform == "win32": pytest.skip("requires unix domain socket", allow_module_level=True) @@ -102,6 +101,26 @@ def fin(): request.addfinalizer(fin) +@pytest.fixture(scope="module") +def ipv6_server(request): + server = make_server(addressbook.AddressBookService, Dispatcher(), + unix_socket=unix_sock, socket_family=socket.AF_INET6) + ps = multiprocessing.Process(target=server.serve) + ps.start() + + time.sleep(0.1) + + def fin(): + if ps.is_alive(): + ps.terminate() + try: + os.remove(unix_sock) + except IOError: + pass + + request.addfinalizer(fin) + + @pytest.fixture(scope="module") def ssl_server(request): ssl_server = make_server(addressbook.AddressBookService, Dispatcher(), @@ -145,6 +164,14 @@ def client(timeout=3000): unix_socket=unix_sock) +def ipv6_client(timeout=3000): + return client_context(addressbook.AddressBookService, + socket_timeout=timeout, + connect_timeout=timeout, + unix_socket=unix_sock, + socket_family=socket.AF_INET6) + + def ssl_client(timeout=3000): return client_context(addressbook.AddressBookService, host='localhost', port=SSL_PORT, @@ -175,6 +202,11 @@ def test_void_api(server): assert c.ping() is None +def test_ipv6_api(ipv6_server): + with ipv6_client() as c: + assert c.ping() is None + + def test_void_api_with_ssl(ssl_server): with ssl_client() as c: assert c.ping() is None diff --git a/thriftpy2/contrib/aio/rpc.py b/thriftpy2/contrib/aio/rpc.py index 6ee8445..6f88025 100644 --- a/thriftpy2/contrib/aio/rpc.py +++ b/thriftpy2/contrib/aio/rpc.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import socket import urllib import warnings @@ -17,7 +18,7 @@ async def make_client(service, host='localhost', port=9090, unix_socket=None, cafile=None, ssl_context=None, certfile=None, keyfile=None, validate=True, url='', - socket_timeout=None): + socket_timeout=None, socket_family=socket.AF_INET): if socket_timeout is not None: warnings.warn( "The 'socket_timeout' argument is deprecated. " @@ -30,22 +31,31 @@ async def make_client(service, host='localhost', port=9090, unix_socket=None, host = parsed_url.hostname or host port = parsed_url.port or port if unix_socket: - socket = TAsyncSocket(unix_socket=unix_socket, - connect_timeout=connect_timeout, - socket_timeout=timeout) + client_socket = TAsyncSocket( + unix_socket=unix_socket, + connect_timeout=connect_timeout, + socket_timeout=timeout, + ) if certfile: warnings.warn("SSL only works with host:port, not unix_socket.") elif host and port: - socket = TAsyncSocket( - host, port, - socket_timeout=timeout, connect_timeout=connect_timeout, - cafile=cafile, ssl_context=ssl_context, - certfile=certfile, keyfile=keyfile, validate=validate) + client_socket = TAsyncSocket( + host, + port, + socket_timeout=timeout, + connect_timeout=connect_timeout, + cafile=cafile, + ssl_context=ssl_context, + certfile=certfile, + keyfile=keyfile, + validate=validate, + socket_family=socket_family, + ) else: raise ValueError("Either host/port or unix_socket" " or url must be provided.") - transport = trans_factory.get_transport(socket) + transport = trans_factory.get_transport(client_socket) protocol = proto_factory.get_protocol(transport) await transport.open() return TAsyncClient(service, protocol) @@ -56,7 +66,8 @@ def make_server(service, handler, proto_factory=TAsyncBinaryProtocolFactory(), trans_factory=TAsyncBufferedTransportFactory(), client_timeout=3000, certfile=None, - keyfile=None, ssl_context=None, loop=None): + keyfile=None, ssl_context=None, loop=None, + socket_family=socket.AF_INET): processor = TAsyncProcessor(service, handler) if unix_socket: @@ -65,9 +76,14 @@ def make_server(service, handler, warnings.warn("SSL only works with host:port, not unix_socket.") elif host and port: server_socket = TAsyncServerSocket( - host=host, port=port, + host=host, + port=port, client_timeout=client_timeout, - certfile=certfile, keyfile=keyfile, ssl_context=ssl_context) + certfile=certfile, + keyfile=keyfile, + ssl_context=ssl_context, + socket_family=socket_family, + ) else: raise ValueError("Either host/port or unix_socket must be provided.") diff --git a/thriftpy2/rpc.py b/thriftpy2/rpc.py index ab6d567..c2c8257 100644 --- a/thriftpy2/rpc.py +++ b/thriftpy2/rpc.py @@ -26,23 +26,30 @@ def make_client(service, host="localhost", port=9090, unix_socket=None, host = parsed_url.hostname or host port = parsed_url.port or port if unix_socket: - socket = TSocket(unix_socket=unix_socket, socket_timeout=timeout) + client_socket = TSocket(unix_socket=unix_socket, socket_timeout=timeout) if certfile: warnings.warn("SSL only works with host:port, not unix_socket.") elif host and port: if cafile or ssl_context: - socket = TSSLSocket(host, port, socket_timeout=timeout, - socket_family=socket_family, cafile=cafile, - certfile=certfile, keyfile=keyfile, - ssl_context=ssl_context) + client_socket = TSSLSocket( + host, + port, + socket_timeout=timeout, + socket_family=socket_family, + cafile=cafile, + certfile=certfile, + keyfile=keyfile, + ssl_context=ssl_context, + ) else: - socket = TSocket(host, port, socket_family=socket_family, - socket_timeout=timeout) + client_socket = TSocket( + host, port, socket_family=socket_family, socket_timeout=timeout + ) else: raise ValueError("Either host/port or unix_socket" " or url must be provided.") - transport = trans_factory.get_transport(socket) + transport = trans_factory.get_transport(client_socket) protocol = proto_factory.get_protocol(transport) transport.open() return TClient(service, protocol) @@ -52,7 +59,8 @@ def make_server(service, handler, host="localhost", port=9090, unix_socket=None, proto_factory=TBinaryProtocolFactory(), trans_factory=TBufferedTransportFactory(), - client_timeout=3000, certfile=None): + client_timeout=3000, certfile=None, + socket_family=socket.AF_INET): processor = TProcessor(service, handler) if unix_socket: @@ -63,10 +71,11 @@ def make_server(service, handler, if certfile: server_socket = TSSLServerSocket( host=host, port=port, client_timeout=client_timeout, - certfile=certfile) + certfile=certfile, socket_family=socket_family) else: server_socket = TServerSocket( - host=host, port=port, client_timeout=client_timeout) + host=host, port=port, client_timeout=client_timeout, + socket_family=socket_family) else: raise ValueError("Either host/port or unix_socket must be provided.") @@ -82,7 +91,7 @@ def client_context(service, host="localhost", port=9090, unix_socket=None, trans_factory=TBufferedTransportFactory(), timeout=None, socket_timeout=3000, connect_timeout=3000, cafile=None, ssl_context=None, certfile=None, keyfile=None, - url=""): + url="", socket_family=socket.AF_INET): if url: parsed_url = urllib.parse.urlparse(url) host = parsed_url.hostname or host @@ -94,29 +103,40 @@ def client_context(service, host="localhost", port=9090, unix_socket=None, socket_timeout = connect_timeout = timeout if unix_socket: - socket = TSocket(unix_socket=unix_socket, - connect_timeout=connect_timeout, - socket_timeout=socket_timeout) + client_socket = TSocket( + unix_socket=unix_socket, + connect_timeout=connect_timeout, + socket_timeout=socket_timeout, + ) if certfile: warnings.warn("SSL only works with host:port, not unix_socket.") elif host and port: if cafile or ssl_context: - socket = TSSLSocket(host, port, - connect_timeout=connect_timeout, - socket_timeout=socket_timeout, - cafile=cafile, - certfile=certfile, keyfile=keyfile, - ssl_context=ssl_context) + client_socket = TSSLSocket( + host, + port, + connect_timeout=connect_timeout, + socket_timeout=socket_timeout, + cafile=cafile, + certfile=certfile, + keyfile=keyfile, + ssl_context=ssl_context, + socket_family=socket_family, + ) else: - socket = TSocket(host, port, - connect_timeout=connect_timeout, - socket_timeout=socket_timeout) + client_socket = TSocket( + host, + port, + connect_timeout=connect_timeout, + socket_timeout=socket_timeout, + socket_family=socket_family, + ) else: raise ValueError("Either host/port or unix_socket" " or url must be provided.") try: - transport = trans_factory.get_transport(socket) + transport = trans_factory.get_transport(client_socket) protocol = proto_factory.get_protocol(transport) transport.open() yield TClient(service, protocol) From 49fe0efc573c94805146636b47b4a880ba38717d Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 7 May 2024 15:39:33 +0800 Subject: [PATCH 02/20] Move some global variables in parser into thread local (#264) --- thriftpy2/parser/__init__.py | 30 ++++++------- thriftpy2/parser/parser.py | 82 +++++++++++++++++++----------------- 2 files changed, 59 insertions(+), 53 deletions(-) diff --git a/thriftpy2/parser/__init__.py b/thriftpy2/parser/__init__.py index 289aa40..8930ce8 100644 --- a/thriftpy2/parser/__init__.py +++ b/thriftpy2/parser/__init__.py @@ -13,7 +13,7 @@ import sys import types -from .parser import parse, parse_fp, incomplete_type, _cast +from .parser import parse, parse_fp, threadlocal, _cast from .exc import ThriftParserError from ..thrift import TPayloadMeta @@ -35,7 +35,7 @@ def load(path, real_module = bool(module_name) thrift = parse(path, module_name, include_dirs=include_dirs, include_dir=include_dir, encoding=encoding) - if incomplete_type: + if threadlocal.incomplete_type: fill_incomplete_ttype(thrift, thrift) # add sub modules to sys.modules recursively @@ -58,18 +58,18 @@ def fill_incomplete_ttype(tmodule, definition): # construct const value if definition[0] == 'UNKNOWN_CONST': ttype = get_definition( - tmodule, incomplete_type[definition[1]][0], definition[3]) + tmodule, threadlocal.incomplete_type[definition[1]][0], definition[3]) return _cast(ttype)(definition[2]) # construct incomplete alias type - elif definition[1] in incomplete_type: + elif definition[1] in threadlocal.incomplete_type: return ( definition[0], - get_definition(tmodule, *incomplete_type[definition[1]]) + get_definition(tmodule, *threadlocal.incomplete_type[definition[1]]) ) # construct incomplete type which is contained in service method's args - elif definition[0] in incomplete_type: + elif definition[0] in threadlocal.incomplete_type: real_type = get_definition( - tmodule, *incomplete_type[definition[0]] + tmodule, *threadlocal.incomplete_type[definition[0]] ) return (real_type[0], definition[1], real_type[1], definition[2]) # construct incomplete compound type @@ -88,10 +88,10 @@ def fill_incomplete_ttype(tmodule, definition): elif isinstance(definition, TPayloadMeta): for index, value in definition.thrift_spec.items(): # if the ttype of the field is a single type and it is incompleted - if value[0] in incomplete_type: + if value[0] in threadlocal.incomplete_type: real_type = fill_incomplete_ttype( tmodule, get_definition( - tmodule, *incomplete_type[value[0]] + tmodule, *threadlocal.incomplete_type[value[0]] ) ) # if the incomplete ttype is a compound type @@ -107,19 +107,19 @@ def fill_incomplete_ttype(tmodule, definition): definition.thrift_spec[index] = ( fill_incomplete_ttype( tmodule, get_definition( - tmodule, *incomplete_type[value[0]] + tmodule, *threadlocal.incomplete_type[value[0]] ) ), ) + tuple(value[1:]) # if the field's ttype is a compound type # and it contains incomplete types - elif value[2] in incomplete_type: + elif value[2] in threadlocal.incomplete_type: definition.thrift_spec[index] = ( value[0], value[1], fill_incomplete_ttype( tmodule, get_definition( - tmodule, *incomplete_type[value[2]] + tmodule, *threadlocal.incomplete_type[value[2]] ) ), value[3]) @@ -129,8 +129,8 @@ def fill_incomplete_ttype(tmodule, definition): def walk(part): if isinstance(part, tuple): return tuple(walk(x) for x in part) - if part in incomplete_type: - return get_definition(tmodule, *incomplete_type[part]) + if part in threadlocal.incomplete_type: + return get_definition(tmodule, *threadlocal.incomplete_type[part]) return part definition.thrift_spec[index] = ( value[0], @@ -158,7 +158,7 @@ def get_definition(thrift, name, lineno): (name, lineno)) if isinstance(ref_type, int) and ref_type < 0: raise ThriftParserError('No type found: %r, at line %d' % - incomplete_type[ref_type]) + threadlocal.incomplete_type[ref_type]) if hasattr(ref_type, '_ttype'): return (getattr(ref_type, '_ttype'), ref_type) else: diff --git a/thriftpy2/parser/parser.py b/thriftpy2/parser/parser.py index b4d861b..8c5f083 100644 --- a/thriftpy2/parser/parser.py +++ b/thriftpy2/parser/parser.py @@ -9,6 +9,7 @@ import collections import os +import threading import types from urllib.parse import urlparse from urllib.request import urlopen @@ -20,6 +21,9 @@ from .lexer import * # noqa +threadlocal = threading.local() + + def p_error(p): if p is None: raise ThriftGrammarError('Grammar error at EOF') @@ -49,12 +53,12 @@ def p_header_unit(p): def p_include(p): '''include : INCLUDE LITERAL''' - thrift = thrift_stack[-1] + thrift = threadlocal.thrift_stack[-1] if thrift.__thrift_file__ is None: raise ThriftParserError('Unexpected include statement while loading ' 'from file like object.') replace_include_dirs = [os.path.dirname(thrift.__thrift_file__)] \ - + include_dirs_ + + threadlocal.include_dirs_ for include_dir in replace_include_dirs: path = os.path.join(include_dir, p[2]) if os.path.exists(path): @@ -74,7 +78,7 @@ def p_namespace(p): '''namespace : NAMESPACE namespace_scope IDENTIFIER''' # namespace is useless in thriftpy2 # if p[2] == 'py' or p[2] == '*': - # setattr(thrift_stack[-1], '__name__', p[3]) + # setattr(threadlocal.thrift_stack[-1], '__name__', p[3]) def p_namespace_scope(p): @@ -114,7 +118,7 @@ def p_const(p): except AssertionError: raise ThriftParserError('Type error for constant %s at line %d' % (p[3], p.lineno(3))) - setattr(thrift_stack[-1], p[3], val) + setattr(threadlocal.thrift_stack[-1], p[3], val) _add_thrift_meta('consts', val) @@ -160,7 +164,7 @@ def p_const_map_item(p): def p_const_ref(p): '''const_ref : IDENTIFIER''' - child = thrift_stack[-1] + child = threadlocal.thrift_stack[-1] for name in p[1].split('.'): father = child child = getattr(child, name, None) @@ -187,13 +191,13 @@ def p_ttype(p): def p_typedef(p): '''typedef : TYPEDEF field_type IDENTIFIER type_annotations''' - setattr(thrift_stack[-1], p[3], p[2]) + setattr(threadlocal.thrift_stack[-1], p[3], p[2]) def p_enum(p): # noqa '''enum : ENUM IDENTIFIER '{' enum_seq '}' type_annotations''' val = _make_enum(p[2], p[4]) - setattr(thrift_stack[-1], p[2], val) + setattr(threadlocal.thrift_stack[-1], p[2], val) _add_thrift_meta('enums', val) @@ -223,7 +227,7 @@ def p_struct(p): def p_seen_struct(p): '''seen_struct : STRUCT IDENTIFIER ''' val = _make_empty_struct(p[2]) - setattr(thrift_stack[-1], p[2], val) + setattr(threadlocal.thrift_stack[-1], p[2], val) p[0] = val @@ -236,14 +240,14 @@ def p_union(p): def p_seen_union(p): '''seen_union : UNION IDENTIFIER ''' val = _make_empty_struct(p[2]) - setattr(thrift_stack[-1], p[2], val) + setattr(threadlocal.thrift_stack[-1], p[2], val) p[0] = val def p_exception(p): '''exception : EXCEPTION IDENTIFIER '{' field_seq '}' type_annotations ''' val = _make_struct(p[2], p[4], base_cls=TException) - setattr(thrift_stack[-1], p[2], val) + setattr(threadlocal.thrift_stack[-1], p[2], val) _add_thrift_meta('exceptions', val) @@ -251,7 +255,7 @@ def p_simple_service(p): '''simple_service : SERVICE IDENTIFIER '{' function_seq '}' | SERVICE IDENTIFIER EXTENDS IDENTIFIER '{' function_seq '}' ''' - thrift = thrift_stack[-1] + thrift = threadlocal.thrift_stack[-1] if len(p) == 8: extends = thrift @@ -386,12 +390,12 @@ def set_info(self, info): return self.index + 1 -incomplete_type = CurrentIncompleteType() +threadlocal.incomplete_type = CurrentIncompleteType() def p_ref_type(p): '''ref_type : IDENTIFIER''' - ref_type = thrift_stack[-1] + ref_type = threadlocal.thrift_stack[-1] for attr in dir(ref_type): if attr in {'__doc__', '__loader__', '__name__', '__package__', @@ -411,7 +415,7 @@ def p_ref_type(p): if index != len(p[1].split('.')) - 1: raise ThriftParserError('No type found: %r, at line %d' % (p[1], p.lineno(1))) - p[0] = incomplete_type.set_info((p[1], p.lineno(1))) + p[0] = threadlocal.incomplete_type.set_info((p[1], p.lineno(1))) return if hasattr(ref_type, '_ttype'): @@ -511,9 +515,9 @@ def p_type_annotation(p): p[0] = p[1], None # Without Value -thrift_stack = [] -include_dirs_ = ['.'] -thrift_cache = {} +threadlocal.thrift_stack = [] +threadlocal.include_dirs_ = ['.'] +threadlocal.thrift_cache = {} def parse(path, module_name=None, include_dirs=None, include_dir=None, @@ -540,29 +544,25 @@ def parse(path, module_name=None, include_dirs=None, include_dir=None, is provided, use it as cache key, else use the `path`. """ # dead include checking on current stack - for thrift in thrift_stack: + for thrift in threadlocal.thrift_stack: if thrift.__thrift_file__ is not None and \ os.path.samefile(path, thrift.__thrift_file__): raise ThriftParserError('Dead including on %s' % path) - global thrift_cache - cache_key = module_name or os.path.normpath(path) - if enable_cache and cache_key in thrift_cache: - return thrift_cache[cache_key] + if enable_cache and cache_key in threadlocal.thrift_cache: + return threadlocal.thrift_cache[cache_key] if lexer is None: lexer = lex.lex() if parser is None: parser = yacc.yacc(debug=False, write_tables=0) - global include_dirs_ - if include_dirs is not None: - include_dirs_ = include_dirs + threadlocal.include_dirs_ = include_dirs if include_dir is not None: - include_dirs_.append(include_dir) + threadlocal.include_dirs_.append(include_dir) if not path.endswith('.thrift'): raise ThriftParserError('Path should end with .thrift') @@ -594,13 +594,13 @@ def parse(path, module_name=None, include_dirs=None, include_dir=None, thrift = types.ModuleType(module_name) setattr(thrift, '__thrift_file__', path) - thrift_stack.append(thrift) + threadlocal.thrift_stack.append(thrift) lexer.lineno = 1 parser.parse(data) - thrift_stack.pop() + threadlocal.thrift_stack.pop() if enable_cache: - thrift_cache[cache_key] = thrift + threadlocal.thrift_cache[cache_key] = thrift return thrift @@ -624,8 +624,8 @@ def parse_fp(source, module_name, lexer=None, parser=None, enable_cache=True): raise ThriftParserError('thriftpy2 can only generate module with ' '\'_thrift\' suffix') - if enable_cache and module_name in thrift_cache: - return thrift_cache[module_name] + if enable_cache and module_name in threadlocal.thrift_cache: + return threadlocal.thrift_cache[module_name] if not hasattr(source, 'read'): raise ThriftParserError('Expected `source` to be a file-like object ' @@ -640,18 +640,18 @@ def parse_fp(source, module_name, lexer=None, parser=None, enable_cache=True): thrift = types.ModuleType(module_name) setattr(thrift, '__thrift_file__', None) - thrift_stack.append(thrift) + threadlocal.thrift_stack.append(thrift) lexer.lineno = 1 parser.parse(data) - thrift_stack.pop() + threadlocal.thrift_stack.pop() if enable_cache: - thrift_cache[module_name] = thrift + threadlocal.thrift_cache[module_name] = thrift return thrift def _add_thrift_meta(key, val): - thrift = thrift_stack[-1] + thrift = threadlocal.thrift_stack[-1] if not hasattr(thrift, '__thrift_meta__'): meta = collections.defaultdict(list) @@ -827,7 +827,10 @@ def __cast_struct(v): def _make_enum(name, kvs): - attrs = {'__module__': thrift_stack[-1].__name__, '_ttype': TType.I32} + attrs = { + '__module__': threadlocal.thrift_stack[-1].__name__, + '_ttype': TType.I32 + } cls = type(name, (object, ), attrs) _values_to_names = {} @@ -851,7 +854,10 @@ def _make_enum(name, kvs): def _make_empty_struct(name, ttype=TType.STRUCT, base_cls=TPayload): - attrs = {'__module__': thrift_stack[-1].__name__, '_ttype': ttype} + attrs = { + '__module__': threadlocal.thrift_stack[-1].__name__, + '_ttype': ttype + } return type(name, (base_cls, ), attrs) @@ -887,7 +893,7 @@ def _make_service(name, funcs, extends): if extends is None: extends = object - attrs = {'__module__': thrift_stack[-1].__name__} + attrs = {'__module__': threadlocal.thrift_stack[-1].__name__} cls = type(name, (extends, ), attrs) thrift_services = [] From a16e0db4a4ac9a5d2d64cb8127384e12e779dcc8 Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 7 May 2024 19:56:08 +0800 Subject: [PATCH 03/20] Bump version 0.5.0 (#265) * Bump version 0.5.0 * Update changelog --- CHANGES.rst | 15 +++++++++++++++ thriftpy2/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 81fb95f..3d8e71e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,21 @@ Changelog ========= +0.5.0 +~~~~~ + +Version 0.5.0 +------------- + +Released on May 7, 2024. + +- Dropped Python2 and Python3.5 Support. +- Added SASL transport client. +- Add submodule to sys.path when loading child idl file. +- Support cythonized module on Windows. +- Support using ipv6 in make_client/make_server method. +- Basic multi-thread support in parser. + 0.4.x ~~~~~ diff --git a/thriftpy2/__init__.py b/thriftpy2/__init__.py index 0410e18..eadf440 100644 --- a/thriftpy2/__init__.py +++ b/thriftpy2/__init__.py @@ -5,7 +5,7 @@ from .hook import install_import_hook, remove_import_hook from .parser import load, load_module, load_fp -__version__ = '0.4.20' +__version__ = '0.5.0' __python__ = sys.version_info __all__ = ["install_import_hook", "remove_import_hook", "load", "load_module", "load_fp"] From 9db168d662235c2b1e42def39885ee21e4644f19 Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 14 May 2024 21:12:50 +0800 Subject: [PATCH 04/20] Initialize the thread local in parser module in every thread (#267) * Initialize the thread local in parser module in every thread * Remove redundant codes --- thriftpy2/parser/parser.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/thriftpy2/parser/parser.py b/thriftpy2/parser/parser.py index 8c5f083..bd1bf6d 100644 --- a/thriftpy2/parser/parser.py +++ b/thriftpy2/parser/parser.py @@ -390,9 +390,6 @@ def set_info(self, info): return self.index + 1 -threadlocal.incomplete_type = CurrentIncompleteType() - - def p_ref_type(p): '''ref_type : IDENTIFIER''' ref_type = threadlocal.thrift_stack[-1] @@ -515,11 +512,6 @@ def p_type_annotation(p): p[0] = p[1], None # Without Value -threadlocal.thrift_stack = [] -threadlocal.include_dirs_ = ['.'] -threadlocal.thrift_cache = {} - - def parse(path, module_name=None, include_dirs=None, include_dir=None, lexer=None, parser=None, enable_cache=True, encoding='utf-8'): """Parse a single thrift file to module object, e.g.:: @@ -543,6 +535,15 @@ def parse(path, module_name=None, include_dirs=None, include_dir=None, cached, this is enabled by default. If `module_name` is provided, use it as cache key, else use the `path`. """ + # threadlocal should be initialized in every threads + initialized = getattr(threadlocal, 'initialized', None) + if initialized is None: + threadlocal.thrift_stack = [] + threadlocal.include_dirs_ = ['.'] + threadlocal.thrift_cache = {} + threadlocal.incomplete_type = CurrentIncompleteType() + threadlocal.initialized = True + # dead include checking on current stack for thrift in threadlocal.thrift_stack: if thrift.__thrift_file__ is not None and \ From 261f90ccff1abb2774176647f3ee4abdc2a19dc2 Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 14 May 2024 21:19:58 +0800 Subject: [PATCH 05/20] Bump version 0.5.1rc1 (#268) --- CHANGES.rst | 5 +++++ thriftpy2/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3d8e71e..cee1a56 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ Changelog ========= +UNRELEASED +~~~~~~~~~~ + +- Fix an issue where loading a thrift file in a sub-thread will cause an error. + 0.5.0 ~~~~~ diff --git a/thriftpy2/__init__.py b/thriftpy2/__init__.py index eadf440..ef58495 100644 --- a/thriftpy2/__init__.py +++ b/thriftpy2/__init__.py @@ -5,7 +5,7 @@ from .hook import install_import_hook, remove_import_hook from .parser import load, load_module, load_fp -__version__ = '0.5.0' +__version__ = '0.5.1rc1' __python__ = sys.version_info __all__ = ["install_import_hook", "remove_import_hook", "load", "load_module", "load_fp"] From 30deec71b4de9d4c17f82b96ce741cacb96a4abe Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 17 May 2024 17:42:34 +0800 Subject: [PATCH 06/20] Test load thrift file in sub-thread (#269) --- setup.py | 2 +- tests/test_parser.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ab45325..c68d3a0 100644 --- a/setup.py +++ b/setup.py @@ -33,9 +33,9 @@ dev_requires = [ "flake8>=2.5", - "pytest>=2.8", "sphinx-rtd-theme>=0.1.9", "sphinx>=1.3", + "pytest-reraise", "pytest>=6.1.1", ] + tornado_requires diff --git a/tests/test_parser.py b/tests/test_parser.py index 1c745fe..a14f29a 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import threading + import pytest from thriftpy2.thrift import TType from thriftpy2.parser import load, load_fp @@ -41,6 +43,16 @@ def test_cpp_include(): load('parser-cases/cpp_include.thrift') +def test_load_in_sub_thread(reraise): + @reraise.wrap + def f(): + load('addressbook.thrift') + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_tutorial(): thrift = load('parser-cases/tutorial.thrift', include_dirs=[ './parser-cases']) From c3d7dc40061465bc2f7798a67ea2f883c032cd38 Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 28 May 2024 21:40:24 +0800 Subject: [PATCH 07/20] pin pytest for it's bug (#272) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c68d3a0..d30343e 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ "sphinx-rtd-theme>=0.1.9", "sphinx>=1.3", "pytest-reraise", - "pytest>=6.1.1", + "pytest>=6.1.1,<8.2.0", ] + tornado_requires From a3db32522b32ae7cd6fd1acd469cdf1ef4c9aedb Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 9 Jun 2024 15:57:44 +0800 Subject: [PATCH 08/20] Add missing init file for cybin module (#256) * Add missing init file for cybin module * rename cybin extension name * fix import in cybin.pyx --- setup.py | 2 +- thriftpy2/protocol/cybin/__init__.py | 1 + thriftpy2/protocol/cybin/cybin.pyx | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 thriftpy2/protocol/cybin/__init__.py diff --git a/setup.py b/setup.py index d30343e..f469a20 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ libraries=libraries)) ext_modules.append(Extension("thriftpy2.transport.sasl.cysasl", ["thriftpy2/transport/sasl/cysasl.c"])) - ext_modules.append(Extension("thriftpy2.protocol.cybin", + ext_modules.append(Extension("thriftpy2.protocol.cybin.cybin", ["thriftpy2/protocol/cybin/cybin.c"], libraries=libraries)) diff --git a/thriftpy2/protocol/cybin/__init__.py b/thriftpy2/protocol/cybin/__init__.py new file mode 100644 index 0000000..82a0121 --- /dev/null +++ b/thriftpy2/protocol/cybin/__init__.py @@ -0,0 +1 @@ +from .cybin import * diff --git a/thriftpy2/protocol/cybin/cybin.pyx b/thriftpy2/protocol/cybin/cybin.pyx index 97d6ec9..3532605 100644 --- a/thriftpy2/protocol/cybin/cybin.pyx +++ b/thriftpy2/protocol/cybin/cybin.pyx @@ -7,10 +7,9 @@ from cpython cimport bool import six +from thriftpy2.thrift import TDecodeException from thriftpy2.transport.cybase cimport CyTransportBase, STACK_STRING_LEN -from ..thrift import TDecodeException - cdef extern from "endian_port.h": int16_t htobe16(int16_t n) int32_t htobe32(int32_t n) From 2c449dcbb6b6e914946fa79896681eadf6ba24dc Mon Sep 17 00:00:00 2001 From: AN Long Date: Wed, 12 Jun 2024 18:26:58 +0800 Subject: [PATCH 09/20] Fix typos in comments, docs and tests (#273) --- CHANGES.rst | 8 ++++---- tests/addressbook.py | 2 +- tests/compatible/version_2/tracking.py | 2 +- tests/container.py | 2 +- tests/test_rpc.py | 2 +- thriftpy2/contrib/tracking/__init__.py | 2 +- thriftpy2/hook.py | 2 +- thriftpy2/parser/parser.py | 4 ++-- thriftpy2/transport/base.py | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index cee1a56..b2b5fea 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -185,7 +185,7 @@ Version 0.4.5 Released on August 27, 2019. - Support kwargs style parameters passing in TSimpleServer, via `2-#67`_. -- Fix #65 allow double const to ommit integer part, via `2-#66`_. +- Fix #65 allow double const to omit integer part, via `2-#66`_. .. _2-#67: https://github.com/Thriftpy/thriftpy2/pull/67 .. _2-#66: https://github.com/Thriftpy/thriftpy2/pull/66 @@ -461,7 +461,7 @@ Released on April 15, 2015. - add limitation on thrift reserved keyword for compatible with upstream, via `#115`_. - bugfix EOF grammar error, via `#103`_. -- bugfix for mis-mach transport in client caused server crash, via `#119`_. +- bugfix for mismatch transport in client caused server crash, via `#119`_. - bugfix for typedef on included thrift files, via `#121`_. .. _`#96`: https://github.com/eleme/thriftpy/pull/96 @@ -487,7 +487,7 @@ Released on March 3, 2015. - bugfix for transport clean in read_struct in cybin, via `#70`_. - bugfix for large reading size in framed transport, via `#73`_. - bugfix for cython build failed in older CentOS, via `#92`_. -- bugfix for thrift file version mis-match caused message corrupt in +- bugfix for thrift file version mismatch caused message corrupt in `read_struct`, via `#95`_. Non-Backward Compatible changes: @@ -719,7 +719,7 @@ Released on June 7, 2014. - disabled the magic import hook by default. and add install/remove function to switch the hook on and off. - reworked benchmark suit and add benchmark results. -- new `__init__` function code generator. get a noticable speed boost. +- new `__init__` function code generator. get a noticeable speed boost. - bug fixes diff --git a/tests/addressbook.py b/tests/addressbook.py index 5a17cfe..4db06b3 100644 --- a/tests/addressbook.py +++ b/tests/addressbook.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -"""This file is a demo for what the dynamiclly generated code would be like. +"""This file is a demo for what the dynamically generated code would be like. """ from thriftpy2.thrift import ( diff --git a/tests/compatible/version_2/tracking.py b/tests/compatible/version_2/tracking.py index 0abc9ca..da1bf06 100644 --- a/tests/compatible/version_2/tracking.py +++ b/tests/compatible/version_2/tracking.py @@ -36,7 +36,7 @@ def __init__(self, request_id, api, seq, client, server, status, start, :status: request status :start: start timestamp :end: end timestamp - :annotation: application-level key-value datas + :annotation: application-level key-value data """ self.request_id = request_id self.api = api diff --git a/tests/container.py b/tests/container.py index d9c5dda..c7086ff 100644 --- a/tests/container.py +++ b/tests/container.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -"""This file is a demo for what the dynamiclly generated code would be like. +"""This file is a demo for what the dynamically generated code would be like. """ from thriftpy2.thrift import ( diff --git a/tests/test_rpc.py b/tests/test_rpc.py index 3864500..fdd1598 100644 --- a/tests/test_rpc.py +++ b/tests/test_rpc.py @@ -279,7 +279,7 @@ def test_exception(): c.remove("Bob") -def test_exception_iwth_ssl(): +def test_exception_with_ssl(): with pytest.raises(addressbook.PersonNotExistsError): with ssl_client() as c: c.remove("Bob") diff --git a/thriftpy2/contrib/tracking/__init__.py b/thriftpy2/contrib/tracking/__init__.py index 0035e84..2dc7106 100644 --- a/thriftpy2/contrib/tracking/__init__.py +++ b/thriftpy2/contrib/tracking/__init__.py @@ -37,7 +37,7 @@ def __init__(self, request_id, api, seq, client, server, status, start, :status: request status :start: start timestamp :end: end timestamp - :annotation: application-level key-value datas + :annotation: application-level key-value data """ self.request_id = request_id self.api = api diff --git a/thriftpy2/hook.py b/thriftpy2/hook.py index 6d709e1..014ee1b 100644 --- a/thriftpy2/hook.py +++ b/thriftpy2/hook.py @@ -10,7 +10,7 @@ # TODO: The load process does not compatible with Python standard, e.g., if the -# specified thrift file does not exists, it raises FileNotFoundError, and skiped +# specified thrift file does not exists, it raises FileNotFoundError, and skipped # the other meta finders in the sys.meta_path. class ThriftImporter(importlib.abc.MetaPathFinder): def __init__(self, extension="_thrift"): diff --git a/thriftpy2/parser/parser.py b/thriftpy2/parser/parser.py index bd1bf6d..a67d87f 100644 --- a/thriftpy2/parser/parser.py +++ b/thriftpy2/parser/parser.py @@ -527,7 +527,7 @@ def parse(path, module_name=None, include_dirs=None, include_dir=None, the `include` directive, by default: ['.']. :param include_dir: directory to find child thrift files. Note this keyword parameter will be deprecated in the future, it exists - for compatiable reason. If it's provided (not `None`), + for compatible reason. If it's provided (not `None`), it will be appended to `include_dirs`. :param lexer: ply lexer to use, if not provided, `parse` will new one. :param parser: ply parser to use, if not provided, `parse` will new one. @@ -614,7 +614,7 @@ def parse_fp(source, module_name, lexer=None, parser=None, enable_cache=True): :param source: file-like object, expected to have a method named `read`. - :param module_name: the name for parsed module, shoule be endswith + :param module_name: the name for parsed module, should be endswith '_thrift'. :param lexer: ply lexer to use, if not provided, `parse` will new one. :param parser: ply parser to use, if not provided, `parse` will new one. diff --git a/thriftpy2/transport/base.py b/thriftpy2/transport/base.py index 5166322..3060475 100644 --- a/thriftpy2/transport/base.py +++ b/thriftpy2/transport/base.py @@ -61,7 +61,7 @@ def read(self, sz): def write(self, buf): """ - Submit some data to tbe written to the connection. May be + Submit some data to be written to the connection. May be buffered until flush is called. """ raise NotImplementedError From 07aa2d5b2c71ff4e2436553472ae5e6c3be59df1 Mon Sep 17 00:00:00 2001 From: AN Long Date: Wed, 12 Jun 2024 19:14:59 +0800 Subject: [PATCH 10/20] Using ThriftGrammarError instead of Thrift GrammerError in tests (#274) * Using ThriftGrammarError instead of Thrift GrammerError in tests * Continue fix --- ...f.thrift => e_grammar_error_at_eof.thrift} | 0 tests/test_parser.py | 20 +++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) rename tests/parser-cases/{e_grammer_error_at_eof.thrift => e_grammar_error_at_eof.thrift} (100%) diff --git a/tests/parser-cases/e_grammer_error_at_eof.thrift b/tests/parser-cases/e_grammar_error_at_eof.thrift similarity index 100% rename from tests/parser-cases/e_grammer_error_at_eof.thrift rename to tests/parser-cases/e_grammar_error_at_eof.thrift diff --git a/tests/test_parser.py b/tests/test_parser.py index a14f29a..7c4fd7f 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -5,7 +5,7 @@ import pytest from thriftpy2.thrift import TType from thriftpy2.parser import load, load_fp -from thriftpy2.parser.exc import ThriftParserError, ThriftGrammerError +from thriftpy2.parser.exc import ThriftParserError, ThriftGrammarError def test_comments(): @@ -216,9 +216,9 @@ def test_e_dead_include(): assert 'Dead including' in str(excinfo.value) -def test_e_grammer_error_at_eof(): - with pytest.raises(ThriftGrammerError) as excinfo: - load('parser-cases/e_grammer_error_at_eof.thrift') +def test_e_grammar_error_at_eof(): + with pytest.raises(ThriftGrammarError) as excinfo: + load('parser-cases/e_grammar_error_at_eof.thrift') assert str(excinfo.value) == 'Grammar error at EOF' @@ -229,28 +229,28 @@ def test_e_use_thrift_reserved_keywords(): def test_e_duplicate_field_id_or_name(): - with pytest.raises(ThriftGrammerError) as excinfo: + with pytest.raises(ThriftGrammarError) as excinfo: load('parser-cases/e_duplicate_field_id.thrift') assert 'field identifier/name has already been used' in str(excinfo.value) - with pytest.raises(ThriftGrammerError) as excinfo: + with pytest.raises(ThriftGrammarError) as excinfo: load('parser-cases/e_duplicate_field_name.thrift') assert 'field identifier/name has already been used' in str(excinfo.value) def test_e_duplicate_struct_exception_service(): - with pytest.raises(ThriftGrammerError) as excinfo: + with pytest.raises(ThriftGrammarError) as excinfo: load('parser-cases/e_duplicate_struct.thrift') assert 'type is already defined in' in str(excinfo.value) - with pytest.raises(ThriftGrammerError) as excinfo: + with pytest.raises(ThriftGrammarError) as excinfo: load('parser-cases/e_duplicate_exception.thrift') assert 'type is already defined in' in str(excinfo.value) - with pytest.raises(ThriftGrammerError) as excinfo: + with pytest.raises(ThriftGrammarError) as excinfo: load('parser-cases/e_duplicate_service.thrift') assert 'type is already defined in' in str(excinfo.value) def test_e_duplicate_function(): - with pytest.raises(ThriftGrammerError) as excinfo: + with pytest.raises(ThriftGrammarError) as excinfo: load('parser-cases/e_duplicate_function.thrift') assert 'function is already defined in' in str(excinfo.value) From cfe909611a56035263c4d22aa850665e2ecdcc07 Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 18 Jun 2024 15:52:35 +0800 Subject: [PATCH 11/20] Fix typos (#275) * Fix typo for Hander * Fix typos in error message --- thriftpy2/http.py | 4 ++-- thriftpy2/parser/lexer.py | 2 +- thriftpy2/parser/parser.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/thriftpy2/http.py b/thriftpy2/http.py index ae18c3a..33fd35e 100644 --- a/thriftpy2/http.py +++ b/thriftpy2/http.py @@ -125,7 +125,7 @@ def __init__(self, thttpserver = self - class RequestHander(http_server.BaseHTTPRequestHandler): + class RequestHandler(http_server.BaseHTTPRequestHandler): # Don't care about the request path. def do_POST(self): @@ -153,7 +153,7 @@ def do_POST(self): self.end_headers() self.wfile.write(otrans.getvalue()) - self.httpd = server_class(server_address, RequestHander) + self.httpd = server_class(server_address, RequestHandler) def serve(self): self.httpd.serve_forever() diff --git a/thriftpy2/parser/lexer.py b/thriftpy2/parser/lexer.py index 091c927..581cd6e 100644 --- a/thriftpy2/parser/lexer.py +++ b/thriftpy2/parser/lexer.py @@ -238,7 +238,7 @@ def t_LITERAL(t): if s[i] in maps: val += maps[s[i]] else: - msg = 'Unexcepted escaping characher: %s' % s[i] + msg = 'Unexpected escaping character: %s' % s[i] raise ThriftLexerError(msg) else: val += s[i] diff --git a/thriftpy2/parser/parser.py b/thriftpy2/parser/parser.py index a67d87f..bf6c525 100644 --- a/thriftpy2/parser/parser.py +++ b/thriftpy2/parser/parser.py @@ -169,7 +169,7 @@ def p_const_ref(p): father = child child = getattr(child, name, None) if child is None: - raise ThriftParserError('Cann\'t find name %r at line %d' + raise ThriftParserError('Can\'t find name %r at line %d' % (p[1], p.lineno(1))) if _get_ttype(child) is None or _get_ttype(father) == TType.I32: From c105b360380a753ae3ee2682fbc0e3f9fbb17ec5 Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 18 Jun 2024 16:15:02 +0800 Subject: [PATCH 12/20] Bump version v0.5.0rc2 (#277) --- CHANGES.rst | 1 + thriftpy2/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b2b5fea..4db0b06 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,7 @@ UNRELEASED ~~~~~~~~~~ - Fix an issue where loading a thrift file in a sub-thread will cause an error. +- Some typo fixes. 0.5.0 ~~~~~ diff --git a/thriftpy2/__init__.py b/thriftpy2/__init__.py index ef58495..ec5007c 100644 --- a/thriftpy2/__init__.py +++ b/thriftpy2/__init__.py @@ -5,7 +5,7 @@ from .hook import install_import_hook, remove_import_hook from .parser import load, load_module, load_fp -__version__ = '0.5.1rc1' +__version__ = '0.5.1rc2' __python__ = sys.version_info __all__ = ["install_import_hook", "remove_import_hook", "load", "load_module", "load_fp"] From 1bd814ad01b3bd7222b6f299ffe0a5f9a8e7b0c6 Mon Sep 17 00:00:00 2001 From: AN Long Date: Mon, 24 Jun 2024 21:30:27 +0800 Subject: [PATCH 13/20] Bump version 0.5.1 (#280) --- CHANGES.rst | 12 +++++++----- thriftpy2/__init__.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4db0b06..be8a2f3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,15 +1,17 @@ Changelog ========= -UNRELEASED -~~~~~~~~~~ +0.5.x +~~~~~ + +Version0.5.1 +------------ + +Released on Jun 24, 2024. - Fix an issue where loading a thrift file in a sub-thread will cause an error. - Some typo fixes. -0.5.0 -~~~~~ - Version 0.5.0 ------------- diff --git a/thriftpy2/__init__.py b/thriftpy2/__init__.py index ec5007c..7263ff6 100644 --- a/thriftpy2/__init__.py +++ b/thriftpy2/__init__.py @@ -5,7 +5,7 @@ from .hook import install_import_hook, remove_import_hook from .parser import load, load_module, load_fp -__version__ = '0.5.1rc2' +__version__ = '0.5.1' __python__ = sys.version_info __all__ = ["install_import_hook", "remove_import_hook", "load", "load_module", "load_fp"] From e4baaf5014b976bb273bc766ee91ea681059d314 Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 28 Jun 2024 16:39:01 +0800 Subject: [PATCH 14/20] Initialize parser's threadlocal in parse_fp (#284) --- tests/test_parser.py | 30 +++++++++++++++++++++++++++++- thriftpy2/parser/parser.py | 9 +++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/test_parser.py b/tests/test_parser.py index 7c4fd7f..cb51a71 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -3,6 +3,7 @@ import threading import pytest + from thriftpy2.thrift import TType from thriftpy2.parser import load, load_fp from thriftpy2.parser.exc import ThriftParserError, ThriftGrammarError @@ -43,7 +44,23 @@ def test_cpp_include(): load('parser-cases/cpp_include.thrift') -def test_load_in_sub_thread(reraise): +@pytest.fixture +def reset_parser_threadlocal(): + def delattr_no_error(o, name): + try: + delattr(o, name) + except AttributeError: + pass + + from thriftpy2.parser.parser import threadlocal + delattr_no_error(threadlocal, 'thrift_stack') + delattr_no_error(threadlocal, 'include_dirs_') + delattr_no_error(threadlocal, 'thrift_cache') + delattr_no_error(threadlocal, 'incomplete_type') + delattr_no_error(threadlocal, 'initialized') + + +def test_load_in_sub_thread(reraise, reset_parser_threadlocal): @reraise.wrap def f(): load('addressbook.thrift') @@ -53,6 +70,17 @@ def f(): t.join() +def test_load_fp_in_sub_thread(reraise, reset_parser_threadlocal): + @reraise.wrap + def f(): + with open('container.thrift') as fp: + load_fp(fp, 'container_thrift') + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_tutorial(): thrift = load('parser-cases/tutorial.thrift', include_dirs=[ './parser-cases']) diff --git a/thriftpy2/parser/parser.py b/thriftpy2/parser/parser.py index bf6c525..9ad2ebd 100644 --- a/thriftpy2/parser/parser.py +++ b/thriftpy2/parser/parser.py @@ -621,6 +621,15 @@ def parse_fp(source, module_name, lexer=None, parser=None, enable_cache=True): :param enable_cache: if this is set to be `True`, parsed module will be cached by `module_name`, this is enabled by default. """ + # threadlocal should be initialized in every threads + initialized = getattr(threadlocal, 'initialized', None) + if initialized is None: + threadlocal.thrift_stack = [] + threadlocal.include_dirs_ = ['.'] + threadlocal.thrift_cache = {} + threadlocal.incomplete_type = CurrentIncompleteType() + threadlocal.initialized = True + if not module_name.endswith('_thrift'): raise ThriftParserError('thriftpy2 can only generate module with ' '\'_thrift\' suffix') From ca6389d11a5a7cd2dc322fdf7a516541303e706d Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 28 Jun 2024 16:54:02 +0800 Subject: [PATCH 15/20] Bump version 0.5.2b1 (#286) --- CHANGES.rst | 10 ++++++++-- thriftpy2/__init__.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index be8a2f3..c036b43 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,8 +4,14 @@ Changelog 0.5.x ~~~~~ -Version0.5.1 ------------- +UNRELEASED +---------- + +- Fix an issue where loading a thrift file in a sub-thread will cause an error with ``load_fp``. + + +Version 0.5.1 +------------- Released on Jun 24, 2024. diff --git a/thriftpy2/__init__.py b/thriftpy2/__init__.py index 7263ff6..0064e91 100644 --- a/thriftpy2/__init__.py +++ b/thriftpy2/__init__.py @@ -5,7 +5,7 @@ from .hook import install_import_hook, remove_import_hook from .parser import load, load_module, load_fp -__version__ = '0.5.1' +__version__ = '0.5.2b1' __python__ = sys.version_info __all__ = ["install_import_hook", "remove_import_hook", "load", "load_module", "load_fp"] From 5b319bd5138b32ebc27a63acd74e9a9331465f4b Mon Sep 17 00:00:00 2001 From: Hai Zhu <35182391+cocolato@users.noreply.github.com> Date: Tue, 2 Jul 2024 23:58:45 +0800 Subject: [PATCH 16/20] move static metadata from setup.py to pyproject.toml (#283) --- .gitignore | 3 +++ pyproject.toml | 37 +++++++++++++++++++++++++++++++++++++ setup.cfg | 3 --- setup.py | 41 +++-------------------------------------- 4 files changed, 43 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 58f2d90..7b25862 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,9 @@ pip-log.txt env/ .vscode/ +# lock file +*.lock + pyvenv.cfg share/* diff --git a/pyproject.toml b/pyproject.toml index 3b109da..af83bb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,39 @@ [build-system] requires = ["setuptools", "cython>=0.28.4,<4"] + +[project] +name = "thriftpy2" +version = "0.5.1" +description = "Pure python implementation of Apache Thrift." +authors = [ + {name = "ThriftPy Organization", email = "gotzehsing@gmail.com"}, +] +dependencies = [ + "Cython>=3.0.10", + "ply>=3.4,<4.0", + "six~=1.15", +] +requires-python = ">=3.6" +readme = "README.rst" +license = {file = "LICENSE"} +keywords = ["thrift python thriftpy thriftpy2"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Software Development", +] + +[project.urls] +Homepage = "https://thriftpy2.readthedocs.io/" +Source = "https://github.com/Thriftpy/thriftpy2" diff --git a/setup.cfg b/setup.cfg index f47f307..d8f9ee1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[metadata] -license_files = LICENSE - [wheel] universal = 1 diff --git a/setup.py b/setup.py index f469a20..c52a3ea 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import re import sys import platform -from os.path import join, dirname +from setuptools import setup, find_packages, Extension -from setuptools import setup, find_packages -from setuptools.extension import Extension - -with open(join(dirname(__file__), 'thriftpy2', '__init__.py'), 'r') as f: - version = re.match(r".*__version__ = '(.*?)'", f.read(), re.S).group(1) install_requires = [ "ply>=3.4,<4.0", @@ -39,7 +33,6 @@ "pytest>=6.1.1,<8.2.0", ] + tornado_requires - cmdclass = {} ext_modules = [] @@ -74,24 +67,11 @@ ["thriftpy2/protocol/cybin/cybin.c"], libraries=libraries)) -setup(name="thriftpy2", - version=version, - description="Pure python implementation of Apache Thrift.", - keywords="thrift python thriftpy thriftpy2", - author="ThriftPy Organization", - author_email="gotzehsing@gmail.com", +setup( packages=find_packages(exclude=['benchmark', 'docs', 'tests']), - entry_points={}, - url="https://thriftpy2.readthedocs.io/", - project_urls={ - "Source": "https://github.com/Thriftpy/thriftpy2", - }, - license="MIT", zip_safe=False, long_description=open("README.rst").read(), install_requires=install_requires, - tests_require=tornado_requires, - python_requires='>=3.6', extras_require={ "dev": dev_requires, "tornado": tornado_requires @@ -99,19 +79,4 @@ cmdclass=cmdclass, ext_modules=ext_modules, include_package_data=True, - classifiers=[ - "Topic :: Software Development", - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - ]) +) From c52cd9cc92335c369ccd146c6f2ca33823a69962 Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 4 Jul 2024 23:23:28 +0800 Subject: [PATCH 17/20] Using a thread pool to avoid TAsyncSocket.open block the event loop (#287) * Using a thread pool to avoid TAsyncSocket.open block the event loop * Compatible with Python3.6 --- thriftpy2/contrib/aio/socket.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/thriftpy2/contrib/aio/socket.py b/thriftpy2/contrib/aio/socket.py index 8d1e49d..8f4195a 100644 --- a/thriftpy2/contrib/aio/socket.py +++ b/thriftpy2/contrib/aio/socket.py @@ -2,13 +2,17 @@ from __future__ import absolute_import, division -import ssl import asyncio import errno import os import socket +import ssl import struct import sys +if sys.version_info >= (3, 7, 0): + from asyncio import get_running_loop +else: + from asyncio import _get_running_loop as get_running_loop from thriftpy2.transport import TTransportException from thriftpy2.transport._ssl import ( @@ -17,6 +21,7 @@ DEFAULT_CIPHERS ) + MAC_OR_BSD = sys.platform == 'darwin' or sys.platform.startswith('freebsd') @@ -145,7 +150,11 @@ async def open(self): if self.connect_timeout: self.raw_sock.settimeout(self.connect_timeout) - self.raw_sock.connect(addr) + loop = get_running_loop() + # The raw_sock.connect may block the event loop if the target + # server is slow or unreachable. Using a thread pool to solve it + # as a quick and dirty way. See #270. + await loop.run_in_executor(None, lambda: self.raw_sock.connect(addr)) if self.socket_timeout: self.raw_sock.settimeout(self.socket_timeout) From 9158d0abbb0ab132645def6f2998c5cbb46db065 Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 4 Jul 2024 23:42:33 +0800 Subject: [PATCH 18/20] Bump version 0.5.2b2 (#288) * Bump version 0.5.2b2 * update changelog --- CHANGES.rst | 2 ++ pyproject.toml | 2 +- thriftpy2/__init__.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c036b43..d668537 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,8 @@ UNRELEASED ---------- - Fix an issue where loading a thrift file in a sub-thread will cause an error with ``load_fp``. +- Move static metadata from ``setup.py`` to ``pyproject.toml``. +- Using a thread pool to avoid ``TAsyncSocket.open`` block the event loop. Version 0.5.1 diff --git a/pyproject.toml b/pyproject.toml index af83bb1..842c259 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools", "cython>=0.28.4,<4"] [project] name = "thriftpy2" -version = "0.5.1" +version = "0.5.2b2" description = "Pure python implementation of Apache Thrift." authors = [ {name = "ThriftPy Organization", email = "gotzehsing@gmail.com"}, diff --git a/thriftpy2/__init__.py b/thriftpy2/__init__.py index 0064e91..be8fb3c 100644 --- a/thriftpy2/__init__.py +++ b/thriftpy2/__init__.py @@ -5,7 +5,7 @@ from .hook import install_import_hook, remove_import_hook from .parser import load, load_module, load_fp -__version__ = '0.5.2b1' +__version__ = '0.5.2b2' __python__ = sys.version_info __all__ = ["install_import_hook", "remove_import_hook", "load", "load_module", "load_fp"] From 9d89758b3f0acf0bc7f89192c0260d3a71f83f81 Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 5 Jul 2024 19:56:55 +0800 Subject: [PATCH 19/20] Bump version 0.5.2 (#289) --- CHANGES.rst | 6 ++++-- pyproject.toml | 2 +- thriftpy2/__init__.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index d668537..7163ca8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,8 +4,10 @@ Changelog 0.5.x ~~~~~ -UNRELEASED ----------- +Version 0.5.2 +------------- + +Released on Jul 5, 2024. - Fix an issue where loading a thrift file in a sub-thread will cause an error with ``load_fp``. - Move static metadata from ``setup.py`` to ``pyproject.toml``. diff --git a/pyproject.toml b/pyproject.toml index 842c259..52f83da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools", "cython>=0.28.4,<4"] [project] name = "thriftpy2" -version = "0.5.2b2" +version = "0.5.2" description = "Pure python implementation of Apache Thrift." authors = [ {name = "ThriftPy Organization", email = "gotzehsing@gmail.com"}, diff --git a/thriftpy2/__init__.py b/thriftpy2/__init__.py index be8fb3c..42e0584 100644 --- a/thriftpy2/__init__.py +++ b/thriftpy2/__init__.py @@ -5,7 +5,7 @@ from .hook import install_import_hook, remove_import_hook from .parser import load, load_module, load_fp -__version__ = '0.5.2b2' +__version__ = '0.5.2' __python__ = sys.version_info __all__ = ["install_import_hook", "remove_import_hook", "load", "load_module", "load_fp"] From 8e226b12750829ee48abc1f883ea0fcf0b13f717 Mon Sep 17 00:00:00 2001 From: Hai Zhu <35182391+cocolato@users.noreply.github.com> Date: Wed, 10 Jul 2024 12:18:53 +0800 Subject: [PATCH 20/20] move dependencies from setup.py to project.toml (#291) --- pyproject.toml | 19 +++++++++++++++++-- setup.py | 26 +++++++------------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 52f83da..cbf07d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools", "cython>=0.28.4,<4"] +requires = ["setuptools", "cython>=0.28.4,<4", "toml"] [project] name = "thriftpy2" @@ -9,7 +9,6 @@ authors = [ {name = "ThriftPy Organization", email = "gotzehsing@gmail.com"}, ] dependencies = [ - "Cython>=3.0.10", "ply>=3.4,<4.0", "six~=1.15", ] @@ -37,3 +36,19 @@ classifiers = [ [project.urls] Homepage = "https://thriftpy2.readthedocs.io/" Source = "https://github.com/Thriftpy/thriftpy2" + +[project.optional-dependencies] +dev = [ + "flake8>=2.5", + "sphinx-rtd-theme>=0.1.9", + "sphinx>=1.3", + "pytest-reraise", + "pytest>=6.1.1,<8.2.0", + "tornado>=4.0,<7.0; python_version>='3.12'", + "tornado>=4.0,<6.0; python_version<'3.12'", +] + +tornado = [ + "tornado>=4.0,<7.0; python_version>='3.12'", + "tornado>=4.0,<6.0; python_version<'3.12'", +] diff --git a/setup.py b/setup.py index c52a3ea..e4741c6 100644 --- a/setup.py +++ b/setup.py @@ -3,37 +3,26 @@ import sys import platform +import toml +from os.path import join, dirname from setuptools import setup, find_packages, Extension -install_requires = [ - "ply>=3.4,<4.0", - "six~=1.15", -] - -tornado_requires = [ - "tornado>=4.0,<7.0; python_version>='3.12'", - "tornado>=4.0,<6.0; python_version<'3.12'", -] +meta = toml.load(join(dirname(__file__), 'pyproject.toml') ) +install_requires = meta["project"]["dependencies"] +dev_requires = meta["project"]["optional-dependencies"]["dev"] +tornado_requires = meta["project"]["optional-dependencies"]["tornado"] try: from tornado import version as tornado_version if tornado_version < '5.0': tornado_requires.append("toro>=0.6") + dev_requires.append("toro>=0.6") except ImportError: # tornado will now only get installed and we'll get the newer one pass -dev_requires = [ - "flake8>=2.5", - "sphinx-rtd-theme>=0.1.9", - "sphinx>=1.3", - "pytest-reraise", - "pytest>=6.1.1,<8.2.0", -] + tornado_requires - -cmdclass = {} ext_modules = [] # pypy detection @@ -76,7 +65,6 @@ "dev": dev_requires, "tornado": tornado_requires }, - cmdclass=cmdclass, ext_modules=ext_modules, include_package_data=True, )