From 4033a8843fee0f421eb995519dfb13d3eb79762c Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Wed, 15 May 2019 16:20:28 +0200 Subject: [PATCH 01/32] Add coment parsing for messge and enum --- .gitignore | 3 +- pyrobuf/parse_proto.py | 84 ++++++++++++++++++++++- pyrobuf/protobuf/templates/proto_pyx.tmpl | 36 ++++++++-- 3 files changed, 113 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 311da27..140ac80 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ tests/out/ tests/build/ /.tox pyrobuf/src/pyrobuf_defs.pxi -.pytest_cache \ No newline at end of file +.pytest_cache +.mypy_cache \ No newline at end of file diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index 92d8a3b..a1fe738 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -8,8 +8,8 @@ class Parser(object): # Tokens ordered roughly by priority tokens = [ - ('COMMENT_OL', r'\/\/.*?\n'), - ('COMMENT_ML', r'\/\*(?:.|[\r\n])*?\*\/'), + ('COMMENT_OL', r'\/\/(.*?)\n'), + ('COMMENT_ML', r'\/\*((?:.|[\r\n])*?)\*\/'), ('OPTION', r'option\s+((?:.|[\n\r])*?);'), ('IMPORT', r'import\s+"(.+?).proto"\s*;'), ('MESSAGE', r'message\s+([A-Za-z_][0-9A-Za-z_]*)'), @@ -41,6 +41,8 @@ class Parser(object): ] parsable_tokens = ( + 'COMMENT_OL', + 'COMMENT_ML', 'OPTION', 'SYNTAX', 'IMPORT', @@ -93,6 +95,24 @@ class Parser(object): 'bytes': 'BytesList' } + py_type_map = { + 'float': 'float', + 'double': 'float', + 'int32': 'int', + 'sint32': 'int', + 'sfixed32': 'int', + 'uint32': 'int', + 'fixed32': 'int', + 'int64': 'int', + 'sint64': 'int', + 'sfixed64': 'int', + 'uint64': 'int', + 'fixed64': 'int', + 'bool': 'bool', + 'string': 'str', + 'bytes': 'bytes' + } + c_type_map = { 'float': 'float', 'double': 'double', @@ -140,6 +160,7 @@ class Parser(object): def __init__(self, string): self.string = string self.lines = string.split('\n') + self._lastComment = None def tokenize(self, disabled_token_types): token_type_to_token_class = self.get_token_type_to_token_class_map() @@ -173,6 +194,22 @@ def tokenize(self, disabled_token_types): raise Exception("Unexpected character '{}' on line {}: '{}'".format( self.string[pos], line + 1, self.lines[line])) + def _handleComment(self, token, previous_token): + if token.token_type == 'MESSAGE' or token.token_type == 'ENUM' : + token.comment = self._lastComment + self._lastComment = None + + if token.token_type != "COMMENT_OL" and token.token_type != "COMMENT_ML": + return False + + # use only regular comments, ignore normal, development comments + if token.comment[0] == '*' or token.comment[0] == '/': + if previous_token.token_type == "FIELD" or previous_token.token_type == "ENUM_FIELD": + previous_token.comment = token.comment[1:].strip() + else: + self._lastComment = token.comment[1:].strip() + return True + def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): self.verify_parsable_tokens() tokens = self.tokenize(disabled_tokens) @@ -182,8 +219,12 @@ def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): messages = {} includes = includes or [] scope = {} + previous = self.LBrace(-1) for token in tokens: + if self._handleComment(token,previous): + continue + if token.token_type == 'OPTION': continue @@ -322,6 +363,10 @@ def _parse_message(self, current_message, tokens, messages, enums, imported_enum token.line + 1, self.lines[token.line]) for token in tokens: + if self._handleComment(token, previous): + previous = token + continue + if token.token_type == 'MESSAGE': token.full_name = current_message.full_name + token.name ret = self._parse_message(token, tokens, messages, enums.copy(), imported_enums) @@ -521,8 +566,13 @@ def _parse_enum(self, current, tokens, scope, current_message=None): token = next(tokens) assert token.token_type == 'LBRACE', "missing opening brace on line {}: '{}'".format( token.line + 1, self.lines[token.line]) + previous = self.LBrace(-1) for num, token in enumerate(tokens): + if self._handleComment(token,previous): + previous = token + continue + if token.token_type == 'ENUM_FIELD': if num == 0: if self.syntax == 3: @@ -548,7 +598,8 @@ def _parse_enum(self, current, tokens, scope, current_message=None): assert token.token_type == 'RBRACE', "unexpected {} token on line {}: '{}'".format( token.token_type, token.line + 1, self.lines[token.line]) return current - + + previous = token raise Exception("unexpected EOF on line {}: '{}'".format( token.line + 1, self.lines[token.line])) @@ -627,6 +678,19 @@ class Token(object): line = -1 type = None + class Comment_OL(Token): + token_type = 'COMMENT_OL' + def __init__(self, line, comment): + self.line = line + self.comment = comment + + class Comment_ML(Token): + token_type = 'COMMENT_ML' + def __init__(self, line, comment): + + self.line = line + self.comment = comment + class Option(Token): token_type = 'OPTION' @@ -654,6 +718,7 @@ class Message(Token): def __init__(self, line, name): self.line = line self.name = name + self.comment = None # full_name may later be overridden with parent hierarchy self.full_name = name self.messages = {} @@ -678,6 +743,8 @@ def __init__(self, line, ftype, name, index): self.modifier = None self.type = ftype self.name = name + # comment for attriute in class docstring + self.comment = None self.index = int(index) self.default = None self.packed = False @@ -695,6 +762,16 @@ def get_key(self): else: return (self.index << 3) | 0 + def get_python_type(self): + prefix ="" + if self.modifier == 'repeated': + prefix = "list of " + if self.type == "message": + return prefix + self.message_name + if self.type == "enum" : + return prefix + self.enum_name + return prefix + Parser.py_type_map[self.type] + class MapField(Token): token_type = 'MAP_FIELD' @@ -780,6 +857,7 @@ class Enum(Token): def __init__(self, line, name): self.line = line self.name = name + self.comment = None self.fields = {} self.default = None # full_name may later be overridden with parent hierarchy diff --git a/pyrobuf/protobuf/templates/proto_pyx.tmpl b/pyrobuf/protobuf/templates/proto_pyx.tmpl index 8cccf0d..aa60548 100644 --- a/pyrobuf/protobuf/templates/proto_pyx.tmpl +++ b/pyrobuf/protobuf/templates/proto_pyx.tmpl @@ -8,6 +8,7 @@ from pyrobuf_util cimport * import base64 import json import warnings +import enum {%- for import in imports %} from {{ import }}_proto cimport * @@ -21,8 +22,19 @@ from {{ import }}_proto cimport * {%- macro classdef(message) %} -cdef class {{ message.full_name }}: - +cdef class {{ message.full_name }}: + """ + {%- if message.comment != None %} + {{message.comment}} + {%- endif %} + {%- if message.fields.values()|first is defined %} + Attributes + ---------- + {%- for field in message.fields.values() %} + - `{{field.name}}`: {{field.get_python_type()}}{%- if field.modifier != 'required'%},optional{%- endif %} {% if field.comment != None %} \– {{field.comment}} {%- endif %} + {%- endfor %} + {%- endif %} + """ def __cinit__(self): self._listener = null_listener @@ -134,6 +146,9 @@ cdef class {{ message.full_name }}: {% for index, field in message.fields|dictsort() %} @property def {{ field.name }}(self): + {%- if field.comment != None %} + """{{field.comment}}""" + {%- endif %} {%- if field.deprecated|default(false) == true %} warnings.warn("field '{{ field.name }}' is deprecated", DeprecationWarning) {%- endif %} @@ -147,7 +162,7 @@ cdef class {{ message.full_name }}: return self._{{ field.name }} @{{ field.name }}.setter - def {{ field.name }}(self, value): + def {{ field.name }}(self, value): {%- if field.deprecated|default(false) == true %} warnings.warn("field '{{ field.name }}' is deprecated", DeprecationWarning) {%- endif %} @@ -1036,9 +1051,18 @@ class DecodeError(Exception): {%- endfor %} {%- macro enum_fields_def(enum) %} -{%- for field in enum.fields.values() %} -{{ field.name }} = _{{ field.full_name }} -{%- endfor %} +class {{enum.name}}(enum.Enum): + """ + {%- if enum.comment != None %} + {{enum.comment}} + {%- endif %} + {%for field in enum.fields.values()|sort(attribute='name') %} + - `{{field.name}}` {% if field.comment != None %} \– {{field.comment}} {%- endif %} + {%-endfor %} + """ + {%- for field in enum.fields.values() %} + {{ field.name }} = _{{ field.full_name }} {%- if field.comment != None %} # {{field.comment}}{%- endif %} + {%- endfor %} {%- endmacro %} {%- for enum in enums %} From 1c4ef13fec851bdb8ee7dd14eb0e503f2469c81b Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Mon, 20 May 2019 10:17:22 +0200 Subject: [PATCH 02/32] Update nested enum, enum in messages --- pyrobuf/protobuf/templates/proto_pyx.tmpl | 35 +++++++++++------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/pyrobuf/protobuf/templates/proto_pyx.tmpl b/pyrobuf/protobuf/templates/proto_pyx.tmpl index aa60548..c2621f7 100644 --- a/pyrobuf/protobuf/templates/proto_pyx.tmpl +++ b/pyrobuf/protobuf/templates/proto_pyx.tmpl @@ -14,9 +14,21 @@ import enum from {{ import }}_proto cimport * {%- endfor %} -{%- macro message_enum_fields_def(enum) %} - {%- for field in enum.fields.values() %} - {{ field.name }} = _{{ field.full_name }} + +{%- macro enum_fields_def(enum) %} +class {{enum.full_name}}(enum.Enum): + """ + {%- if enum.comment != None %} + {{enum.comment}} + {%- endif %} + Enum Constants + -------------- + {%for field in enum.fields.values()|sort(attribute='name') %} + - `{{field.name}}` {% if field.comment != None %} \– {{field.comment}} {%- endif %} + {%-endfor %} + """ + {%- for field in enum.fields.values() %} + {{ field.name }} = _{{ field.full_name }} {%- if field.comment != None %} # {{field.comment}}{%- endif %} {%- endfor %} {%- endmacro %} @@ -1022,7 +1034,7 @@ cdef class {{ message.full_name }}: {%- endfor %} {% for message_enum_name, message_enum in message.enums.items() %} - {{ message_enum_fields_def(message_enum) }} + {{ enum_fields_def(message_enum) }} {% endfor %} def Setters(self): @@ -1050,21 +1062,6 @@ class DecodeError(Exception): {{ classdef(message) }} {%- endfor %} -{%- macro enum_fields_def(enum) %} -class {{enum.name}}(enum.Enum): - """ - {%- if enum.comment != None %} - {{enum.comment}} - {%- endif %} - {%for field in enum.fields.values()|sort(attribute='name') %} - - `{{field.name}}` {% if field.comment != None %} \– {{field.comment}} {%- endif %} - {%-endfor %} - """ - {%- for field in enum.fields.values() %} - {{ field.name }} = _{{ field.full_name }} {%- if field.comment != None %} # {{field.comment}}{%- endif %} - {%- endfor %} -{%- endmacro %} - {%- for enum in enums %} {{ enum_fields_def(enum) }} {%- endfor %} From 9d284bd81826bb16b9b65230a6f96b261fb1d800 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Thu, 6 Jun 2019 10:43:53 +0200 Subject: [PATCH 03/32] fix set default parse enum, first token a comment --- pyrobuf/parse_proto.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index a1fe738..cd642cd 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -567,18 +567,20 @@ def _parse_enum(self, current, tokens, scope, current_message=None): assert token.token_type == 'LBRACE', "missing opening brace on line {}: '{}'".format( token.line + 1, self.lines[token.line]) previous = self.LBrace(-1) + setDefault = False - for num, token in enumerate(tokens): + for token in tokens: if self._handleComment(token,previous): previous = token continue if token.token_type == 'ENUM_FIELD': - if num == 0: + if (setDefault is False): if self.syntax == 3: assert token.value == 0, "expected zero as first enum element on line {}, got {}: '{}'".format( token.line + 1, token.value, self.lines[token.line]) current.default = token + setDefault = True token.full_name = "{}_{}".format(current.full_name, token.name) From bc253982bffeb54e2151f02c9ad3b30f614f344b Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 2 Jul 2019 10:40:01 +0200 Subject: [PATCH 04/32] add base package pyrogen --- pyrobuf/compile.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/pyrobuf/compile.py b/pyrobuf/compile.py index 4df2fef..9dc6781 100644 --- a/pyrobuf/compile.py +++ b/pyrobuf/compile.py @@ -5,6 +5,8 @@ from setuptools import setup from Cython.Build import cythonize +from Cython.Distutils import build_ext +from pathlib import Path from jinja2 import Environment, PackageLoader from pyrobuf.parse_proto import Parser, Proto3Parser @@ -17,6 +19,14 @@ _VM = sys.version_info.major +class BasePackagePatch_BuildExt(build_ext): + """ Create __init__.py for base package, after build + """ + def run(self): + build_ext.run(self) + filename = Path(self.build_lib).joinpath(self.package).joinpath('__init__.py') + filename.touch(exist_ok=True) + class Compiler(object): _env = Environment(loader=PackageLoader('pyrobuf.protobuf', 'templates')) @@ -91,9 +101,14 @@ def compile(self): self._package() setup(name='pyrobuf-generated', - ext_modules=cythonize(self._pyx_files, - include_path=self.include_path), - script_args=script_args) + ext_modules=cythonize(self._pyx_files, + include_path=self.include_path), + ext_package= 'pyrogen', + # This base package option is needed, to use mypy also as package: pyrogen-stubs + # I'm still unclear why __init__.py is not automatically generated. Maybe a bug? + # With this build_ext adaption it works. CT_02Jul19 + cmdclass= dict(build_ext=BasePackagePatch_BuildExt), + script_args=script_args) def extend(self, dist): self._compile_spec() From e73c28f6d35943aa76e1022c95d3e6b95bc7d18e Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Fri, 5 Jul 2019 13:12:56 +0200 Subject: [PATCH 05/32] add comment handling for oneof (test_oneof) --- pyrobuf/parse_proto.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index cd642cd..86dc117 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -474,6 +474,10 @@ def _parse_oneof(self, oneof_token, tokens, current_message, messages, enums, im # setting previous as a place holder for the inner fields parsing previous = self.LBrace(-1) for token in tokens: + if self._handleComment(token,previous): + previous = token + continue + if token.token_type == 'FIELD': # fields will be added to the proper message by the following # parser function that takes care of fields generally From a7b34bfd7b870d021fc44291cc38ea516a16b44f Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Fri, 5 Jul 2019 13:18:50 +0200 Subject: [PATCH 06/32] fix nested class enum defintion (test_setters) --- pyrobuf/protobuf/templates/proto_pyx.tmpl | 32 +++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pyrobuf/protobuf/templates/proto_pyx.tmpl b/pyrobuf/protobuf/templates/proto_pyx.tmpl index c2621f7..1ef449c 100644 --- a/pyrobuf/protobuf/templates/proto_pyx.tmpl +++ b/pyrobuf/protobuf/templates/proto_pyx.tmpl @@ -16,37 +16,37 @@ from {{ import }}_proto cimport * {%- macro enum_fields_def(enum) %} -class {{enum.full_name}}(enum.Enum): +class {{enum.full_name}}(enum.Enum): """ {%- if enum.comment != None %} - {{enum.comment}} + {{enum.comment}} {%- endif %} Enum Constants - -------------- - {%for field in enum.fields.values()|sort(attribute='name') %} + -------------- + {%for field in enum.fields.values()|sort(attribute='name') %} - `{{field.name}}` {% if field.comment != None %} \– {{field.comment}} {%- endif %} {%-endfor %} - """ - {%- for field in enum.fields.values() %} + """ + {%- for field in enum.fields.values() %} {{ field.name }} = _{{ field.full_name }} {%- if field.comment != None %} # {{field.comment}}{%- endif %} {%- endfor %} {%- endmacro %} {%- macro classdef(message) %} -cdef class {{ message.full_name }}: +cdef class {{ message.full_name }}: """ - {%- if message.comment != None %} - {{message.comment}} + {%- if message.comment != None %} + {{message.comment}} {%- endif %} - {%- if message.fields.values()|first is defined %} + {%- if message.fields.values()|first is defined %} Attributes ---------- {%- for field in message.fields.values() %} - `{{field.name}}`: {{field.get_python_type()}}{%- if field.modifier != 'required'%},optional{%- endif %} {% if field.comment != None %} \– {{field.comment}} {%- endif %} {%- endfor %} {%- endif %} - """ + """ def __cinit__(self): self._listener = null_listener @@ -174,7 +174,7 @@ cdef class {{ message.full_name }}: return self._{{ field.name }} @{{ field.name }}.setter - def {{ field.name }}(self, value): + def {{ field.name }}(self, value): {%- if field.deprecated|default(false) == true %} warnings.warn("field '{{ field.name }}' is deprecated", DeprecationWarning) {%- endif %} @@ -1033,10 +1033,6 @@ cdef class {{ message.full_name }}: yield self.{{ field.name }} {%- endfor %} - {% for message_enum_name, message_enum in message.enums.items() %} - {{ enum_fields_def(message_enum) }} - {% endfor %} - def Setters(self): """ Iterator over functions to set the fields in a message. @@ -1050,6 +1046,10 @@ cdef class {{ message.full_name }}: yield setter {%- endfor %} + {% for message_enum_name, message_enum in message.enums.items() %} +{{ enum_fields_def(message_enum) }} + {% endfor %} + {% for message_name, message_message in message.messages.items() %} {{ classdef(message_message) }} {% endfor %} From 8b400b7401598b26c4173d597629139024941cab Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Fri, 5 Jul 2019 13:19:31 +0200 Subject: [PATCH 07/32] upgrade tests for class enum --- tests/test_imported_enums.py | 20 ++++++++++---------- tests/test_merge_from.py | 9 +++++---- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/test_imported_enums.py b/tests/test_imported_enums.py index 6598bf1..8021df2 100644 --- a/tests/test_imported_enums.py +++ b/tests/test_imported_enums.py @@ -2,27 +2,27 @@ # These can't be imported until the test_imported_enums_proto module has been built. -CLOSE = None -MSG_ONE = None -ExposesInternalEnumConstantsMessage = None +Status = None +MessageID = None +ExposesInternalEnumConstantsMessageinternal_enum = None UsesImportedEnumsMessage = None class ImportedEnumsTest(unittest.TestCase): @classmethod def setUpClass(cls): - global CLOSE, MSG_ONE, ExposesInternalEnumConstantsMessage, UsesImportedEnumsMessage - from test_multi_messages_toplevel_enums_proto import MSG_ONE, CLOSE - from test_imported_enums_proto import UsesImportedEnumsMessage, ExposesInternalEnumConstantsMessage + global Status, MessageID, ExposesInternalEnumConstantsMessageinternal_enum, UsesImportedEnumsMessage + from test_multi_messages_toplevel_enums_proto import Status, MessageID + from test_imported_enums_proto import UsesImportedEnumsMessage, ExposesInternalEnumConstantsMessageinternal_enum def test_message_id_has_default_of_msg_one(self): message = UsesImportedEnumsMessage() - self.assertEqual(message.message_id, MSG_ONE) + self.assertEqual(message.message_id, MessageID.MSG_ONE.value) def test_status_has_default_of_close(self): message = UsesImportedEnumsMessage() - self.assertEqual(message.status, CLOSE) + self.assertEqual(message.status, Status.CLOSE.value) def test_internal_enum_constants_exposed(self): - self.assertEqual(ExposesInternalEnumConstantsMessage.INTERNAL, 0) - self.assertEqual(ExposesInternalEnumConstantsMessage.EXTERNAL, 1) + self.assertEqual(ExposesInternalEnumConstantsMessageinternal_enum.INTERNAL.value, 0) + self.assertEqual(ExposesInternalEnumConstantsMessageinternal_enum.EXTERNAL.value, 1) diff --git a/tests/test_merge_from.py b/tests/test_merge_from.py index f9720a5..98ebace 100644 --- a/tests/test_merge_from.py +++ b/tests/test_merge_from.py @@ -1,6 +1,7 @@ import unittest Test = None +TestEnumField = None TestSs1 = None TestSs3 = None @@ -8,8 +9,8 @@ class MergeFromTest(unittest.TestCase): @classmethod def setUpClass(cls): - global Test, TestSs1, TestSs3 - from test_message_proto import Test, TestSs1, TestSs3 + global Test, TestEnumField, TestSs1, TestSs3 + from test_message_proto import Test, TestEnumField, TestSs1, TestSs3 def test_merge_from_wrong_type_raises_type_error(self): dest = Test() @@ -49,9 +50,9 @@ def test_merge_from_does_set_scalar_field_that_is_set_in_source(self): def test_merge_from_does_set_enum_field_that_is_set_in_source(self): source = Test() dest = Test() - source.enum_field = Test.TEST_ENUM_FIELD_2 + source.enum_field = TestEnumField.TEST_ENUM_FIELD_2.value dest.MergeFrom(source) - self.assertEqual(dest.enum_field, Test.TEST_ENUM_FIELD_2) + self.assertEqual(dest.enum_field, TestEnumField.TEST_ENUM_FIELD_2.value) def test_merge_from_does_merge_message_field_that_is_set_in_source(self): source = TestSs3() From 8b666022f7763905f70b1830a71241e0a0eb7355 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 9 Jul 2019 13:27:32 +0200 Subject: [PATCH 08/32] fix pyrogen package creation --- pyrobuf/compile.py | 12 +++++------- pyrobuf/protobuf/templates/proto_pxd.tmpl | 2 +- pyrobuf/protobuf/templates/proto_pyx.tmpl | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pyrobuf/compile.py b/pyrobuf/compile.py index 9dc6781..6cb6eed 100644 --- a/pyrobuf/compile.py +++ b/pyrobuf/compile.py @@ -24,7 +24,7 @@ class BasePackagePatch_BuildExt(build_ext): """ def run(self): build_ext.run(self) - filename = Path(self.build_lib).joinpath(self.package).joinpath('__init__.py') + filename = Path(self.build_lib).joinpath('pyrogen').joinpath('__init__.py') filename.touch(exist_ok=True) class Compiler(object): @@ -37,7 +37,7 @@ def __init__(self, sources, out="out", build="build", install=False, proto3=False, force=False, package=None, includes=None, clean=False): self.sources = sources - self.out = out + self.out = os.path.join(out, 'pyrogen') self.build = build self.install = install self.force = force @@ -45,7 +45,7 @@ def __init__(self, sources, out="out", build="build", install=False, self.includes = includes or [] self.clean = clean here = os.path.dirname(os.path.abspath(__file__)) - self.include_path = [os.path.join(here, 'src'), self.out] + self.include_path = [os.path.join(here, 'src'), out] self._generated = set() self._messages = [] self._pyx_files = [] @@ -103,10 +103,6 @@ def compile(self): setup(name='pyrobuf-generated', ext_modules=cythonize(self._pyx_files, include_path=self.include_path), - ext_package= 'pyrogen', - # This base package option is needed, to use mypy also as package: pyrogen-stubs - # I'm still unclear why __init__.py is not automatically generated. Maybe a bug? - # With this build_ext adaption it works. CT_02Jul19 cmdclass= dict(build_ext=BasePackagePatch_BuildExt), script_args=script_args) @@ -125,6 +121,8 @@ def extend(self, dist): def _compile_spec(self): try: os.makedirs(self.out) + filename = Path(self.out).joinpath('__init__.py') + filename.touch(exist_ok=True) except _FileExistsError: pass diff --git a/pyrobuf/protobuf/templates/proto_pxd.tmpl b/pyrobuf/protobuf/templates/proto_pxd.tmpl index 63a2473..37fff57 100644 --- a/pyrobuf/protobuf/templates/proto_pxd.tmpl +++ b/pyrobuf/protobuf/templates/proto_pxd.tmpl @@ -8,7 +8,7 @@ from pyrobuf_util cimport * import json {%- for import in imports %} -from {{ import }}_proto cimport * +from pyrogen.{{ import }}_proto cimport * {%- endfor %} {%- macro classdef(message) %} diff --git a/pyrobuf/protobuf/templates/proto_pyx.tmpl b/pyrobuf/protobuf/templates/proto_pyx.tmpl index 1ef449c..3e86488 100644 --- a/pyrobuf/protobuf/templates/proto_pyx.tmpl +++ b/pyrobuf/protobuf/templates/proto_pyx.tmpl @@ -11,7 +11,7 @@ import warnings import enum {%- for import in imports %} -from {{ import }}_proto cimport * +from pyrogen.{{ import }}_proto cimport * {%- endfor %} From 389fed7344d0782885c95d686ce2cedcadd718e9 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 9 Jul 2019 13:29:46 +0200 Subject: [PATCH 09/32] upgrade tests for package --- tests/create_message.py | 2 +- tests/test_custom_options.py | 2 +- tests/test_deprecated_field.py | 2 +- tests/test_field_defaults.py | 4 ++-- tests/test_has_field.py | 4 ++-- tests/test_has_field_many.py | 2 +- tests/test_imported_enums.py | 4 ++-- tests/test_is_initialized.py | 2 +- tests/test_issue_11.py | 4 ++-- tests/test_issue_69.py | 4 ++-- tests/test_items.py | 2 +- tests/test_merge_from.py | 2 +- tests/test_message_field_types.py | 2 +- tests/test_message_init_kwargs.py | 2 +- tests/test_message_with_no_fields.py | 2 +- tests/test_nested_issue55.py | 2 +- tests/test_repeated_enum.py | 2 +- tests/test_signed_integer.py | 12 ++++++------ tests/test_typed_lists.py | 4 ++-- tests/test_unicode_strings.py | 2 +- tests/test_unknown_field.py | 2 +- 21 files changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/create_message.py b/tests/create_message.py index 332c3a5..65ca109 100644 --- a/tests/create_message.py +++ b/tests/create_message.py @@ -7,7 +7,7 @@ def create_an_test(): # print LIB - import test_message_proto as an_test + import pyrogen.test_message_proto as an_test test = an_test.Test() test.timestamp = 539395200 test.field = 10689 diff --git a/tests/test_custom_options.py b/tests/test_custom_options.py index cd9a469..d7d11c4 100644 --- a/tests/test_custom_options.py +++ b/tests/test_custom_options.py @@ -8,7 +8,7 @@ class TestCustomOptionsTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestCustomOptions - from test_custom_options_proto import TestCustomOptions + from pyrogen.test_custom_options_proto import TestCustomOptions def test_fields(self): test = TestCustomOptions() diff --git a/tests/test_deprecated_field.py b/tests/test_deprecated_field.py index 078ce7b..a698fe1 100644 --- a/tests/test_deprecated_field.py +++ b/tests/test_deprecated_field.py @@ -4,7 +4,7 @@ class DecimalDefaultsTest(unittest.TestCase): def setUp(self): - from test_message_field_types_proto import TestDeprecatedField + from pyrogen.test_message_field_types_proto import TestDeprecatedField warnings.filterwarnings("error") self.message = TestDeprecatedField() diff --git a/tests/test_field_defaults.py b/tests/test_field_defaults.py index 083bf13..eb479ee 100644 --- a/tests/test_field_defaults.py +++ b/tests/test_field_defaults.py @@ -9,7 +9,7 @@ class DecimalDefaultsTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestDecimalDefaultsMessage - from test_field_defaults_proto import TestDecimalDefaultsMessage + from pyrogen.test_field_defaults_proto import TestDecimalDefaultsMessage def setUp(self): self.message = TestDecimalDefaultsMessage() @@ -61,7 +61,7 @@ class StringDefaultsTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestStringDefaultsMessage - from test_field_defaults_proto import TestStringDefaultsMessage + from pyrogen.test_field_defaults_proto import TestStringDefaultsMessage def setUp(self): self.message = TestStringDefaultsMessage() diff --git a/tests/test_has_field.py b/tests/test_has_field.py index 0c9a17c..7e3eadd 100644 --- a/tests/test_has_field.py +++ b/tests/test_has_field.py @@ -11,8 +11,8 @@ class HasFieldTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestRef, TestSs1, TestSs1Thing - from test_message_proto import Test, TestSs1, TestSs1Thing - from test_ref_message_proto import TestRef + from pyrogen.test_ref_message_proto import TestRef + from pyrogen.test_message_proto import Test, TestSs1, TestSs1Thing def test_has_field_for_repeated_field_raises_value_error(self): message = Test() diff --git a/tests/test_has_field_many.py b/tests/test_has_field_many.py index 05b9ee0..77823af 100644 --- a/tests/test_has_field_many.py +++ b/tests/test_has_field_many.py @@ -1,5 +1,5 @@ def test_has_field(): - from test_many_fields_proto import TestManyFields + from pyrogen.test_many_fields_proto import TestManyFields test = TestManyFields() # Assert HasField false on clean message diff --git a/tests/test_imported_enums.py b/tests/test_imported_enums.py index 8021df2..6962daf 100644 --- a/tests/test_imported_enums.py +++ b/tests/test_imported_enums.py @@ -12,8 +12,8 @@ class ImportedEnumsTest(unittest.TestCase): @classmethod def setUpClass(cls): global Status, MessageID, ExposesInternalEnumConstantsMessageinternal_enum, UsesImportedEnumsMessage - from test_multi_messages_toplevel_enums_proto import Status, MessageID - from test_imported_enums_proto import UsesImportedEnumsMessage, ExposesInternalEnumConstantsMessageinternal_enum + from pyrogen.test_multi_messages_toplevel_enums_proto import Status, MessageID + from pyrogen.test_imported_enums_proto import UsesImportedEnumsMessage, ExposesInternalEnumConstantsMessageinternal_enum def test_message_id_has_default_of_msg_one(self): message = UsesImportedEnumsMessage() diff --git a/tests/test_is_initialized.py b/tests/test_is_initialized.py index 1c57322..b913739 100644 --- a/tests/test_is_initialized.py +++ b/tests/test_is_initialized.py @@ -10,7 +10,7 @@ class MergeFromTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestIsInitialized, SubMessage, TestWithRequiredSubMessage - from test_is_initialized_proto import TestIsInitialized, SubMessage, TestWithRequiredSubMessage + from pyrogen.test_is_initialized_proto import TestIsInitialized, SubMessage, TestWithRequiredSubMessage def test_new_message_is_not_initialized(self): message = TestIsInitialized() diff --git a/tests/test_issue_11.py b/tests/test_issue_11.py index 65bbfa4..8dad0e8 100644 --- a/tests/test_issue_11.py +++ b/tests/test_issue_11.py @@ -1,12 +1,12 @@ def test_before_overflow(): - from issue_11_proto import A + from pyrogen.issue_11_proto import A a = A() a.a0 = 0x7FFFFFFF assert A.FromString(a.SerializeToString()).a0 == 2147483647 def test_after_overflow(): - from issue_11_proto import A + from pyrogen.issue_11_proto import A a = A() a.a0 = 0x80000000 assert A.FromString(a.SerializeToString()).a0 == 2147483648 diff --git a/tests/test_issue_69.py b/tests/test_issue_69.py index 2c2dc7f..6a3ce37 100644 --- a/tests/test_issue_69.py +++ b/tests/test_issue_69.py @@ -1,10 +1,10 @@ def test_message_a(): - from issue_69_proto import A, B + from pyrogen.issue_69_proto import A, B a = A() assert type(a.b.a.b) == B def test_message_b(): - from issue_69_proto import A, B + from pyrogen.issue_69_proto import A, B b = B() assert type(b.a.b.a) == A diff --git a/tests/test_items.py b/tests/test_items.py index d562d5e..d3d63ee 100644 --- a/tests/test_items.py +++ b/tests/test_items.py @@ -8,7 +8,7 @@ class ItemsTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test - from test_message_proto import Test + from pyrogen.test_message_proto import Test def test_fields(self): test = create_an_test() diff --git a/tests/test_merge_from.py b/tests/test_merge_from.py index 98ebace..f16c692 100644 --- a/tests/test_merge_from.py +++ b/tests/test_merge_from.py @@ -10,7 +10,7 @@ class MergeFromTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestEnumField, TestSs1, TestSs3 - from test_message_proto import Test, TestEnumField, TestSs1, TestSs3 + from pyrogen.test_message_proto import Test, TestEnumField, TestSs1, TestSs3 def test_merge_from_wrong_type_raises_type_error(self): dest = Test() diff --git a/tests/test_message_field_types.py b/tests/test_message_field_types.py index 6511bca..157e7f5 100644 --- a/tests/test_message_field_types.py +++ b/tests/test_message_field_types.py @@ -7,7 +7,7 @@ class MessageFieldTypesTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestFieldTypes - from test_message_field_types_proto import TestFieldTypes + from pyrogen.test_message_field_types_proto import TestFieldTypes def test_bytes_payload_serialize_to_string(self): message = TestFieldTypes() diff --git a/tests/test_message_init_kwargs.py b/tests/test_message_init_kwargs.py index f1a30d5..d37ed04 100644 --- a/tests/test_message_init_kwargs.py +++ b/tests/test_message_init_kwargs.py @@ -9,7 +9,7 @@ class MergeFromTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestSs1, TestSs3 - from test_message_proto import Test, TestSs1, TestSs3 + from pyrogen.test_message_proto import Test, TestSs1, TestSs3 def test_message_init_with_bad_keyword_arg_raises_value_error(self): self.assertRaises(ValueError, Test, non_existant=3) diff --git a/tests/test_message_with_no_fields.py b/tests/test_message_with_no_fields.py index 75f01f5..6da83cf 100644 --- a/tests/test_message_with_no_fields.py +++ b/tests/test_message_with_no_fields.py @@ -7,7 +7,7 @@ class EmptyMessageTest(unittest.TestCase): @classmethod def setUpClass(cls): global EmptyMessageWithNoFields - from test_message_field_types_proto import EmptyMessageWithNoFields + from pyrogen.test_message_field_types_proto import EmptyMessageWithNoFields def setUp(self): self.message = EmptyMessageWithNoFields() diff --git a/tests/test_nested_issue55.py b/tests/test_nested_issue55.py index ca4d266..a4b64cc 100644 --- a/tests/test_nested_issue55.py +++ b/tests/test_nested_issue55.py @@ -16,7 +16,7 @@ def setUpClass(cls): global M global MN global MN2 - from test_nested_issue55_proto import M, MN, MN2 + from pyrogen.test_nested_issue55_proto import M, MN, MN2 def test_use_nested(self): """ diff --git a/tests/test_repeated_enum.py b/tests/test_repeated_enum.py index 679d59a..3a0da2c 100644 --- a/tests/test_repeated_enum.py +++ b/tests/test_repeated_enum.py @@ -8,7 +8,7 @@ class RepeatedEnumTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestRepeatedEnum - from test_repeated_enum_proto import TestRepeatedEnum + from pyrogen.test_repeated_enum_proto import TestRepeatedEnum def test_repeated_enum_serde(self): message1 = TestRepeatedEnum() diff --git a/tests/test_signed_integer.py b/tests/test_signed_integer.py index da9f663..a79814f 100644 --- a/tests/test_signed_integer.py +++ b/tests/test_signed_integer.py @@ -18,7 +18,7 @@ def gen_rand_uint64(): def test_int32(): - from test_signed_integer_proto import Int + from pyrogen.test_signed_integer_proto import Int a = Int() v = gen_rand_int32() a.Int32 = v @@ -26,7 +26,7 @@ def test_int32(): def test_int64(): - from test_signed_integer_proto import Int + from pyrogen.test_signed_integer_proto import Int a = Int() v = gen_rand_int64() a.Int64 = v @@ -34,7 +34,7 @@ def test_int64(): def test_uint32(): - from test_signed_integer_proto import Int + from pyrogen.test_signed_integer_proto import Int a = Int() v = gen_rand_uint32() a.UInt32 = v @@ -42,7 +42,7 @@ def test_uint32(): def test_uint64(): - from test_signed_integer_proto import Int + from pyrogen.test_signed_integer_proto import Int a = Int() v = gen_rand_uint64() a.UInt64 = v @@ -50,7 +50,7 @@ def test_uint64(): def test_sint32(): - from test_signed_integer_proto import Int + from pyrogen.test_signed_integer_proto import Int a = Int() v = gen_rand_int32() a.SInt32 = v @@ -58,7 +58,7 @@ def test_sint32(): def test_sint64(): - from test_signed_integer_proto import Int + from pyrogen.test_signed_integer_proto import Int a = Int() v = gen_rand_int64() a.SInt64 = v diff --git a/tests/test_typed_lists.py b/tests/test_typed_lists.py index 6e5e3a7..239aed2 100644 --- a/tests/test_typed_lists.py +++ b/tests/test_typed_lists.py @@ -8,8 +8,8 @@ class MergeFromTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestRef - from test_message_proto import Test - from test_ref_message_proto import TestRef + from pyrogen.test_ref_message_proto import TestRef + from pyrogen.test_message_proto import Test def test_extend_with_generic_list(self): t = Test() diff --git a/tests/test_unicode_strings.py b/tests/test_unicode_strings.py index 1c2ea03..14b695d 100644 --- a/tests/test_unicode_strings.py +++ b/tests/test_unicode_strings.py @@ -11,7 +11,7 @@ class TestUnicodeStrings(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestSs1, TestFieldTypes - from test_message_proto import Test, TestSs1 + from pyrogen.test_message_proto import Test, TestSs1 def test_unicode_string_parse_from_string(self): message = Test.FromString(b'\x1a\x08\xce\x91\xce\x92\xce\x93\xce\x94') diff --git a/tests/test_unknown_field.py b/tests/test_unknown_field.py index cada5dd..a479f28 100644 --- a/tests/test_unknown_field.py +++ b/tests/test_unknown_field.py @@ -15,7 +15,7 @@ class MessageTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestTruncated - from test_truncated_proto import TestTruncated + from pyrogen.test_truncated_proto import TestTruncated def test_deser(self): test = TestTruncated() From 8a1b6eede498b95716c9dad081c6bc419960ffd1 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 9 Jul 2019 14:11:30 +0200 Subject: [PATCH 10/32] remove whitespaces, fix typo --- pyrobuf/parse_proto.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index 86dc117..9df289f 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -195,11 +195,11 @@ def tokenize(self, disabled_token_types): self.string[pos], line + 1, self.lines[line])) def _handleComment(self, token, previous_token): - if token.token_type == 'MESSAGE' or token.token_type == 'ENUM' : - token.comment = self._lastComment + if token.token_type == 'MESSAGE' or token.token_type == 'ENUM': + token.comment = self._lastComment self._lastComment = None - if token.token_type != "COMMENT_OL" and token.token_type != "COMMENT_ML": + if token.token_type != "COMMENT_OL" and token.token_type != "COMMENT_ML": return False # use only regular comments, ignore normal, development comments @@ -219,10 +219,10 @@ def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): messages = {} includes = includes or [] scope = {} - previous = self.LBrace(-1) + previous = self.LBrace(-1) for token in tokens: - if self._handleComment(token,previous): + if self._handleComment(token, previous): continue if token.token_type == 'OPTION': @@ -686,14 +686,13 @@ class Token(object): class Comment_OL(Token): token_type = 'COMMENT_OL' - def __init__(self, line, comment): + def __init__(self, line, comment): self.line = line self.comment = comment class Comment_ML(Token): token_type = 'COMMENT_ML' def __init__(self, line, comment): - self.line = line self.comment = comment @@ -749,7 +748,7 @@ def __init__(self, line, ftype, name, index): self.modifier = None self.type = ftype self.name = name - # comment for attriute in class docstring + # comment for attribute in class docstring self.comment = None self.index = int(index) self.default = None @@ -775,7 +774,7 @@ def get_python_type(self): if self.type == "message": return prefix + self.message_name if self.type == "enum" : - return prefix + self.enum_name + return prefix + self.enum_name return prefix + Parser.py_type_map[self.type] class MapField(Token): From 6d3061a0e3d57935bbf0ed6011965ad2161349f2 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 9 Jul 2019 14:17:43 +0200 Subject: [PATCH 11/32] add version prefix vistec --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c8bd9c5..d32ddcc 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.0" +VERSION = "0.9.0.vistec" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" From a3e297dc54a99f5a7c02c636bdf3b755b89eee0d Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 20 Aug 2019 09:16:02 +0200 Subject: [PATCH 12/32] fix empty comment line --- pyrobuf/parse_proto.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index 9df289f..e76912f 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -203,7 +203,7 @@ def _handleComment(self, token, previous_token): return False # use only regular comments, ignore normal, development comments - if token.comment[0] == '*' or token.comment[0] == '/': + if len(token.comment) and ( token.comment[0] == '*' or token.comment[0] == '/'): if previous_token.token_type == "FIELD" or previous_token.token_type == "ENUM_FIELD": previous_token.comment = token.comment[1:].strip() else: diff --git a/setup.py b/setup.py index d32ddcc..5015eb8 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.0.vistec" +VERSION = "0.9.0.2" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" From 5facef089475e68037856cc2982631ab15a6f314 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 20 Aug 2019 09:34:49 +0200 Subject: [PATCH 13/32] add handling comments with double quotes --- pyrobuf/protobuf/templates/proto_pyx.tmpl | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyrobuf/protobuf/templates/proto_pyx.tmpl b/pyrobuf/protobuf/templates/proto_pyx.tmpl index 3e86488..68396a5 100644 --- a/pyrobuf/protobuf/templates/proto_pyx.tmpl +++ b/pyrobuf/protobuf/templates/proto_pyx.tmpl @@ -159,7 +159,7 @@ cdef class {{ message.full_name }}: @property def {{ field.name }}(self): {%- if field.comment != None %} - """{{field.comment}}""" + """{{field.comment}} """ {%- endif %} {%- if field.deprecated|default(false) == true %} warnings.warn("field '{{ field.name }}' is deprecated", DeprecationWarning) diff --git a/setup.py b/setup.py index 5015eb8..cfa016c 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.0.2" +VERSION = "0.9.0.3" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" From ef933e37108b5cfbc23fd40ada8de66f6a5ec90c Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 20 Aug 2019 13:07:43 +0200 Subject: [PATCH 14/32] upgrade field comment parsing --- pyrobuf/parse_proto.py | 17 ++++++++++------- setup.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index e76912f..5a9ffad 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -195,18 +195,21 @@ def tokenize(self, disabled_token_types): self.string[pos], line + 1, self.lines[line])) def _handleComment(self, token, previous_token): - if token.token_type == 'MESSAGE' or token.token_type == 'ENUM': + if token.token_type == 'MESSAGE' or token.token_type == 'FIELD' or token.token_type == 'ENUM' or token.token_type == 'ENUM_FIELD': token.comment = self._lastComment - self._lastComment = None + if token.token_type != 'MODIFIER': + self._lastComment = None if token.token_type != "COMMENT_OL" and token.token_type != "COMMENT_ML": return False - # use only regular comments, ignore normal, development comments - if len(token.comment) and ( token.comment[0] == '*' or token.comment[0] == '/'): - if previous_token.token_type == "FIELD" or previous_token.token_type == "ENUM_FIELD": - previous_token.comment = token.comment[1:].strip() - else: + if token.comment: + # use only regular comments, ignore normal, development comments + if token.comment[0] == '*' or token.comment[0] == '/': + if previous_token.token_type == "FIELD" or previous_token.token_type == "ENUM_FIELD": + if token.line == previous_token.line: + previous_token.comment = token.comment[1:].strip() + return True self._lastComment = token.comment[1:].strip() return True diff --git a/setup.py b/setup.py index cfa016c..1770c8a 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.0.3" +VERSION = "0.9.0.4" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" From 4acf65a6cd6bbc179d99dc701560e33f76e91f57 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Wed, 21 Aug 2019 09:05:40 +0200 Subject: [PATCH 15/32] upgrade block comment parsing --- pyrobuf/parse_proto.py | 34 ++++++++++++++++++------- setup.py | 2 +- tests/proto/test_message_comments.proto | 26 +++++++++++++++++++ tests/test_message_comments.py | 19 ++++++++++++++ 4 files changed, 71 insertions(+), 10 deletions(-) create mode 100644 tests/proto/test_message_comments.proto create mode 100644 tests/test_message_comments.py diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index 5a9ffad..a8b36c2 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -194,7 +194,23 @@ def tokenize(self, disabled_token_types): raise Exception("Unexpected character '{}' on line {}: '{}'".format( self.string[pos], line + 1, self.lines[line])) - def _handleComment(self, token, previous_token): + def _regular_comment(self, comment): + if comment[0] == '*': + # remove asterisk and trim block comment + block_comment = "" + for line in comment.splitlines(): + resline = line.strip() + if resline: + if resline[0] == '*': + block_comment += resline[1:].lstrip() + else: + block_comment += resline + block_comment += '\n' + return block_comment.strip() + + return comment[1:].strip() + + def _handle_comment(self, token, previous_token): if token.token_type == 'MESSAGE' or token.token_type == 'FIELD' or token.token_type == 'ENUM' or token.token_type == 'ENUM_FIELD': token.comment = self._lastComment if token.token_type != 'MODIFIER': @@ -208,9 +224,9 @@ def _handleComment(self, token, previous_token): if token.comment[0] == '*' or token.comment[0] == '/': if previous_token.token_type == "FIELD" or previous_token.token_type == "ENUM_FIELD": if token.line == previous_token.line: - previous_token.comment = token.comment[1:].strip() + previous_token.comment = self._regular_comment(token.comment) return True - self._lastComment = token.comment[1:].strip() + self._lastComment = self._regular_comment(token.comment) return True def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): @@ -225,7 +241,7 @@ def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): previous = self.LBrace(-1) for token in tokens: - if self._handleComment(token, previous): + if self._handle_comment(token, previous): continue if token.token_type == 'OPTION': @@ -366,7 +382,7 @@ def _parse_message(self, current_message, tokens, messages, enums, imported_enum token.line + 1, self.lines[token.line]) for token in tokens: - if self._handleComment(token, previous): + if self._handle_comment(token, previous): previous = token continue @@ -477,7 +493,7 @@ def _parse_oneof(self, oneof_token, tokens, current_message, messages, enums, im # setting previous as a place holder for the inner fields parsing previous = self.LBrace(-1) for token in tokens: - if self._handleComment(token,previous): + if self._handle_comment(token,previous): previous = token continue @@ -573,11 +589,11 @@ def _parse_enum(self, current, tokens, scope, current_message=None): token = next(tokens) assert token.token_type == 'LBRACE', "missing opening brace on line {}: '{}'".format( token.line + 1, self.lines[token.line]) - previous = self.LBrace(-1) + previous = self.LBrace(-1) setDefault = False for token in tokens: - if self._handleComment(token,previous): + if self._handle_comment(token,previous): previous = token continue @@ -607,7 +623,7 @@ def _parse_enum(self, current, tokens, scope, current_message=None): assert token.token_type == 'RBRACE', "unexpected {} token on line {}: '{}'".format( token.token_type, token.line + 1, self.lines[token.line]) return current - + previous = token raise Exception("unexpected EOF on line {}: '{}'".format( token.line + 1, self.lines[token.line])) diff --git a/setup.py b/setup.py index 1770c8a..31d3fd5 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.0.4" +VERSION = "0.9.0.5" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" diff --git a/tests/proto/test_message_comments.proto b/tests/proto/test_message_comments.proto new file mode 100644 index 0000000..e385a51 --- /dev/null +++ b/tests/proto/test_message_comments.proto @@ -0,0 +1,26 @@ +// normal development comments +// next lines is a empty line comment syntax test +// + +/// Message with line comment +message TestLineMessageComment { +} + +/** Message with block comment in one line */ +message TestBlockMessageCommentLine { +} + +/** + * Message with block comment + */ +message TestBlockMessageComment { +} + +/** +* Message with multiline block comment +* +* More details +* "stop" +*/ +message TestBlockMessageComments{ +} diff --git a/tests/test_message_comments.py b/tests/test_message_comments.py new file mode 100644 index 0000000..1ee1ab6 --- /dev/null +++ b/tests/test_message_comments.py @@ -0,0 +1,19 @@ +import unittest + +class MessageComments(unittest.TestCase): + + def test_message_block_comment_line(self): + from pyrogen.test_message_comments_proto import TestBlockMessageCommentLine + self.assertEqual(TestBlockMessageCommentLine.__doc__, '\n Message with block comment in one line\n ') + + def test_message_block_comment(self): + from pyrogen.test_message_comments_proto import TestBlockMessageComment + self.assertEqual(TestBlockMessageComment.__doc__, '\n Message with block comment\n ') + + def test_message_line_comment(self): + from pyrogen.test_message_comments_proto import TestLineMessageComment + self.assertEqual(TestLineMessageComment.__doc__, '\n Message with line comment\n ') + + def test_message_block_comment_multi(self): + from pyrogen.test_message_comments_proto import TestBlockMessageComments + self.assertEqual(TestBlockMessageComments.__doc__, '\n Message with multiline block comment\n\nMore details\n"stop"\n ') \ No newline at end of file From eb88829a96e0cfa0d5aa2f63a9febc418d148af7 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Wed, 21 Aug 2019 10:59:36 +0200 Subject: [PATCH 16/32] add field message comment tests --- tests/proto/test_message_comments.proto | 17 +++++++++++++ tests/test_message_comments.py | 33 ++++++++++++++----------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/tests/proto/test_message_comments.proto b/tests/proto/test_message_comments.proto index e385a51..a03f80d 100644 --- a/tests/proto/test_message_comments.proto +++ b/tests/proto/test_message_comments.proto @@ -24,3 +24,20 @@ message TestBlockMessageComment { */ message TestBlockMessageComments{ } + +message TestFieldMessageCommentStandard{ + required int32 hello = 1; /// Hello comment +} + +message TestFieldMessageCommentBefore{ + /// Hello comment before definition + required int32 hello = 1; +} + +message TestFieldMessageCommentLong{ + /** + * Hello long block comment + * with several lines + */ + required int32 hello = 1; +} \ No newline at end of file diff --git a/tests/test_message_comments.py b/tests/test_message_comments.py index 1ee1ab6..2112069 100644 --- a/tests/test_message_comments.py +++ b/tests/test_message_comments.py @@ -1,19 +1,24 @@ import unittest class MessageComments(unittest.TestCase): - - def test_message_block_comment_line(self): - from pyrogen.test_message_comments_proto import TestBlockMessageCommentLine - self.assertEqual(TestBlockMessageCommentLine.__doc__, '\n Message with block comment in one line\n ') - - def test_message_block_comment(self): - from pyrogen.test_message_comments_proto import TestBlockMessageComment - self.assertEqual(TestBlockMessageComment.__doc__, '\n Message with block comment\n ') - - def test_message_line_comment(self): + def test_message_block_comment_line(self): + from pyrogen.test_message_comments_proto import TestBlockMessageCommentLine + self.assertEqual(TestBlockMessageCommentLine.__doc__, '\n Message with block comment in one line\n ') + def test_message_block_comment(self): + from pyrogen.test_message_comments_proto import TestBlockMessageComment + self.assertEqual(TestBlockMessageComment.__doc__, '\n Message with block comment\n ') + def test_message_line_comment(self): from pyrogen.test_message_comments_proto import TestLineMessageComment self.assertEqual(TestLineMessageComment.__doc__, '\n Message with line comment\n ') - - def test_message_block_comment_multi(self): - from pyrogen.test_message_comments_proto import TestBlockMessageComments - self.assertEqual(TestBlockMessageComments.__doc__, '\n Message with multiline block comment\n\nMore details\n"stop"\n ') \ No newline at end of file + def test_message_block_comment_multi(self): + from pyrogen.test_message_comments_proto import TestBlockMessageComments + self.assertEqual(TestBlockMessageComments.__doc__, '\n Message with multiline block comment\n\nMore details\n"stop"\n ') + def test_field_message_comment_standard(self): + from pyrogen.test_message_comments_proto import TestFieldMessageCommentStandard + self.assertEqual(TestFieldMessageCommentStandard.hello.__doc__, 'Hello comment ') + def test_field_message_comment_before(self): + from pyrogen.test_message_comments_proto import TestFieldMessageCommentBefore + self.assertEqual(TestFieldMessageCommentBefore.hello.__doc__, 'Hello comment before definition ') + def test_field_message_comment_long(self): + from pyrogen.test_message_comments_proto import TestFieldMessageCommentLong + self.assertEqual(TestFieldMessageCommentLong.hello.__doc__, 'Hello long block comment\nwith several lines ') From 4ac9dd7239a941aaa5111e9d4ec001c8d871715e Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Mon, 21 Oct 2019 12:41:58 +0200 Subject: [PATCH 17/32] fix docstring class and enum --- pyrobuf/protobuf/templates/proto_pyx.tmpl | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyrobuf/protobuf/templates/proto_pyx.tmpl b/pyrobuf/protobuf/templates/proto_pyx.tmpl index 68396a5..fcb2514 100644 --- a/pyrobuf/protobuf/templates/proto_pyx.tmpl +++ b/pyrobuf/protobuf/templates/proto_pyx.tmpl @@ -24,7 +24,7 @@ class {{enum.full_name}}(enum.Enum): Enum Constants -------------- {%for field in enum.fields.values()|sort(attribute='name') %} - - `{{field.name}}` {% if field.comment != None %} \– {{field.comment}} {%- endif %} + - `{{field.name}}` {% if field.comment != None %} – {{field.comment}} {%- endif %} {%-endfor %} """ {%- for field in enum.fields.values() %} @@ -43,7 +43,7 @@ cdef class {{ message.full_name }}: Attributes ---------- {%- for field in message.fields.values() %} - - `{{field.name}}`: {{field.get_python_type()}}{%- if field.modifier != 'required'%},optional{%- endif %} {% if field.comment != None %} \– {{field.comment}} {%- endif %} + - `{{field.name}}:` {{field.get_python_type()}}{%- if field.modifier != 'required'%},optional{%- endif %} {% if field.comment != None %} – {{field.comment}} {%- endif %} {%- endfor %} {%- endif %} """ diff --git a/setup.py b/setup.py index 31d3fd5..a44d504 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.0.5" +VERSION = "0.9.0.6" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" From c2faed38fd7a19e8edbb65aeaa228d8164f14366 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 5 Nov 2019 10:32:36 +0100 Subject: [PATCH 18/32] Add hex value parsing to set default --- pyrobuf/parse_proto.py | 14 +++++++++++++- setup.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index a8b36c2..bf9750b 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -33,6 +33,7 @@ class Parser(object): ('COMMA', r','), ('SKIP', r'\s'), ('SEMICOLON', r';'), + ('HEXVALUE', r'(0x[0-9A-Fa-f]+)'), ('NUMERIC', r'(-?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?)'), ('STRING', r'("(?:\\.|[^"\\])*"|\'(?:\\.|[^"\\])*\')'), ('BOOLEAN', r'(true|false)'), @@ -546,8 +547,12 @@ def _parse_default(self, field, tokens): # This will get updated later field.default = token.full_name return + elif token.token_type == 'HEXVALUE': + assert field.type in self.scalars.difference({'bool', 'enum'}), \ + "attempting to set hex value as default for non-numeric field on line {}: '{}'".format( + token.line + 1, self.lines[token.line]) elif token.token_type == 'NUMERIC': - assert field.type in self.scalars, \ + assert field.type in self.scalars.difference({'bool', 'enum'}), \ "attempting to set numeric as default for non-numeric field on line {}: '{}'".format( token.line + 1, self.lines[token.line]) if field.type not in self.floats: @@ -907,6 +912,13 @@ def __init__(self, line, value): self.line = line self.value = float(value) + class HexValue(Token): + token_type = 'HEXVALUE' + + def __init__(self, line, value): + self.line = line + self.value = int(value, 16) + class String(Token): token_type = 'STRING' diff --git a/setup.py b/setup.py index a44d504..08fc794 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.0.6" +VERSION = "0.9.0.7" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" From 711b193295f19a0fba194d96f087b00860649e14 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Mon, 27 Jul 2020 12:26:54 +0200 Subject: [PATCH 19/32] take back import base package pyrogen in tests --- tests/conftest.py | 4 ++-- tests/create_message.py | 2 +- tests/test_custom_options.py | 2 +- tests/test_deprecated_field.py | 2 +- tests/test_field_defaults.py | 4 ++-- tests/test_has_field.py | 4 ++-- tests/test_has_field_many.py | 2 +- tests/test_imported_enums.py | 4 ++-- tests/test_is_initialized.py | 2 +- tests/test_issue_11.py | 4 ++-- tests/test_issue_69.py | 4 ++-- tests/test_items.py | 2 +- tests/test_merge_from.py | 2 +- tests/test_message_comments.py | 14 +++++++------- tests/test_message_field_types.py | 2 +- tests/test_message_init_kwargs.py | 2 +- tests/test_message_with_no_fields.py | 2 +- tests/test_nested_issue55.py | 2 +- tests/test_repeated_enum.py | 2 +- tests/test_signed_integer.py | 12 ++++++------ tests/test_typed_lists.py | 4 ++-- tests/test_unicode_strings.py | 2 +- tests/test_unknown_field.py | 2 +- 23 files changed, 41 insertions(+), 41 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 77b12b7..547e36f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,8 +19,8 @@ def pytest_sessionstart(session): # Insert built messages into path build = os.path.join(here, 'build') - lib_path = os.path.join(build, "lib.{0}-{1}".format(get_platform(), - sys.version[0:3])) + lib_path = os.path.join(build, "lib.{0}-{1}".format(get_platform(),sys.version[0:3])) + lib_path = os.path.join(lib_path,'pyrogen') if lib_path not in sys.path: sys.path.insert(0, lib_path) diff --git a/tests/create_message.py b/tests/create_message.py index 65ca109..332c3a5 100644 --- a/tests/create_message.py +++ b/tests/create_message.py @@ -7,7 +7,7 @@ def create_an_test(): # print LIB - import pyrogen.test_message_proto as an_test + import test_message_proto as an_test test = an_test.Test() test.timestamp = 539395200 test.field = 10689 diff --git a/tests/test_custom_options.py b/tests/test_custom_options.py index d7d11c4..cd9a469 100644 --- a/tests/test_custom_options.py +++ b/tests/test_custom_options.py @@ -8,7 +8,7 @@ class TestCustomOptionsTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestCustomOptions - from pyrogen.test_custom_options_proto import TestCustomOptions + from test_custom_options_proto import TestCustomOptions def test_fields(self): test = TestCustomOptions() diff --git a/tests/test_deprecated_field.py b/tests/test_deprecated_field.py index a698fe1..078ce7b 100644 --- a/tests/test_deprecated_field.py +++ b/tests/test_deprecated_field.py @@ -4,7 +4,7 @@ class DecimalDefaultsTest(unittest.TestCase): def setUp(self): - from pyrogen.test_message_field_types_proto import TestDeprecatedField + from test_message_field_types_proto import TestDeprecatedField warnings.filterwarnings("error") self.message = TestDeprecatedField() diff --git a/tests/test_field_defaults.py b/tests/test_field_defaults.py index eb479ee..083bf13 100644 --- a/tests/test_field_defaults.py +++ b/tests/test_field_defaults.py @@ -9,7 +9,7 @@ class DecimalDefaultsTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestDecimalDefaultsMessage - from pyrogen.test_field_defaults_proto import TestDecimalDefaultsMessage + from test_field_defaults_proto import TestDecimalDefaultsMessage def setUp(self): self.message = TestDecimalDefaultsMessage() @@ -61,7 +61,7 @@ class StringDefaultsTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestStringDefaultsMessage - from pyrogen.test_field_defaults_proto import TestStringDefaultsMessage + from test_field_defaults_proto import TestStringDefaultsMessage def setUp(self): self.message = TestStringDefaultsMessage() diff --git a/tests/test_has_field.py b/tests/test_has_field.py index 7e3eadd..65643ca 100644 --- a/tests/test_has_field.py +++ b/tests/test_has_field.py @@ -11,8 +11,8 @@ class HasFieldTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestRef, TestSs1, TestSs1Thing - from pyrogen.test_ref_message_proto import TestRef - from pyrogen.test_message_proto import Test, TestSs1, TestSs1Thing + from test_ref_message_proto import TestRef + from test_message_proto import Test, TestSs1, TestSs1Thing def test_has_field_for_repeated_field_raises_value_error(self): message = Test() diff --git a/tests/test_has_field_many.py b/tests/test_has_field_many.py index 77823af..05b9ee0 100644 --- a/tests/test_has_field_many.py +++ b/tests/test_has_field_many.py @@ -1,5 +1,5 @@ def test_has_field(): - from pyrogen.test_many_fields_proto import TestManyFields + from test_many_fields_proto import TestManyFields test = TestManyFields() # Assert HasField false on clean message diff --git a/tests/test_imported_enums.py b/tests/test_imported_enums.py index 6962daf..8021df2 100644 --- a/tests/test_imported_enums.py +++ b/tests/test_imported_enums.py @@ -12,8 +12,8 @@ class ImportedEnumsTest(unittest.TestCase): @classmethod def setUpClass(cls): global Status, MessageID, ExposesInternalEnumConstantsMessageinternal_enum, UsesImportedEnumsMessage - from pyrogen.test_multi_messages_toplevel_enums_proto import Status, MessageID - from pyrogen.test_imported_enums_proto import UsesImportedEnumsMessage, ExposesInternalEnumConstantsMessageinternal_enum + from test_multi_messages_toplevel_enums_proto import Status, MessageID + from test_imported_enums_proto import UsesImportedEnumsMessage, ExposesInternalEnumConstantsMessageinternal_enum def test_message_id_has_default_of_msg_one(self): message = UsesImportedEnumsMessage() diff --git a/tests/test_is_initialized.py b/tests/test_is_initialized.py index b913739..1c57322 100644 --- a/tests/test_is_initialized.py +++ b/tests/test_is_initialized.py @@ -10,7 +10,7 @@ class MergeFromTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestIsInitialized, SubMessage, TestWithRequiredSubMessage - from pyrogen.test_is_initialized_proto import TestIsInitialized, SubMessage, TestWithRequiredSubMessage + from test_is_initialized_proto import TestIsInitialized, SubMessage, TestWithRequiredSubMessage def test_new_message_is_not_initialized(self): message = TestIsInitialized() diff --git a/tests/test_issue_11.py b/tests/test_issue_11.py index 8dad0e8..65bbfa4 100644 --- a/tests/test_issue_11.py +++ b/tests/test_issue_11.py @@ -1,12 +1,12 @@ def test_before_overflow(): - from pyrogen.issue_11_proto import A + from issue_11_proto import A a = A() a.a0 = 0x7FFFFFFF assert A.FromString(a.SerializeToString()).a0 == 2147483647 def test_after_overflow(): - from pyrogen.issue_11_proto import A + from issue_11_proto import A a = A() a.a0 = 0x80000000 assert A.FromString(a.SerializeToString()).a0 == 2147483648 diff --git a/tests/test_issue_69.py b/tests/test_issue_69.py index 6a3ce37..2c2dc7f 100644 --- a/tests/test_issue_69.py +++ b/tests/test_issue_69.py @@ -1,10 +1,10 @@ def test_message_a(): - from pyrogen.issue_69_proto import A, B + from issue_69_proto import A, B a = A() assert type(a.b.a.b) == B def test_message_b(): - from pyrogen.issue_69_proto import A, B + from issue_69_proto import A, B b = B() assert type(b.a.b.a) == A diff --git a/tests/test_items.py b/tests/test_items.py index d3d63ee..d562d5e 100644 --- a/tests/test_items.py +++ b/tests/test_items.py @@ -8,7 +8,7 @@ class ItemsTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test - from pyrogen.test_message_proto import Test + from test_message_proto import Test def test_fields(self): test = create_an_test() diff --git a/tests/test_merge_from.py b/tests/test_merge_from.py index f16c692..98ebace 100644 --- a/tests/test_merge_from.py +++ b/tests/test_merge_from.py @@ -10,7 +10,7 @@ class MergeFromTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestEnumField, TestSs1, TestSs3 - from pyrogen.test_message_proto import Test, TestEnumField, TestSs1, TestSs3 + from test_message_proto import Test, TestEnumField, TestSs1, TestSs3 def test_merge_from_wrong_type_raises_type_error(self): dest = Test() diff --git a/tests/test_message_comments.py b/tests/test_message_comments.py index 2112069..836f4dc 100644 --- a/tests/test_message_comments.py +++ b/tests/test_message_comments.py @@ -2,23 +2,23 @@ class MessageComments(unittest.TestCase): def test_message_block_comment_line(self): - from pyrogen.test_message_comments_proto import TestBlockMessageCommentLine + from test_message_comments_proto import TestBlockMessageCommentLine self.assertEqual(TestBlockMessageCommentLine.__doc__, '\n Message with block comment in one line\n ') def test_message_block_comment(self): - from pyrogen.test_message_comments_proto import TestBlockMessageComment + from test_message_comments_proto import TestBlockMessageComment self.assertEqual(TestBlockMessageComment.__doc__, '\n Message with block comment\n ') def test_message_line_comment(self): - from pyrogen.test_message_comments_proto import TestLineMessageComment + from test_message_comments_proto import TestLineMessageComment self.assertEqual(TestLineMessageComment.__doc__, '\n Message with line comment\n ') def test_message_block_comment_multi(self): - from pyrogen.test_message_comments_proto import TestBlockMessageComments + from test_message_comments_proto import TestBlockMessageComments self.assertEqual(TestBlockMessageComments.__doc__, '\n Message with multiline block comment\n\nMore details\n"stop"\n ') def test_field_message_comment_standard(self): - from pyrogen.test_message_comments_proto import TestFieldMessageCommentStandard + from test_message_comments_proto import TestFieldMessageCommentStandard self.assertEqual(TestFieldMessageCommentStandard.hello.__doc__, 'Hello comment ') def test_field_message_comment_before(self): - from pyrogen.test_message_comments_proto import TestFieldMessageCommentBefore + from test_message_comments_proto import TestFieldMessageCommentBefore self.assertEqual(TestFieldMessageCommentBefore.hello.__doc__, 'Hello comment before definition ') def test_field_message_comment_long(self): - from pyrogen.test_message_comments_proto import TestFieldMessageCommentLong + from test_message_comments_proto import TestFieldMessageCommentLong self.assertEqual(TestFieldMessageCommentLong.hello.__doc__, 'Hello long block comment\nwith several lines ') diff --git a/tests/test_message_field_types.py b/tests/test_message_field_types.py index 157e7f5..6511bca 100644 --- a/tests/test_message_field_types.py +++ b/tests/test_message_field_types.py @@ -7,7 +7,7 @@ class MessageFieldTypesTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestFieldTypes - from pyrogen.test_message_field_types_proto import TestFieldTypes + from test_message_field_types_proto import TestFieldTypes def test_bytes_payload_serialize_to_string(self): message = TestFieldTypes() diff --git a/tests/test_message_init_kwargs.py b/tests/test_message_init_kwargs.py index d37ed04..f1a30d5 100644 --- a/tests/test_message_init_kwargs.py +++ b/tests/test_message_init_kwargs.py @@ -9,7 +9,7 @@ class MergeFromTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestSs1, TestSs3 - from pyrogen.test_message_proto import Test, TestSs1, TestSs3 + from test_message_proto import Test, TestSs1, TestSs3 def test_message_init_with_bad_keyword_arg_raises_value_error(self): self.assertRaises(ValueError, Test, non_existant=3) diff --git a/tests/test_message_with_no_fields.py b/tests/test_message_with_no_fields.py index 6da83cf..75f01f5 100644 --- a/tests/test_message_with_no_fields.py +++ b/tests/test_message_with_no_fields.py @@ -7,7 +7,7 @@ class EmptyMessageTest(unittest.TestCase): @classmethod def setUpClass(cls): global EmptyMessageWithNoFields - from pyrogen.test_message_field_types_proto import EmptyMessageWithNoFields + from test_message_field_types_proto import EmptyMessageWithNoFields def setUp(self): self.message = EmptyMessageWithNoFields() diff --git a/tests/test_nested_issue55.py b/tests/test_nested_issue55.py index a4b64cc..ca4d266 100644 --- a/tests/test_nested_issue55.py +++ b/tests/test_nested_issue55.py @@ -16,7 +16,7 @@ def setUpClass(cls): global M global MN global MN2 - from pyrogen.test_nested_issue55_proto import M, MN, MN2 + from test_nested_issue55_proto import M, MN, MN2 def test_use_nested(self): """ diff --git a/tests/test_repeated_enum.py b/tests/test_repeated_enum.py index 3a0da2c..679d59a 100644 --- a/tests/test_repeated_enum.py +++ b/tests/test_repeated_enum.py @@ -8,7 +8,7 @@ class RepeatedEnumTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestRepeatedEnum - from pyrogen.test_repeated_enum_proto import TestRepeatedEnum + from test_repeated_enum_proto import TestRepeatedEnum def test_repeated_enum_serde(self): message1 = TestRepeatedEnum() diff --git a/tests/test_signed_integer.py b/tests/test_signed_integer.py index a79814f..da9f663 100644 --- a/tests/test_signed_integer.py +++ b/tests/test_signed_integer.py @@ -18,7 +18,7 @@ def gen_rand_uint64(): def test_int32(): - from pyrogen.test_signed_integer_proto import Int + from test_signed_integer_proto import Int a = Int() v = gen_rand_int32() a.Int32 = v @@ -26,7 +26,7 @@ def test_int32(): def test_int64(): - from pyrogen.test_signed_integer_proto import Int + from test_signed_integer_proto import Int a = Int() v = gen_rand_int64() a.Int64 = v @@ -34,7 +34,7 @@ def test_int64(): def test_uint32(): - from pyrogen.test_signed_integer_proto import Int + from test_signed_integer_proto import Int a = Int() v = gen_rand_uint32() a.UInt32 = v @@ -42,7 +42,7 @@ def test_uint32(): def test_uint64(): - from pyrogen.test_signed_integer_proto import Int + from test_signed_integer_proto import Int a = Int() v = gen_rand_uint64() a.UInt64 = v @@ -50,7 +50,7 @@ def test_uint64(): def test_sint32(): - from pyrogen.test_signed_integer_proto import Int + from test_signed_integer_proto import Int a = Int() v = gen_rand_int32() a.SInt32 = v @@ -58,7 +58,7 @@ def test_sint32(): def test_sint64(): - from pyrogen.test_signed_integer_proto import Int + from test_signed_integer_proto import Int a = Int() v = gen_rand_int64() a.SInt64 = v diff --git a/tests/test_typed_lists.py b/tests/test_typed_lists.py index 239aed2..de98cd2 100644 --- a/tests/test_typed_lists.py +++ b/tests/test_typed_lists.py @@ -8,8 +8,8 @@ class MergeFromTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestRef - from pyrogen.test_ref_message_proto import TestRef - from pyrogen.test_message_proto import Test + from test_ref_message_proto import TestRef + from test_message_proto import Test def test_extend_with_generic_list(self): t = Test() diff --git a/tests/test_unicode_strings.py b/tests/test_unicode_strings.py index 14b695d..1c2ea03 100644 --- a/tests/test_unicode_strings.py +++ b/tests/test_unicode_strings.py @@ -11,7 +11,7 @@ class TestUnicodeStrings(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestSs1, TestFieldTypes - from pyrogen.test_message_proto import Test, TestSs1 + from test_message_proto import Test, TestSs1 def test_unicode_string_parse_from_string(self): message = Test.FromString(b'\x1a\x08\xce\x91\xce\x92\xce\x93\xce\x94') diff --git a/tests/test_unknown_field.py b/tests/test_unknown_field.py index a479f28..cada5dd 100644 --- a/tests/test_unknown_field.py +++ b/tests/test_unknown_field.py @@ -15,7 +15,7 @@ class MessageTest(unittest.TestCase): @classmethod def setUpClass(cls): global TestTruncated - from pyrogen.test_truncated_proto import TestTruncated + from test_truncated_proto import TestTruncated def test_deser(self): test = TestTruncated() From 283fbd11bd5f11d7db1a862293eefe10287d0669 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Mon, 27 Jul 2020 14:48:51 +0200 Subject: [PATCH 20/32] add parsing atomic type from custom field option --- pyrobuf/parse_proto.py | 2 +- setup.py | 2 +- tests/proto/test_custom_options.proto | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index 68f50a7..fd11ea1 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -25,7 +25,7 @@ class Parser(object): ('DEFAULT', r'default\s*='), ('PACKED', r'packed\s*=\s*(true|false)'), ('DEPRECATED', r'deprecated\s*=\s*(true|false)'), - ('CUSTOM', r'(\([A-Za-z][0-9A-Za-z_]*\).[A-Za-z][0-9A-Za-z_]*)\s*='), + ('CUSTOM', r'(\([A-Za-z][0-9A-Za-z_]*\)(?:.[A-Za-z][0-9A-Za-z_]*)?)\s*='), ('LBRACKET', r'\['), ('RBRACKET', r'\]\s*;'), ('LBRACE', r'\{'), diff --git a/setup.py b/setup.py index 0f77990..7962107 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.3.7" +VERSION = "0.9.3.8" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" diff --git a/tests/proto/test_custom_options.proto b/tests/proto/test_custom_options.proto index d7429b5..b0683ae 100644 --- a/tests/proto/test_custom_options.proto +++ b/tests/proto/test_custom_options.proto @@ -9,4 +9,5 @@ message TestCustomOptions { VALUE1 = 1; } optional CustomEnum field3 = 3 [ default = VALUE1, (my_options).custom1 = 4 ]; + optional double field4 = 4 [ (my_option)= 3.4 ]; } \ No newline at end of file From 99c5fada28dbaff1bcea62492d51bfea4d423583 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 28 Jul 2020 10:21:32 +0200 Subject: [PATCH 21/32] fix running tests, master branch merge --- tests/conftest.py | 8 +++++++- tests/test_has_field.py | 2 +- tests/test_typed_lists.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d0691ee..0fee484 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,7 +25,13 @@ def pytest_sessionstart(session): # Insert built messages into path build = os.path.join(here, 'build') lib_path = os.path.join(build, "lib.{0}-{1}".format(get_platform(),sys.version[0:3])) - lib_path = os.path.join(lib_path,'pyrogen') + # for import with full_name (used in templates) + if lib_path not in sys.path: + sys.path.insert(0, lib_path) + + # for short import wihtout pyrogen prefix + lib_path = os.path.join(lib_path,'pyrogen') if lib_path not in sys.path: sys.path.insert(0, lib_path) + diff --git a/tests/test_has_field.py b/tests/test_has_field.py index 65643ca..0c9a17c 100644 --- a/tests/test_has_field.py +++ b/tests/test_has_field.py @@ -11,8 +11,8 @@ class HasFieldTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestRef, TestSs1, TestSs1Thing - from test_ref_message_proto import TestRef from test_message_proto import Test, TestSs1, TestSs1Thing + from test_ref_message_proto import TestRef def test_has_field_for_repeated_field_raises_value_error(self): message = Test() diff --git a/tests/test_typed_lists.py b/tests/test_typed_lists.py index de98cd2..6e5e3a7 100644 --- a/tests/test_typed_lists.py +++ b/tests/test_typed_lists.py @@ -8,8 +8,8 @@ class MergeFromTest(unittest.TestCase): @classmethod def setUpClass(cls): global Test, TestRef - from test_ref_message_proto import TestRef from test_message_proto import Test + from test_ref_message_proto import TestRef def test_extend_with_generic_list(self): t = Test() From 94285bc21165bb555e6cfa5b989289ac33f39a91 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 28 Jul 2020 10:58:57 +0200 Subject: [PATCH 22/32] add version for generated package using datetime --- pyrobuf/compile.py | 11 ++++++++--- setup.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pyrobuf/compile.py b/pyrobuf/compile.py index ef8a156..d29c729 100644 --- a/pyrobuf/compile.py +++ b/pyrobuf/compile.py @@ -3,6 +3,7 @@ import os import sys from setuptools import setup +from datetime import datetime from Cython.Build import cythonize from Cython.Distutils import build_ext @@ -18,6 +19,9 @@ _VM = sys.version_info.major +THE_TIME = datetime.now() +VERSION = f'''{THE_TIME.year}.{THE_TIME.month}.{THE_TIME.day}'''\ + f'''{THE_TIME.hour*3600+THE_TIME.minute*60+THE_TIME.second}''' class BasePackagePatch_BuildExt(build_ext): """ Create __init__.py for base package, after build @@ -101,10 +105,11 @@ def compile(self): self._package() setup(name='pyrobuf-generated' if not self.package else self.package, - ext_modules=cythonize(self._pyx_files, + version=VERSION, + ext_modules=cythonize(self._pyx_files, include_path=self.include_path), - cmdclass= dict(build_ext=BasePackagePatch_BuildExt), - script_args=script_args) + cmdclass= dict(build_ext=BasePackagePatch_BuildExt), + script_args=script_args) def extend(self, dist): self._compile_spec() diff --git a/setup.py b/setup.py index 7962107..74b7056 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.3.8" +VERSION = "0.9.3.9" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" From 8fbbd6a2c8dd044aaa80b66d1a13d2874f4cc390 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Mon, 14 Jun 2021 15:35:49 +0200 Subject: [PATCH 23/32] update package format to wheel --- pyrobuf/compile.py | 18 +++++++++++++++--- setup.py | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pyrobuf/compile.py b/pyrobuf/compile.py index d29c729..75c473e 100644 --- a/pyrobuf/compile.py +++ b/pyrobuf/compile.py @@ -1,8 +1,11 @@ import argparse +import atexit import glob import os import sys from setuptools import setup +from setuptools.command.install import install +from setuptools.dist import Distribution from datetime import datetime from Cython.Build import cythonize @@ -23,6 +26,14 @@ VERSION = f'''{THE_TIME.year}.{THE_TIME.month}.{THE_TIME.day}'''\ f'''{THE_TIME.hour*3600+THE_TIME.minute*60+THE_TIME.second}''' +def _post_install(*args): + bdist_wheel_cmd = args[0].get_command_obj('bdist_wheel') + tag = '-'.join(bdist_wheel_cmd.get_tag()) + os.system(f'python -m pip install --force-reinstall ./dist/{bdist_wheel_cmd.wheel_dist_name}-{tag}.whl') +class new_install(install): + def __init__(self, *args, **kwargs): + super(new_install, self).__init__(*args, **kwargs) + atexit.register(_post_install, *args) class BasePackagePatch_BuildExt(build_ext): """ Create __init__.py for base package, after build """ @@ -91,10 +102,11 @@ def parse_cli_args(cls): clean=args.clean) def compile(self): - script_args = ['build', '--build-base={0}'.format(self.build)] + script_args = ['build', '--build-base={0}'.format(self.build), 'bdist_wheel'] + cmds = dict(build_ext=BasePackagePatch_BuildExt) if self.install: - script_args.append('install') + cmds['install'] = new_install if self.force: script_args.append('--force') @@ -108,7 +120,7 @@ def compile(self): version=VERSION, ext_modules=cythonize(self._pyx_files, include_path=self.include_path), - cmdclass= dict(build_ext=BasePackagePatch_BuildExt), + cmdclass=cmds, script_args=script_args) def extend(self, dist): diff --git a/setup.py b/setup.py index 74b7056..0b668b7 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.3.9" +VERSION = "0.9.3.10" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" From 03d4cc85f9ca1a4b629b5831d081a57589a505d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20T=C3=B6pfer?= Date: Thu, 17 Nov 2022 09:43:45 +0100 Subject: [PATCH 24/32] Remove comment parsing, change Enum to IntEnum --- pyrobuf/parse_proto.py | 101 +--------------------- pyrobuf/protobuf/templates/proto_pyx.tmpl | 29 +------ setup.py | 2 +- tests/proto/test_message_comments.proto | 43 --------- tests/test_imported_enums.py | 8 +- tests/test_merge_from.py | 4 +- tests/test_message_comments.py | 24 ----- 7 files changed, 11 insertions(+), 200 deletions(-) delete mode 100644 tests/proto/test_message_comments.proto delete mode 100644 tests/test_message_comments.py diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index fd11ea1..edfbd52 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -8,8 +8,8 @@ class Parser(object): # Tokens ordered roughly by priority tokens = [ - ('COMMENT_OL', r'\/\/(.*?)\n'), - ('COMMENT_ML', r'\/\*((?:.|[\r\n])*?)\*\/'), + ('COMMENT_OL', r'\/\/.*?\n'), + ('COMMENT_ML', r'\/\*(?:.|[\r\n])*?\*\/'), ('OPTION', r'option\s+((?:.|[\n\r])*?);'), ('IMPORT', r'import\s+"(.+?).proto"\s*;'), ('MESSAGE', r'message\s+([A-Za-z_][0-9A-Za-z_]*)'), @@ -42,8 +42,6 @@ class Parser(object): ] parsable_tokens = ( - 'COMMENT_OL', - 'COMMENT_ML', 'OPTION', 'SYNTAX', 'IMPORT', @@ -96,24 +94,6 @@ class Parser(object): 'bytes': 'BytesList' } - py_type_map = { - 'float': 'float', - 'double': 'float', - 'int32': 'int', - 'sint32': 'int', - 'sfixed32': 'int', - 'uint32': 'int', - 'fixed32': 'int', - 'int64': 'int', - 'sint64': 'int', - 'sfixed64': 'int', - 'uint64': 'int', - 'fixed64': 'int', - 'bool': 'bool', - 'string': 'str', - 'bytes': 'bytes' - } - c_type_map = { 'float': 'float', 'double': 'double', @@ -161,7 +141,6 @@ class Parser(object): def __init__(self, string): self.string = string self.lines = string.split('\n') - self._lastComment = None def tokenize(self, disabled_token_types): token_type_to_token_class = self.get_token_type_to_token_class_map() @@ -195,41 +174,6 @@ def tokenize(self, disabled_token_types): raise Exception("Unexpected character '{}' on line {}: '{}'".format( self.string[pos], line + 1, self.lines[line])) - def _regular_comment(self, comment): - if comment[0] == '*': - # remove asterisk and trim block comment - block_comment = "" - for line in comment.splitlines(): - resline = line.strip() - if resline: - if resline[0] == '*': - block_comment += resline[1:].lstrip() - else: - block_comment += resline - block_comment += '\n' - return block_comment.strip() - - return comment[1:].strip() - - def _handle_comment(self, token, previous_token): - if token.token_type == 'MESSAGE' or token.token_type == 'FIELD' or token.token_type == 'ENUM' or token.token_type == 'ENUM_FIELD': - token.comment = self._lastComment - if token.token_type != 'MODIFIER': - self._lastComment = None - - if token.token_type != "COMMENT_OL" and token.token_type != "COMMENT_ML": - return False - - if token.comment: - # use only regular comments, ignore normal, development comments - if token.comment[0] == '*' or token.comment[0] == '/': - if previous_token.token_type == "FIELD" or previous_token.token_type == "ENUM_FIELD": - if token.line == previous_token.line: - previous_token.comment = self._regular_comment(token.comment) - return True - self._lastComment = self._regular_comment(token.comment) - return True - def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): self.verify_parsable_tokens() tokens = self.tokenize(disabled_tokens) @@ -242,9 +186,6 @@ def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): previous = self.LBrace(-1) for token in tokens: - if self._handle_comment(token, previous): - continue - if token.token_type == 'OPTION': continue @@ -385,10 +326,6 @@ def _parse_message(self, current_message, tokens, messages, enums, imported_enum token.line + 1, self.lines[token.line]) for token in tokens: - if self._handle_comment(token, previous): - previous = token - continue - if token.token_type == 'MESSAGE': token.full_name = current_message.full_name + token.name ret = self._parse_message(token, tokens, messages, enums.copy(), imported_enums) @@ -496,10 +433,6 @@ def _parse_oneof(self, oneof_token, tokens, current_message, messages, enums, im # setting previous as a place holder for the inner fields parsing previous = self.LBrace(-1) for token in tokens: - if self._handle_comment(token,previous): - previous = token - continue - if token.token_type == 'FIELD': # fields will be added to the proper message by the following # parser function that takes care of fields generally @@ -600,10 +533,6 @@ def _parse_enum(self, current, tokens, scope, current_message=None): setDefault = False for token in tokens: - if self._handle_comment(token,previous): - previous = token - continue - if token.token_type == 'ENUM_FIELD': if (setDefault is False): if self.syntax == 3: @@ -710,18 +639,6 @@ class Token(object): line = -1 type = None - class Comment_OL(Token): - token_type = 'COMMENT_OL' - def __init__(self, line, comment): - self.line = line - self.comment = comment - - class Comment_ML(Token): - token_type = 'COMMENT_ML' - def __init__(self, line, comment): - self.line = line - self.comment = comment - class Option(Token): token_type = 'OPTION' @@ -749,7 +666,6 @@ class Message(Token): def __init__(self, line, name): self.line = line self.name = name - self.comment = None # full_name may later be overridden with parent hierarchy self.full_name = name self.messages = {} @@ -774,8 +690,6 @@ def __init__(self, line, ftype, name, index): self.modifier = None self.type = ftype self.name = name - # comment for attribute in class docstring - self.comment = None self.index = int(index) self.default = None self.packed = False @@ -793,16 +707,6 @@ def get_key(self): else: return (self.index << 3) | 0 - def get_python_type(self): - prefix ="" - if self.modifier == 'repeated': - prefix = "list of " - if self.type == "message": - return prefix + self.message_name - if self.type == "enum" : - return prefix + self.enum_name - return prefix + Parser.py_type_map[self.type] - class MapField(Token): token_type = 'MAP_FIELD' @@ -888,7 +792,6 @@ class Enum(Token): def __init__(self, line, name): self.line = line self.name = name - self.comment = None self.fields = {} self.default = None # full_name may later be overridden with parent hierarchy diff --git a/pyrobuf/protobuf/templates/proto_pyx.tmpl b/pyrobuf/protobuf/templates/proto_pyx.tmpl index 8c1f8df..fb30b09 100644 --- a/pyrobuf/protobuf/templates/proto_pyx.tmpl +++ b/pyrobuf/protobuf/templates/proto_pyx.tmpl @@ -15,36 +15,14 @@ from pyrogen.{{ import }}_proto cimport * {%- macro enum_fields_def(enum) %} -class {{enum.full_name}}(enum.Enum): - """ - {%- if enum.comment != None %} - {{enum.comment}} - {%- endif %} - Enum Constants - -------------- - {%for field in enum.fields.values()|sort(attribute='name') %} - - `{{field.name}}` {% if field.comment != None %} – {{field.comment}} {%- endif %} - {%-endfor %} - """ +class {{enum.full_name}}(enum.IntEnum): {%- for field in enum.fields.values() %} - {{ field.name }} = _{{ field.full_name }} {%- if field.comment != None %} # {{field.comment}}{%- endif %} + {{ field.name }} = _{{ field.full_name }} {%- endfor %} {%- endmacro %} {%- macro classdef(message) %} cdef class {{ message.full_name }}: - """ - {%- if message.comment != None %} - {{message.comment}} - {%- endif %} - {%- if message.fields.values()|first is defined %} - Attributes - ---------- - {%- for field in message.fields.values() %} - - `{{field.name}}:` {{field.get_python_type()}}{%- if field.modifier != 'required'%},optional{%- endif %} {% if field.comment != None %} – {{field.comment}} {%- endif %} - {%- endfor %} - {%- endif %} - """ def __cinit__(self): self._listener = noop_listener @@ -161,9 +139,6 @@ cdef class {{ message.full_name }}: {% for index, field in message.fields|dictsort() %} @property def {{ field.name }}(self): - {%- if field.comment != None %} - """{{field.comment}} """ - {%- endif %} {%- if field.deprecated|default(false) == true %} warnings.warn("field '{{ field.name }}' is deprecated", DeprecationWarning) {%- endif %} diff --git a/setup.py b/setup.py index 0b668b7..f5de88b 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.3.10" +VERSION = "0.9.3.11" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" diff --git a/tests/proto/test_message_comments.proto b/tests/proto/test_message_comments.proto deleted file mode 100644 index a03f80d..0000000 --- a/tests/proto/test_message_comments.proto +++ /dev/null @@ -1,43 +0,0 @@ -// normal development comments -// next lines is a empty line comment syntax test -// - -/// Message with line comment -message TestLineMessageComment { -} - -/** Message with block comment in one line */ -message TestBlockMessageCommentLine { -} - -/** - * Message with block comment - */ -message TestBlockMessageComment { -} - -/** -* Message with multiline block comment -* -* More details -* "stop" -*/ -message TestBlockMessageComments{ -} - -message TestFieldMessageCommentStandard{ - required int32 hello = 1; /// Hello comment -} - -message TestFieldMessageCommentBefore{ - /// Hello comment before definition - required int32 hello = 1; -} - -message TestFieldMessageCommentLong{ - /** - * Hello long block comment - * with several lines - */ - required int32 hello = 1; -} \ No newline at end of file diff --git a/tests/test_imported_enums.py b/tests/test_imported_enums.py index 8021df2..6721ca1 100644 --- a/tests/test_imported_enums.py +++ b/tests/test_imported_enums.py @@ -17,12 +17,12 @@ def setUpClass(cls): def test_message_id_has_default_of_msg_one(self): message = UsesImportedEnumsMessage() - self.assertEqual(message.message_id, MessageID.MSG_ONE.value) + self.assertEqual(message.message_id, MessageID.MSG_ONE) def test_status_has_default_of_close(self): message = UsesImportedEnumsMessage() - self.assertEqual(message.status, Status.CLOSE.value) + self.assertEqual(message.status, Status.CLOSE) def test_internal_enum_constants_exposed(self): - self.assertEqual(ExposesInternalEnumConstantsMessageinternal_enum.INTERNAL.value, 0) - self.assertEqual(ExposesInternalEnumConstantsMessageinternal_enum.EXTERNAL.value, 1) + self.assertEqual(ExposesInternalEnumConstantsMessageinternal_enum.INTERNAL, 0) + self.assertEqual(ExposesInternalEnumConstantsMessageinternal_enum.EXTERNAL, 1) diff --git a/tests/test_merge_from.py b/tests/test_merge_from.py index 98ebace..78b9eb2 100644 --- a/tests/test_merge_from.py +++ b/tests/test_merge_from.py @@ -50,9 +50,9 @@ def test_merge_from_does_set_scalar_field_that_is_set_in_source(self): def test_merge_from_does_set_enum_field_that_is_set_in_source(self): source = Test() dest = Test() - source.enum_field = TestEnumField.TEST_ENUM_FIELD_2.value + source.enum_field = TestEnumField.TEST_ENUM_FIELD_2 dest.MergeFrom(source) - self.assertEqual(dest.enum_field, TestEnumField.TEST_ENUM_FIELD_2.value) + self.assertEqual(dest.enum_field, TestEnumField.TEST_ENUM_FIELD_2) def test_merge_from_does_merge_message_field_that_is_set_in_source(self): source = TestSs3() diff --git a/tests/test_message_comments.py b/tests/test_message_comments.py deleted file mode 100644 index 836f4dc..0000000 --- a/tests/test_message_comments.py +++ /dev/null @@ -1,24 +0,0 @@ -import unittest - -class MessageComments(unittest.TestCase): - def test_message_block_comment_line(self): - from test_message_comments_proto import TestBlockMessageCommentLine - self.assertEqual(TestBlockMessageCommentLine.__doc__, '\n Message with block comment in one line\n ') - def test_message_block_comment(self): - from test_message_comments_proto import TestBlockMessageComment - self.assertEqual(TestBlockMessageComment.__doc__, '\n Message with block comment\n ') - def test_message_line_comment(self): - from test_message_comments_proto import TestLineMessageComment - self.assertEqual(TestLineMessageComment.__doc__, '\n Message with line comment\n ') - def test_message_block_comment_multi(self): - from test_message_comments_proto import TestBlockMessageComments - self.assertEqual(TestBlockMessageComments.__doc__, '\n Message with multiline block comment\n\nMore details\n"stop"\n ') - def test_field_message_comment_standard(self): - from test_message_comments_proto import TestFieldMessageCommentStandard - self.assertEqual(TestFieldMessageCommentStandard.hello.__doc__, 'Hello comment ') - def test_field_message_comment_before(self): - from test_message_comments_proto import TestFieldMessageCommentBefore - self.assertEqual(TestFieldMessageCommentBefore.hello.__doc__, 'Hello comment before definition ') - def test_field_message_comment_long(self): - from test_message_comments_proto import TestFieldMessageCommentLong - self.assertEqual(TestFieldMessageCommentLong.hello.__doc__, 'Hello long block comment\nwith several lines ') From cad43f4ecfd5ef865631f57ea53987f1fa91c366 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Mon, 28 Nov 2022 14:05:11 +0100 Subject: [PATCH 25/32] remove wheel build/install, add site_pkg option --- pyrobuf/compile.py | 54 +++++++++++++++--------------------------- pyrobuf/parse_proto.py | 14 +++++++---- 2 files changed, 28 insertions(+), 40 deletions(-) diff --git a/pyrobuf/compile.py b/pyrobuf/compile.py index 75c473e..8108f43 100644 --- a/pyrobuf/compile.py +++ b/pyrobuf/compile.py @@ -1,15 +1,11 @@ import argparse -import atexit import glob import os import sys from setuptools import setup -from setuptools.command.install import install -from setuptools.dist import Distribution -from datetime import datetime +from distutils.sysconfig import get_python_lib from Cython.Build import cythonize -from Cython.Distutils import build_ext from pathlib import Path from jinja2 import Environment, PackageLoader @@ -22,26 +18,6 @@ _VM = sys.version_info.major -THE_TIME = datetime.now() -VERSION = f'''{THE_TIME.year}.{THE_TIME.month}.{THE_TIME.day}'''\ - f'''{THE_TIME.hour*3600+THE_TIME.minute*60+THE_TIME.second}''' - -def _post_install(*args): - bdist_wheel_cmd = args[0].get_command_obj('bdist_wheel') - tag = '-'.join(bdist_wheel_cmd.get_tag()) - os.system(f'python -m pip install --force-reinstall ./dist/{bdist_wheel_cmd.wheel_dist_name}-{tag}.whl') -class new_install(install): - def __init__(self, *args, **kwargs): - super(new_install, self).__init__(*args, **kwargs) - atexit.register(_post_install, *args) -class BasePackagePatch_BuildExt(build_ext): - """ Create __init__.py for base package, after build - """ - def run(self): - build_ext.run(self) - filename = Path(self.build_lib).joinpath('pyrogen').joinpath('__init__.py') - filename.touch(exist_ok=True) - class Compiler(object): _env = Environment(loader=PackageLoader('pyrobuf.protobuf', 'templates')) @@ -50,7 +26,7 @@ class Compiler(object): def __init__(self, sources, out="out", build="build", install=False, proto3=False, force=False, package=None, includes=None, - clean=False): + clean=False, site_pkg=None): self.sources = sources self.out = os.path.join(out, 'pyrogen') self.build = build @@ -61,6 +37,11 @@ def __init__(self, sources, out="out", build="build", install=False, self.clean = clean here = os.path.dirname(os.path.abspath(__file__)) self.include_path = [os.path.join(here, 'src'), out] + self.site_pkg_path= None + if site_pkg: + site_pkg_path = os.path.join(get_python_lib(),site_pkg) + self.include_path.append(site_pkg_path) + self.site_pkg_path = os.path.join(site_pkg_path,'pyrogen/') self._generated = set() self._messages = [] self._pyx_files = [] @@ -92,6 +73,8 @@ def parse_cli_args(cls): help="name of package to compile to") parser.add_argument('--include', action='append', help="add directory to includes path") + parser.add_argument('--site_pkg', type=str, default=None, + help="build with installed site-package source") parser.add_argument('--clean', action='store_true', help="force recompilation of messages") args = parser.parse_args() @@ -99,14 +82,13 @@ def parse_cli_args(cls): return cls(args.sources, out=args.out_dir, build=args.build_dir, install=args.install, proto3=args.proto3, force=args.force, package=args.package, includes=args.include, - clean=args.clean) + clean=args.clean, site_pkg=args.site_pkg) def compile(self): - script_args = ['build', '--build-base={0}'.format(self.build), 'bdist_wheel'] + script_args = ['build', '--build-base={0}'.format(self.build)] - cmds = dict(build_ext=BasePackagePatch_BuildExt) if self.install: - cmds['install'] = new_install + script_args.append('install') if self.force: script_args.append('--force') @@ -117,10 +99,8 @@ def compile(self): self._package() setup(name='pyrobuf-generated' if not self.package else self.package, - version=VERSION, ext_modules=cythonize(self._pyx_files, - include_path=self.include_path), - cmdclass=cmds, + include_path=self.include_path), script_args=script_args) def extend(self, dist): @@ -160,10 +140,14 @@ def _generate(self, filename): print("generating {0}".format(filename)) self._generated.add(name) - msg_def = self.parser.parse_from_filename(filename, self.includes) + msg_def = self.parser.parse_from_filename(filename, self.includes,self.site_pkg_path) self._messages.append(msg_def) for f in msg_def['imports']: + + if self.site_pkg_path and os.path.exists(self.site_pkg_path+f+'_proto.pyx'): + continue + print("parsing dependency '{}'".format(f)) depends = None @@ -176,7 +160,7 @@ def _generate(self, filename): self._generate(depends) except FileNotFoundError: raise FileNotFoundError("can't find message spec for '{}'" - .format(f)) + .format(f)) if self.package is None: self._write(name, msg_def) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index edfbd52..bc84522 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -174,7 +174,7 @@ def tokenize(self, disabled_token_types): raise Exception("Unexpected character '{}' on line {}: '{}'".format( self.string[pos], line + 1, self.lines[line])) - def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): + def parse(self, site_pkg_path=None, cython_info=True, fname='', includes=None, disabled_tokens=()): self.verify_parsable_tokens() tokens = self.tokenize(disabled_tokens) rep = {'imports': [], 'messages': [], 'enums': []} @@ -205,6 +205,9 @@ def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): rep['imports'].append(token.value) + if site_pkg_path and os.path.exists(site_pkg_path+token.value+'_proto.pyx'): + continue + # Google's protoc only supports the use of messages and enums # from direct imports. So messages and enums from indirect # imports are not fetched here. @@ -212,6 +215,7 @@ def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): token.value + '.proto', fname, includes, + site_pkg_path, disabled_tokens ) imported['messages'].update( @@ -254,18 +258,18 @@ def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): return rep @classmethod - def parse_from_filename(cls, fname, includes, disabled_tokens=None): + def parse_from_filename(cls, fname, includes, site_pkg_path, disabled_tokens=None): disabled_tokens = disabled_tokens or cls.unsupported_tokens with open(fname, 'r') as fp: s = fp.read() try: - return cls(s).parse(fname=fname, includes=includes, disabled_tokens=disabled_tokens) + return cls(s).parse(site_pkg_path, fname=fname, includes=includes, disabled_tokens=disabled_tokens) except Exception as e: print('Exception while parsing {}'.format(fname)) raise e - def _parse_import(self, fname, parent_fname, includes, disabled_tokens): + def _parse_import(self, fname, parent_fname, includes, site_pkg_path, disabled_tokens): actual_fname = fname if not os.path.isabs(fname): for d in [os.path.dirname(parent_fname)] + includes: @@ -273,7 +277,7 @@ def _parse_import(self, fname, parent_fname, includes, disabled_tokens): if os.path.exists(actual_fname): break - return self.__class__.parse_from_filename(actual_fname, includes, disabled_tokens) + return self.__class__.parse_from_filename(actual_fname, includes, site_pkg_path, disabled_tokens) def _process_token_enum(self, token, enums): """ From 67d44ef5419ada3a8fc763cd2c57bf50f6611532 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Mon, 28 Nov 2022 14:16:14 +0100 Subject: [PATCH 26/32] fix test with python 3.10 --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0fee484..ba47eb4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,7 +24,8 @@ def pytest_sessionstart(session): # Insert built messages into path build = os.path.join(here, 'build') - lib_path = os.path.join(build, "lib.{0}-{1}".format(get_platform(),sys.version[0:3])) + lib_path = os.path.join(build, "lib.{0}-{1}".format(get_platform(), + sys.version[0:4])) # for import with full_name (used in templates) if lib_path not in sys.path: From 5915d0682015d0841bb80022b21ac052c97a7447 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Tue, 29 Nov 2022 11:02:09 +0100 Subject: [PATCH 27/32] fix many field (>64) with unused field index --- pyrobuf/parse_proto.py | 6 ++++-- tests/proto/test_many_fields.proto | 7 ++++++- tests/test_has_field_many.py | 8 ++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index bc84522..6c4de5e 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -598,9 +598,11 @@ def _parse_extend(self, current, tokens): return current def add_cython_info(self, message): + count = 0 for index, field in message.fields.items(): - field.bitmap_idx = (index - 1) // 64 - field.bitmap_mask = 1 << ((index - 1) % 64) + field.bitmap_idx = count // 64 + field.bitmap_mask = 1 << (count % 64) + count += 1 field.list_type = self.list_type_map.get(field.type, 'TypedList') field.fixed_width = (field.type in { 'float', 'double', 'fixed32', 'sfixed32', 'fixed64', 'sfixed64' diff --git a/tests/proto/test_many_fields.proto b/tests/proto/test_many_fields.proto index 2050e96..3b8d52a 100644 --- a/tests/proto/test_many_fields.proto +++ b/tests/proto/test_many_fields.proto @@ -127,4 +127,9 @@ message TestManyFields { optional int64 field126 = 126; optional int64 field127 = 127; optional int64 field128 = 128; -} \ No newline at end of file +} + +message TestUnusedFieldIndex { + optional int64 field64 = 64; + optional int64 field65 = 65; +} diff --git a/tests/test_has_field_many.py b/tests/test_has_field_many.py index 05b9ee0..3ab6f3e 100644 --- a/tests/test_has_field_many.py +++ b/tests/test_has_field_many.py @@ -19,3 +19,11 @@ def test_has_field(): for field in test.Fields(): assert not test.HasField(field) + +def test_has_field_withUnusedIndex(): + from test_many_fields_proto import TestUnusedFieldIndex + test = TestUnusedFieldIndex() + + # Assert HasField false on clean message + for field in test.Fields(): + assert not test.HasField(field) From 189999c95a022c82f3ac0dfecb415d9d5ed4116c Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Wed, 30 Nov 2022 13:27:41 +0100 Subject: [PATCH 28/32] remove site_pkg option, skip build for includes --- pyrobuf/compile.py | 21 +++++++-------------- pyrobuf/parse_proto.py | 14 +++++--------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/pyrobuf/compile.py b/pyrobuf/compile.py index 8108f43..88746fd 100644 --- a/pyrobuf/compile.py +++ b/pyrobuf/compile.py @@ -26,22 +26,17 @@ class Compiler(object): def __init__(self, sources, out="out", build="build", install=False, proto3=False, force=False, package=None, includes=None, - clean=False, site_pkg=None): + clean=False): self.sources = sources self.out = os.path.join(out, 'pyrogen') self.build = build self.install = install self.force = force self.package = package - self.includes = includes or [] + self.includes = [os.path.normpath(inc) for inc in includes or []] self.clean = clean here = os.path.dirname(os.path.abspath(__file__)) self.include_path = [os.path.join(here, 'src'), out] - self.site_pkg_path= None - if site_pkg: - site_pkg_path = os.path.join(get_python_lib(),site_pkg) - self.include_path.append(site_pkg_path) - self.site_pkg_path = os.path.join(site_pkg_path,'pyrogen/') self._generated = set() self._messages = [] self._pyx_files = [] @@ -73,8 +68,6 @@ def parse_cli_args(cls): help="name of package to compile to") parser.add_argument('--include', action='append', help="add directory to includes path") - parser.add_argument('--site_pkg', type=str, default=None, - help="build with installed site-package source") parser.add_argument('--clean', action='store_true', help="force recompilation of messages") args = parser.parse_args() @@ -82,7 +75,7 @@ def parse_cli_args(cls): return cls(args.sources, out=args.out_dir, build=args.build_dir, install=args.install, proto3=args.proto3, force=args.force, package=args.package, includes=args.include, - clean=args.clean, site_pkg=args.site_pkg) + clean=args.clean) def compile(self): script_args = ['build', '--build-base={0}'.format(self.build)] @@ -140,14 +133,11 @@ def _generate(self, filename): print("generating {0}".format(filename)) self._generated.add(name) - msg_def = self.parser.parse_from_filename(filename, self.includes,self.site_pkg_path) + msg_def = self.parser.parse_from_filename(filename, self.includes) self._messages.append(msg_def) for f in msg_def['imports']: - if self.site_pkg_path and os.path.exists(self.site_pkg_path+f+'_proto.pyx'): - continue - print("parsing dependency '{}'".format(f)) depends = None @@ -164,6 +154,9 @@ def _generate(self, filename): if self.package is None: self._write(name, msg_def) + if (directory) in self.includes: + self._pyx_files.pop() + print("skip building {0}".format(filename)) def _write(self, name, msg_def): name_pxd = "{}_proto.pxd".format(name) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index 6c4de5e..cb195ec 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -174,7 +174,7 @@ def tokenize(self, disabled_token_types): raise Exception("Unexpected character '{}' on line {}: '{}'".format( self.string[pos], line + 1, self.lines[line])) - def parse(self, site_pkg_path=None, cython_info=True, fname='', includes=None, disabled_tokens=()): + def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): self.verify_parsable_tokens() tokens = self.tokenize(disabled_tokens) rep = {'imports': [], 'messages': [], 'enums': []} @@ -205,9 +205,6 @@ def parse(self, site_pkg_path=None, cython_info=True, fname='', includes=None, d rep['imports'].append(token.value) - if site_pkg_path and os.path.exists(site_pkg_path+token.value+'_proto.pyx'): - continue - # Google's protoc only supports the use of messages and enums # from direct imports. So messages and enums from indirect # imports are not fetched here. @@ -215,7 +212,6 @@ def parse(self, site_pkg_path=None, cython_info=True, fname='', includes=None, d token.value + '.proto', fname, includes, - site_pkg_path, disabled_tokens ) imported['messages'].update( @@ -258,18 +254,18 @@ def parse(self, site_pkg_path=None, cython_info=True, fname='', includes=None, d return rep @classmethod - def parse_from_filename(cls, fname, includes, site_pkg_path, disabled_tokens=None): + def parse_from_filename(cls, fname, includes, disabled_tokens=None): disabled_tokens = disabled_tokens or cls.unsupported_tokens with open(fname, 'r') as fp: s = fp.read() try: - return cls(s).parse(site_pkg_path, fname=fname, includes=includes, disabled_tokens=disabled_tokens) + return cls(s).parse(fname=fname, includes=includes, disabled_tokens=disabled_tokens) except Exception as e: print('Exception while parsing {}'.format(fname)) raise e - def _parse_import(self, fname, parent_fname, includes, site_pkg_path, disabled_tokens): + def _parse_import(self, fname, parent_fname, includes, disabled_tokens): actual_fname = fname if not os.path.isabs(fname): for d in [os.path.dirname(parent_fname)] + includes: @@ -277,7 +273,7 @@ def _parse_import(self, fname, parent_fname, includes, site_pkg_path, disabled_t if os.path.exists(actual_fname): break - return self.__class__.parse_from_filename(actual_fname, includes, site_pkg_path, disabled_tokens) + return self.__class__.parse_from_filename(actual_fname, includes, disabled_tokens) def _process_token_enum(self, token, enums): """ From 16bc2c1d447d2a20a903617732e62aa0a1e58485 Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Wed, 30 Nov 2022 16:05:04 +0100 Subject: [PATCH 29/32] (remove extra lines) --- .gitignore | 3 +-- pyrobuf/compile.py | 5 ++--- pyrobuf/protobuf/templates/proto_pyx.tmpl | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 140ac80..311da27 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,4 @@ tests/out/ tests/build/ /.tox pyrobuf/src/pyrobuf_defs.pxi -.pytest_cache -.mypy_cache \ No newline at end of file +.pytest_cache \ No newline at end of file diff --git a/pyrobuf/compile.py b/pyrobuf/compile.py index 88746fd..6d01150 100644 --- a/pyrobuf/compile.py +++ b/pyrobuf/compile.py @@ -3,7 +3,6 @@ import os import sys from setuptools import setup -from distutils.sysconfig import get_python_lib from Cython.Build import cythonize from pathlib import Path @@ -18,6 +17,7 @@ _VM = sys.version_info.major + class Compiler(object): _env = Environment(loader=PackageLoader('pyrobuf.protobuf', 'templates')) @@ -137,7 +137,6 @@ def _generate(self, filename): self._messages.append(msg_def) for f in msg_def['imports']: - print("parsing dependency '{}'".format(f)) depends = None @@ -150,7 +149,7 @@ def _generate(self, filename): self._generate(depends) except FileNotFoundError: raise FileNotFoundError("can't find message spec for '{}'" - .format(f)) + .format(f)) if self.package is None: self._write(name, msg_def) diff --git a/pyrobuf/protobuf/templates/proto_pyx.tmpl b/pyrobuf/protobuf/templates/proto_pyx.tmpl index fb30b09..6e758e4 100644 --- a/pyrobuf/protobuf/templates/proto_pyx.tmpl +++ b/pyrobuf/protobuf/templates/proto_pyx.tmpl @@ -13,7 +13,6 @@ import enum from pyrogen.{{ import }}_proto cimport * {%- endfor %} - {%- macro enum_fields_def(enum) %} class {{enum.full_name}}(enum.IntEnum): {%- for field in enum.fields.values() %} @@ -23,6 +22,7 @@ class {{enum.full_name}}(enum.IntEnum): {%- macro classdef(message) %} cdef class {{ message.full_name }}: + def __cinit__(self): self._listener = noop_listener From 4ae30e4b928c79ada3d943756ed1aad60c1e11cc Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Wed, 13 Sep 2023 11:28:55 +0200 Subject: [PATCH 30/32] fix parsing custom field option set with key value --- pyrobuf/parse_proto.py | 40 ++++++++++++++++++++++++++++++++-------- setup.py | 2 +- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index cb195ec..4b01581 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -29,6 +29,7 @@ class Parser(object): ('LBRACKET', r'\['), ('RBRACKET', r'\]\s*;'), ('LBRACE', r'\{'), + ('KEY_VALUE', r'\:'), ('RBRACE', r'\}\s*;{0,1}'), ('COMMA', r','), ('SKIP', r'\s'), @@ -58,6 +59,7 @@ class Parser(object): 'SEMICOLON', 'ENUM', 'LBRACE', + 'KEY_VALUE', 'RBRACE', 'EXTENSION', 'ONEOF', @@ -461,7 +463,7 @@ def _parse_field(self, field, tokens): elif token.token_type == 'DEPRECATED': field.deprecated = token.value elif token.token_type == 'CUSTOM': - if self._parse_custom(field, tokens): + if self._parse_custom(field, tokens, token.name): return elif token.token_type == 'COMMA': continue @@ -504,15 +506,18 @@ def _parse_default(self, field, tokens): field.default = token.value - def _parse_custom(self, field, tokens): + def _parse_custom(self, field, tokens, custom_name): """Parse a custom option and return whether or not we hit the closing RBRACKET""" + + custom_name= custom_name[1:-1] # remove () + field.options = dict() token = next(tokens) if token.token_type == 'STRING': - field.value = token.value + field.options[custom_name] = token.value for token in tokens: if token.token_type == 'STRING': - field.value += token.value + field.options[custom_name] += token.value continue elif token.token_type == 'COMMA': return False @@ -520,9 +525,22 @@ def _parse_custom(self, field, tokens): assert token.token_type == 'RBRACKET' return True else: - assert token.token_type in {'NUMERIC', 'BOOLEAN'}, "unexpected custom option value on line {}: '{}'".format( - token.line + 1, self.lines[token.line]) - field.value = token.value + if token.token_type == 'LBRACE': + field.options[custom_name] = dict() + while token: + token = next(tokens) + if token.token_type == 'RBRACE': + return False + elif token.token_type == 'KEY_VALUE': + continue + elif hasattr(token,'name'): + key = token.name + elif hasattr(token,'value'): + field.options[custom_name][key]=token.value + else: + assert token.token_type in {'NUMERIC', 'BOOLEAN'}, "unexpected custom option value on line {}: '{}'".format( + token.line + 1, self.lines[token.line]) + field.options[custom_name] = token.value return False def _parse_enum(self, current, tokens, scope, current_message=None): @@ -570,7 +588,7 @@ def _parse_enum_field(self, field, tokens): if token.token_type == 'LBRACKET': for token in tokens: if token.token_type == 'CUSTOM': - if self._parse_custom(field, tokens): + if self._parse_custom(field, tokens, token.name): return elif token.token_type == 'COMMA': continue @@ -846,6 +864,12 @@ class LBrace(Token): def __init__(self, line): self.line = line + class KEY_VALUE(Token): + token_type = 'KEY_VALUE' + + def __init__(self, line): + self.line = line + class RBrace(Token): token_type = 'RBRACE' diff --git a/setup.py b/setup.py index f5de88b..23f6183 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.3.11" +VERSION = "0.9.3.12" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd" From de28145ecab35b0f58f1385ceb4e58ce3adee77d Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Thu, 14 Sep 2023 15:44:47 +0200 Subject: [PATCH 31/32] update base package naming, add arg: module_name --- pyrobuf/compile.py | 18 +++++++++++++----- pyrobuf/parse_proto.py | 3 ++- pyrobuf/protobuf/templates/proto_pxd.tmpl | 2 +- pyrobuf/protobuf/templates/proto_pyx.tmpl | 2 +- tests/conftest.py | 9 +++++---- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/pyrobuf/compile.py b/pyrobuf/compile.py index 6d01150..8e93ce6 100644 --- a/pyrobuf/compile.py +++ b/pyrobuf/compile.py @@ -25,10 +25,12 @@ class Compiler(object): t_pyx = _env.get_template('proto_pyx.tmpl') def __init__(self, sources, out="out", build="build", install=False, - proto3=False, force=False, package=None, includes=None, + proto3=False, force=False, package=None, module_name=None, includes=None, clean=False): self.sources = sources - self.out = os.path.join(out, 'pyrogen') + self.out = out + if module_name: + self.out = os.path.join(out, module_name) self.build = build self.install = install self.force = force @@ -46,6 +48,9 @@ def __init__(self, sources, out="out", build="build", install=False, else: self.parser = Parser + if module_name: + self.parser.module_name = module_name+'.' + @classmethod def parse_cli_args(cls): parser = argparse.ArgumentParser( @@ -66,6 +71,8 @@ def parse_cli_args(cls): help="force install") parser.add_argument('--package', type=str, default=None, help="name of package to compile to") + parser.add_argument('--module_name', type=str, default=None, + help="name of module to compile to") parser.add_argument('--include', action='append', help="add directory to includes path") parser.add_argument('--clean', action='store_true', @@ -74,7 +81,7 @@ def parse_cli_args(cls): return cls(args.sources, out=args.out_dir, build=args.build_dir, install=args.install, proto3=args.proto3, force=args.force, - package=args.package, includes=args.include, + package=args.package, module_name=args.module_name, includes=args.include, clean=args.clean) def compile(self): @@ -111,8 +118,9 @@ def extend(self, dist): def _compile_spec(self): try: os.makedirs(self.out) - filename = Path(self.out).joinpath('__init__.py') - filename.touch(exist_ok=True) + if self.parser.module_name: + filename = Path(self.out).joinpath('__init__.py') + filename.touch(exist_ok=True) except _FileExistsError: pass diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index 4b01581..0305165 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -143,6 +143,7 @@ class Parser(object): def __init__(self, string): self.string = string self.lines = string.split('\n') + self.module_name = '' def tokenize(self, disabled_token_types): token_type_to_token_class = self.get_token_type_to_token_class_map() @@ -179,7 +180,7 @@ def tokenize(self, disabled_token_types): def parse(self, cython_info=True, fname='', includes=None, disabled_tokens=()): self.verify_parsable_tokens() tokens = self.tokenize(disabled_tokens) - rep = {'imports': [], 'messages': [], 'enums': []} + rep = {'imports': [], 'messages': [], 'enums': [], 'module_name': self.module_name} enums = {} imported = {'messages': {}, 'enums': {}} messages = {} diff --git a/pyrobuf/protobuf/templates/proto_pxd.tmpl b/pyrobuf/protobuf/templates/proto_pxd.tmpl index b4add47..288aa3b 100644 --- a/pyrobuf/protobuf/templates/proto_pxd.tmpl +++ b/pyrobuf/protobuf/templates/proto_pxd.tmpl @@ -8,7 +8,7 @@ from pyrobuf_util cimport * import json {%- for import in imports %} -from pyrogen.{{ import }}_proto cimport * +from {{module_name}}{{ import }}_proto cimport * {%- endfor %} {%- macro classdef(message) %} diff --git a/pyrobuf/protobuf/templates/proto_pyx.tmpl b/pyrobuf/protobuf/templates/proto_pyx.tmpl index 6e758e4..09e4de7 100644 --- a/pyrobuf/protobuf/templates/proto_pyx.tmpl +++ b/pyrobuf/protobuf/templates/proto_pyx.tmpl @@ -10,7 +10,7 @@ import warnings import enum {%- for import in imports %} -from pyrogen.{{ import }}_proto cimport * +from {{module_name}}{{ import }}_proto cimport * {%- endfor %} {%- macro enum_fields_def(enum) %} diff --git a/tests/conftest.py b/tests/conftest.py index ba47eb4..0335937 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,8 +31,9 @@ def pytest_sessionstart(session): if lib_path not in sys.path: sys.path.insert(0, lib_path) - # for short import wihtout pyrogen prefix - lib_path = os.path.join(lib_path,'pyrogen') - if lib_path not in sys.path: - sys.path.insert(0, lib_path) + if compiler.parser.module_name: + # for short import wihtout pyrogen prefix + lib_path = os.path.join(lib_path,compiler.parser.module_name) + if lib_path not in sys.path: + sys.path.insert(0, lib_path) From 9197b00134921108fa14cc2c7bd602847f9a736a Mon Sep 17 00:00:00 2001 From: Christian Toepfer Date: Fri, 22 Sep 2023 14:30:14 +0200 Subject: [PATCH 32/32] fix init static module_name (with pyhton 3.10.12) --- pyrobuf/parse_proto.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyrobuf/parse_proto.py b/pyrobuf/parse_proto.py index 0305165..913ee4b 100644 --- a/pyrobuf/parse_proto.py +++ b/pyrobuf/parse_proto.py @@ -139,11 +139,11 @@ class Parser(object): token_regex = '|'.join('(?P<%s>%s)' % pair for pair in tokens) get_token = re.compile(token_regex).match token_getter = {key: re.compile(val).match for key, val in tokens} + module_name = '' def __init__(self, string): self.string = string self.lines = string.split('\n') - self.module_name = '' def tokenize(self, disabled_token_types): token_type_to_token_class = self.get_token_type_to_token_class_map() diff --git a/setup.py b/setup.py index 23f6183..f835530 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sys -VERSION = "0.9.3.12" +VERSION = "0.9.3.13" HERE = os.path.dirname(os.path.abspath(__file__)) PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi" PYROBUF_LIST_PXD = "pyrobuf_list.pxd"