Skip to content

Commit

Permalink
Add archive-builder functionality to flatdata-py and CLI writer tool (h…
Browse files Browse the repository at this point in the history 
…eremaps#206)

Add rudimentary writing capabilities to flatdata-py:
* Add support for bit-level writing
* Create resources from JSON (slow)
* Add CLI app for direct usage

Signed-off-by: Vlad Bologa <[email protected]>
Signed-off-by: Vishesh Nag <[email protected]>
  • Loading branch information
@vladbologa
vladbologa authored Jul 16, 2021
1 parent 45c1f81 commit c3b7bde
Show file tree
Hide file tree
Showing 29 changed files with 2,944 additions and 95 deletions.
31 changes: 31 additions & 0 deletions flatdata-generator/flatdata/generator/templates/py/python.jinja2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import flatdata.lib as flatdata
class {{ tree.namespace_path(struct, "_") }}_{{ struct.name }}(flatdata.structure.Structure):
"""{{ struct.doc|safe_py_string_line }}"""
_SCHEMA = """{{ tree.schema(struct) }}"""
_NAME = "{{ tree.namespace_path(struct, "_") }}_{{ struct.name }}"
_SIZE_IN_BITS = {{ struct.size_in_bits }}
_SIZE_IN_BYTES = {{ struct.size_in_bytes }}
_FIELDS = {
Expand Down Expand Up @@ -56,10 +57,40 @@ class {{ tree.namespace_path(archive, "_") }}_{{ archive.name }}(flatdata.archiv
flatdata.archive.Archive.__init__(self, resource_storage)
{% endmacro %}

{% macro archive_builder_definition(tree, archive) %}

class {{ tree.namespace_path(archive, "_") }}_{{ archive.name }}Builder(flatdata.archive_builder.ArchiveBuilder):
_SCHEMA = """{{ tree.schema(archive)|safe_py_string_line }}"""
{% for resource in archive.resources if not resource|is_bound_resource %}
_{{ resource.name|upper }}_SCHEMA = """{{ tree.schema(resource)|safe_py_string_line }}"""
_{{ resource.name|upper }}_DOC = """{{ resource.doc|safe_py_string_line }}"""
{% endfor %}
_NAME = "{{ archive.name }}"
_RESOURCES = {
"{{ archive.name }}.archive" : flatdata.archive_builder.ResourceSignature(
container=flatdata.resources.RawData,
initializer=None,
schema=_SCHEMA,
is_optional=False,
doc="Archive signature"),
{% for resource in archive.resources if not resource|is_bound_resource %}
"{{ resource.name }}": flatdata.archive_builder.ResourceSignature(container={{ resource|to_container}},
initializer={{ resource|to_initializer(tree) }},
schema=_{{ resource.name|upper }}_SCHEMA,
is_optional={{ resource.optional }},
doc=_{{ resource.name|upper }}_DOC),
{% endfor %}
}

def __init__(self, resource_storage):
flatdata.archive_builder.ArchiveBuilder.__init__(self, resource_storage)
{% endmacro %}

{% for node in nodes %}
{%- if node|is_structure %}
{{- structure_definition(tree, node) }}
{%- elif node|is_archive %}
{{- archive_definition(tree, node) }}
{{- archive_builder_definition(tree, node) }}
{% endif %}
{% endfor %}
2 changes: 1 addition & 1 deletion flatdata-generator/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

setup(
name="flatdata-generator",
version="0.4.3",
version="0.4.4",
author="Flatdata Developers",
description="Generate source code for C++, Rust, Go or Python from a Flatdata schema file",
long_description=open(os.path.join(SOURCE_FILEPATH, "README.md")).read(),
Expand Down
74 changes: 74 additions & 0 deletions flatdata-generator/tests/generators/py_expectations/archives/comments.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,42 @@ class n_Foo(flatdata.archive.Archive):
def __init__(self, resource_storage):
flatdata.archive.Archive.__init__(self, resource_storage)

class n_FooBuilder(flatdata.archive_builder.ArchiveBuilder):
_SCHEMA = """namespace n {
archive Foo
{
bar : raw_data;
}
}
"""
_BAR_SCHEMA = """namespace n {
archive Foo
{
bar : raw_data;
}
}
"""
_BAR_DOC = """// this is a comment about foo.bar"""
_NAME = "Foo"
_RESOURCES = {
"Foo.archive" : flatdata.archive_builder.ResourceSignature(
container=flatdata.resources.RawData,
initializer=None,
schema=_SCHEMA,
is_optional=False,
doc="Archive signature"),
"bar": flatdata.archive_builder.ResourceSignature(container=flatdata.resources.RawData,
initializer=None,
schema=_BAR_SCHEMA,
is_optional=False,
doc=_BAR_DOC),
}

def __init__(self, resource_storage):
flatdata.archive_builder.ArchiveBuilder.__init__(self, resource_storage)

class n_Bar(flatdata.archive.Archive):
_SCHEMA = """namespace n {
archive Bar
Expand Down Expand Up @@ -72,3 +108,41 @@ class n_Bar(flatdata.archive.Archive):
def __init__(self, resource_storage):
flatdata.archive.Archive.__init__(self, resource_storage)

class n_BarBuilder(flatdata.archive_builder.ArchiveBuilder):
_SCHEMA = """namespace n {
archive Bar
{
foo : raw_data;
}
}
"""
_FOO_SCHEMA = """namespace n {
archive Bar
{
foo : raw_data;
}
}
"""
_FOO_DOC = """/*
* this is a comment about bar.foo
*/"""
_NAME = "Bar"
_RESOURCES = {
"Bar.archive" : flatdata.archive_builder.ResourceSignature(
container=flatdata.resources.RawData,
initializer=None,
schema=_SCHEMA,
is_optional=False,
doc="Archive signature"),
"foo": flatdata.archive_builder.ResourceSignature(container=flatdata.resources.RawData,
initializer=None,
schema=_FOO_SCHEMA,
is_optional=False,
doc=_FOO_DOC),
}

def __init__(self, resource_storage):
flatdata.archive_builder.ArchiveBuilder.__init__(self, resource_storage)

131 changes: 130 additions & 1 deletion flatdata-generator/tests/generators/py_expectations/archives/multivector.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ class n_S(flatdata.structure.Structure):
}
"""
_NAME = "n_S"
_SIZE_IN_BITS = 64
_SIZE_IN_BYTES = 8
_FIELDS = {
Expand All @@ -27,6 +28,7 @@ class n_T(flatdata.structure.Structure):
}
"""
_NAME = "n_T"
_SIZE_IN_BITS = 64
_SIZE_IN_BYTES = 8
_FIELDS = {
Expand All @@ -39,6 +41,7 @@ class n_T(flatdata.structure.Structure):
class _builtin_multivector_IndexType8(flatdata.structure.Structure):
"""/** Builtin type to for MultiVector index */"""
_SCHEMA = """"""
_NAME = "_builtin_multivector_IndexType8"
_SIZE_IN_BITS = 8
_SIZE_IN_BYTES = 1
_FIELDS = {
Expand All @@ -51,6 +54,7 @@ class _builtin_multivector_IndexType8(flatdata.structure.Structure):
class _builtin_multivector_IndexType16(flatdata.structure.Structure):
"""/** Builtin type to for MultiVector index */"""
_SCHEMA = """"""
_NAME = "_builtin_multivector_IndexType16"
_SIZE_IN_BITS = 16
_SIZE_IN_BYTES = 2
_FIELDS = {
Expand All @@ -63,6 +67,7 @@ class _builtin_multivector_IndexType16(flatdata.structure.Structure):
class _builtin_multivector_IndexType64(flatdata.structure.Structure):
"""/** Builtin type to for MultiVector index */"""
_SCHEMA = """"""
_NAME = "_builtin_multivector_IndexType64"
_SIZE_IN_BITS = 64
_SIZE_IN_BYTES = 8
_FIELDS = {
Expand Down Expand Up @@ -193,4 +198,128 @@ class n_A(flatdata.archive.Archive):
}

def __init__(self, resource_storage):
flatdata.archive.Archive.__init__(self, resource_storage)
flatdata.archive.Archive.__init__(self, resource_storage)

class n_ABuilder(flatdata.archive_builder.ArchiveBuilder):
_SCHEMA = """namespace n {
struct S
{
x : u64 : 64;
}
}
namespace n {
struct T
{
x : u64 : 64;
}
}
namespace n {
archive A
{
data : multivector< 8, .n.S, .n.T >;
@optional
optional_data : multivector< 16, .n.S, .n.T >;
data_u64_index : multivector< 64, .n.S, .n.T >;
}
}
"""
_DATA_SCHEMA = """namespace n {
struct S
{
x : u64 : 64;
}
}
namespace n {
struct T
{
x : u64 : 64;
}
}
namespace n {
archive A
{
data : multivector< 8, .n.S, .n.T >;
}
}
"""
_DATA_DOC = """"""
_OPTIONAL_DATA_SCHEMA = """namespace n {
struct S
{
x : u64 : 64;
}
}
namespace n {
struct T
{
x : u64 : 64;
}
}
namespace n {
archive A
{
@optional
optional_data : multivector< 16, .n.S, .n.T >;
}
}
"""
_OPTIONAL_DATA_DOC = """"""
_DATA_U64_INDEX_SCHEMA = """namespace n {
struct S
{
x : u64 : 64;
}
}
namespace n {
struct T
{
x : u64 : 64;
}
}
namespace n {
archive A
{
data_u64_index : multivector< 64, .n.S, .n.T >;
}
}
"""
_DATA_U64_INDEX_DOC = """"""
_NAME = "A"
_RESOURCES = {
"A.archive" : flatdata.archive_builder.ResourceSignature(
container=flatdata.resources.RawData,
initializer=None,
schema=_SCHEMA,
is_optional=False,
doc="Archive signature"),
"data": flatdata.archive_builder.ResourceSignature(container=flatdata.resources.Multivector,
initializer=[_builtin_multivector_IndexType8,n_S,n_T],
schema=_DATA_SCHEMA,
is_optional=False,
doc=_DATA_DOC),
"optional_data": flatdata.archive_builder.ResourceSignature(container=flatdata.resources.Multivector,
initializer=[_builtin_multivector_IndexType16,n_S,n_T],
schema=_OPTIONAL_DATA_SCHEMA,
is_optional=True,
doc=_OPTIONAL_DATA_DOC),
"data_u64_index": flatdata.archive_builder.ResourceSignature(container=flatdata.resources.Multivector,
initializer=[_builtin_multivector_IndexType64,n_S,n_T],
schema=_DATA_U64_INDEX_SCHEMA,
is_optional=False,
doc=_DATA_U64_INDEX_DOC),
}

def __init__(self, resource_storage):
flatdata.archive_builder.ArchiveBuilder.__init__(self, resource_storage)
Loading

0 comments on commit c3b7bde

Please sign in to comment.