From a03497e13a0a7d59e644ba0ad6fccb0ad00e73d9 Mon Sep 17 00:00:00 2001 From: zethson Date: Wed, 17 Jul 2024 18:09:25 +0200 Subject: [PATCH 01/15] __repr__ draft Signed-off-by: zethson --- lnschema_core/models.py | 63 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 9c079370..45c9e3a6 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -13,8 +13,9 @@ from django.db import models from django.db.models import CASCADE, PROTECT from django.db.models.base import ModelBase -from lamin_utils import logger -from lamindb_setup import _check_instance_setup, settings +from lamin_utils import colors, logger +from lamindb_setup import _check_instance_setup +from django.db.models.fields.related import ForeignKey, ManyToManyField from lnschema_core.types import ( CharField, @@ -509,6 +510,61 @@ def __dir__(cls): if not attr.startswith("__") and attr not in result: result.append(attr) return result + + def __repr__(cls) -> str: + repr_str = f"{colors.green(cls.__name__)}\n" + + fields = cls._meta.fields + direct_fields = [] + foreign_key_fields = [] + for f in fields: + if f.is_relation: + foreign_key_fields.append(f.name) + else: + direct_fields.append(f.name) + + # Get all many-to-many relationships + # many_to_many = [ + # field.name for field in cls._meta.get_fields() + # if isinstance(field, ManyToManyField) + # ] + + foreign_key_fields = [ + field.name for field in cls._meta.get_fields() + if isinstance(field, ForeignKey) + ] + + + + + # Provenance + repr_str += f" {colors.italic('Provenance')}\n" + if foreign_key_fields: + type_str = ( + lambda field: f": {cls._meta.get_field(field).related_model.__name__}" + ) + related_msg = "".join( + [ + f" .{field_name}{type_str(field_name)}\n" + for field_name in foreign_key_fields + ] + ) + repr_str += related_msg + + # Linked fields + all_fields = set([f.name for f in cls._meta.get_fields()]) + link_fields = all_fields - set(foreign_key_fields) + + print(link_fields) + + + # 2. Define a blacklist of things that we don't want to show -> remove them + # 3. Divide into provenance related fields and Link fields -> print them + + + # repr_str += f"<{cls.__name__}({', '.join(all_fields)})>" + + return repr_str def from_values( cls, @@ -749,8 +805,7 @@ def save(self, *args, **kwargs) -> Registry: class Meta: abstract = True - - + class FeatureManager: """Feature manager.""" From 030bc4999f5111f5b36f7a22fddee235e4af6c4a Mon Sep 17 00:00:00 2001 From: zethson Date: Wed, 17 Jul 2024 18:54:57 +0200 Subject: [PATCH 02/15] Add __repr__ for Registry.__repr__(cls) Signed-off-by: zethson --- lnschema_core/models.py | 78 ++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 33 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 17ac963f..66155135 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -14,9 +14,14 @@ from django.db import models from django.db.models import CASCADE, PROTECT from django.db.models.base import ModelBase +from django.db.models.fields.related import ( + ForeignKey, + ManyToManyField, + ManyToManyRel, + ManyToOneRel, +) from lamin_utils import colors, logger from lamindb_setup import _check_instance_setup -from django.db.models.fields.related import ForeignKey, ManyToManyField from lnschema_core.types import ( CharField, @@ -520,33 +525,34 @@ def include_attribute(attr_name, attr_value): if not attr.startswith("__") and attr not in result: result.append(attr) return result - + def __repr__(cls) -> str: repr_str = f"{colors.green(cls.__name__)}\n" - + fields = cls._meta.fields - direct_fields = [] foreign_key_fields = [] for f in fields: if f.is_relation: foreign_key_fields.append(f.name) - else: - direct_fields.append(f.name) - - # Get all many-to-many relationships - # many_to_many = [ - # field.name for field in cls._meta.get_fields() - # if isinstance(field, ManyToManyField) - # ] - + foreign_key_fields = [ - field.name for field in cls._meta.get_fields() + field.name + for field in cls._meta.get_fields() if isinstance(field, ForeignKey) ] - - - + many_to_one_rel = [ + field.name + for field in cls._meta.get_fields() + if isinstance(field, ManyToOneRel) + ] + + many_to_many_rel = [ + field.name + for field in cls._meta.get_fields() + if isinstance(field, ManyToManyRel) + ] + # Provenance repr_str += f" {colors.italic('Provenance')}\n" if foreign_key_fields: @@ -559,21 +565,26 @@ def __repr__(cls) -> str: for field_name in foreign_key_fields ] ) - repr_str += related_msg - - # Linked fields - all_fields = set([f.name for f in cls._meta.get_fields()]) - link_fields = all_fields - set(foreign_key_fields) - - print(link_fields) - - - # 2. Define a blacklist of things that we don't want to show -> remove them - # 3. Divide into provenance related fields and Link fields -> print them - - - # repr_str += f"<{cls.__name__}({', '.join(all_fields)})>" - + repr_str += related_msg + + # Relationships + # We do not want to duplicate relationship fields so we remove all *_links fields. + many_to_one_rel = list( + filter(lambda field: not field.endswith("_links"), many_to_one_rel) + ) + + if many_to_one_rel: + repr_str += f" {colors.italic('Many-to-one relationships')}\n" + many_to_one_msg = "".join(f" .{field}\n" for field in many_to_one_rel) + repr_str += many_to_one_msg + + if many_to_many_rel: + repr_str += f" {colors.italic('Many-to-many relationships')}\n" + many_to_many_msg = "".join(f" .{field}\n" for field in many_to_many_rel) + repr_str += many_to_many_msg + + repr_str = repr_str.rstrip("\n") + return repr_str def from_values( @@ -799,7 +810,8 @@ def save(self, *args, **kwargs) -> Record: class Meta: abstract = True - + + class FeatureManager: """Feature manager.""" From ed676022be2a6c26a409ba9ac96e818beffbc734 Mon Sep 17 00:00:00 2001 From: zethson Date: Thu, 18 Jul 2024 11:18:50 +0200 Subject: [PATCH 03/15] Refactoring Signed-off-by: zethson --- lnschema_core/models.py | 43 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 66155135..ca45331d 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -531,9 +531,9 @@ def __repr__(cls) -> str: fields = cls._meta.fields foreign_key_fields = [] - for f in fields: - if f.is_relation: - foreign_key_fields.append(f.name) + for field in fields: + if field.is_relation: + foreign_key_fields.append(field.name) foreign_key_fields = [ field.name @@ -541,33 +541,28 @@ def __repr__(cls) -> str: if isinstance(field, ForeignKey) ] - many_to_one_rel = [ - field.name - for field in cls._meta.get_fields() - if isinstance(field, ManyToOneRel) - ] - - many_to_many_rel = [ - field.name - for field in cls._meta.get_fields() - if isinstance(field, ManyToManyRel) - ] - + get_type_str = ( + lambda field: f": {cls._meta.get_field(field).related_model.__name__}" + ) + # Provenance repr_str += f" {colors.italic('Provenance')}\n" if foreign_key_fields: - type_str = ( - lambda field: f": {cls._meta.get_field(field).related_model.__name__}" - ) related_msg = "".join( [ - f" .{field_name}{type_str(field_name)}\n" + f" .{field_name}{get_type_str(field_name)}\n" for field_name in foreign_key_fields ] ) repr_str += related_msg # Relationships + many_to_one_rel = [ + field.name + for field in cls._meta.get_fields() + if isinstance(field, ManyToOneRel) + ] + # We do not want to duplicate relationship fields so we remove all *_links fields. many_to_one_rel = list( filter(lambda field: not field.endswith("_links"), many_to_one_rel) @@ -575,12 +570,18 @@ def __repr__(cls) -> str: if many_to_one_rel: repr_str += f" {colors.italic('Many-to-one relationships')}\n" - many_to_one_msg = "".join(f" .{field}\n" for field in many_to_one_rel) + many_to_one_msg = "".join(f" .{field_name}{get_type_str(field_name)}\n" for field_name in many_to_one_rel) repr_str += many_to_one_msg + + many_to_many_rel = [ + field.name + for field in cls._meta.get_fields() + if isinstance(field, ManyToManyRel) + ] if many_to_many_rel: repr_str += f" {colors.italic('Many-to-many relationships')}\n" - many_to_many_msg = "".join(f" .{field}\n" for field in many_to_many_rel) + many_to_many_msg = "".join(f" .{field_name}{get_type_str(field_name)}\n" for field_name in many_to_many_rel) repr_str += many_to_many_msg repr_str = repr_str.rstrip("\n") From 2676a4c2744a14407bdeee9644b4b7040eb65eab Mon Sep 17 00:00:00 2001 From: zethson Date: Thu, 18 Jul 2024 11:20:36 +0200 Subject: [PATCH 04/15] pre-commit Signed-off-by: zethson --- lnschema_core/models.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index ca45331d..ab0af7f6 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -542,9 +542,9 @@ def __repr__(cls) -> str: ] get_type_str = ( - lambda field: f": {cls._meta.get_field(field).related_model.__name__}" + lambda field: f": {cls._meta.get_field(field).related_model.__name__}" ) - + # Provenance repr_str += f" {colors.italic('Provenance')}\n" if foreign_key_fields: @@ -562,7 +562,7 @@ def __repr__(cls) -> str: for field in cls._meta.get_fields() if isinstance(field, ManyToOneRel) ] - + # We do not want to duplicate relationship fields so we remove all *_links fields. many_to_one_rel = list( filter(lambda field: not field.endswith("_links"), many_to_one_rel) @@ -570,9 +570,12 @@ def __repr__(cls) -> str: if many_to_one_rel: repr_str += f" {colors.italic('Many-to-one relationships')}\n" - many_to_one_msg = "".join(f" .{field_name}{get_type_str(field_name)}\n" for field_name in many_to_one_rel) + many_to_one_msg = "".join( + f" .{field_name}{get_type_str(field_name)}\n" + for field_name in many_to_one_rel + ) repr_str += many_to_one_msg - + many_to_many_rel = [ field.name for field in cls._meta.get_fields() @@ -581,7 +584,10 @@ def __repr__(cls) -> str: if many_to_many_rel: repr_str += f" {colors.italic('Many-to-many relationships')}\n" - many_to_many_msg = "".join(f" .{field_name}{get_type_str(field_name)}\n" for field_name in many_to_many_rel) + many_to_many_msg = "".join( + f" .{field_name}{get_type_str(field_name)}\n" + for field_name in many_to_many_rel + ) repr_str += many_to_many_msg repr_str = repr_str.rstrip("\n") From 10e166cee38aeaae6c82cb6d179bcd912d04c1fd Mon Sep 17 00:00:00 2001 From: zethson Date: Thu, 18 Jul 2024 14:48:12 +0200 Subject: [PATCH 05/15] Add proper schema types Signed-off-by: zethson --- lnschema_core/models.py | 46 +++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index ab0af7f6..1133372e 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -1,6 +1,7 @@ from __future__ import annotations import builtins +import re from datetime import datetime from typing import ( TYPE_CHECKING, @@ -541,8 +542,8 @@ def __repr__(cls) -> str: if isinstance(field, ForeignKey) ] - get_type_str = ( - lambda field: f": {cls._meta.get_field(field).related_model.__name__}" + _get_type_for_field = ( + lambda field: f"{cls._meta.get_field(field).related_model.__name__}" ) # Provenance @@ -550,45 +551,46 @@ def __repr__(cls) -> str: if foreign_key_fields: related_msg = "".join( [ - f" .{field_name}{get_type_str(field_name)}\n" + f" .{field_name}: {_get_type_for_field(field_name)}\n" for field_name in foreign_key_fields ] ) repr_str += related_msg # Relationships + def _get_related_field_type(field) -> str: + field_type = ( + field.related_model.__get_name_with_schema__() + .replace("Artifact", "") + .replace("Collection", "") + ) + return ( + _get_type_for_field(field.name) + if not field_type.strip() + else field_type + ) + many_to_one_rel = [ - field.name + f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" for field in cls._meta.get_fields() if isinstance(field, ManyToOneRel) + and not field.name.endswith( + "_links" + ) # we're filtering the _links out to not clutter with duplications ] - # We do not want to duplicate relationship fields so we remove all *_links fields. - many_to_one_rel = list( - filter(lambda field: not field.endswith("_links"), many_to_one_rel) - ) - if many_to_one_rel: - repr_str += f" {colors.italic('Many-to-one relationships')}\n" - many_to_one_msg = "".join( - f" .{field_name}{get_type_str(field_name)}\n" - for field_name in many_to_one_rel - ) - repr_str += many_to_one_msg + repr_str += f" {colors.italic('Relational fields')}\n" + repr_str += "".join(many_to_one_rel) many_to_many_rel = [ - field.name + f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" for field in cls._meta.get_fields() if isinstance(field, ManyToManyRel) ] if many_to_many_rel: - repr_str += f" {colors.italic('Many-to-many relationships')}\n" - many_to_many_msg = "".join( - f" .{field_name}{get_type_str(field_name)}\n" - for field_name in many_to_many_rel - ) - repr_str += many_to_many_msg + repr_str += "".join(many_to_many_rel) repr_str = repr_str.rstrip("\n") From 0e08021a7e546b56cb9230f86ccbf8d49a8dcef4 Mon Sep 17 00:00:00 2001 From: zethson Date: Thu, 18 Jul 2024 15:27:08 +0200 Subject: [PATCH 06/15] Add primitive types Signed-off-by: zethson --- lnschema_core/models.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 1133372e..1285a20e 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -530,34 +530,42 @@ def include_attribute(attr_name, attr_value): def __repr__(cls) -> str: repr_str = f"{colors.green(cls.__name__)}\n" + def _get_type_for_field(field_name: str) -> str: + field = cls._meta.get_field(field_name) + related_model_name = ( + field.related_model.__name__ + if hasattr(field, "related_model") and field.related_model + else None + ) + return ( + related_model_name if related_model_name else field.get_internal_type() + ) + + # Primitive fields fields = cls._meta.fields - foreign_key_fields = [] + non_relational_fields = [] for field in fields: if field.is_relation: - foreign_key_fields.append(field.name) + non_relational_fields.append(field.name) - foreign_key_fields = [ + non_relational_fields = [ field.name for field in cls._meta.get_fields() - if isinstance(field, ForeignKey) + if not (isinstance(field, ManyToOneRel) or isinstance(field, ManyToManyRel)) ] + non_relational_fields = sorted(non_relational_fields) - _get_type_for_field = ( - lambda field: f"{cls._meta.get_field(field).related_model.__name__}" - ) - - # Provenance - repr_str += f" {colors.italic('Provenance')}\n" - if foreign_key_fields: + repr_str += f" {colors.italic('Primitive fields')}\n" + if non_relational_fields: related_msg = "".join( [ f" .{field_name}: {_get_type_for_field(field_name)}\n" - for field_name in foreign_key_fields + for field_name in non_relational_fields ] ) repr_str += related_msg - # Relationships + # Relational fields def _get_related_field_type(field) -> str: field_type = ( field.related_model.__get_name_with_schema__() @@ -578,6 +586,7 @@ def _get_related_field_type(field) -> str: "_links" ) # we're filtering the _links out to not clutter with duplications ] + many_to_one_rel = sorted(many_to_one_rel) if many_to_one_rel: repr_str += f" {colors.italic('Relational fields')}\n" @@ -588,6 +597,7 @@ def _get_related_field_type(field) -> str: for field in cls._meta.get_fields() if isinstance(field, ManyToManyRel) ] + many_to_many_rel = sorted(many_to_many_rel) if many_to_many_rel: repr_str += "".join(many_to_many_rel) From 449e87fc01d1563b4805c9b4bb99e47078349a1b Mon Sep 17 00:00:00 2001 From: zethson Date: Thu, 18 Jul 2024 16:22:36 +0200 Subject: [PATCH 07/15] Differentiate between bionty fields and other fields Signed-off-by: zethson --- lnschema_core/models.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 1285a20e..1a6ac34e 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -586,21 +586,26 @@ def _get_related_field_type(field) -> str: "_links" ) # we're filtering the _links out to not clutter with duplications ] - many_to_one_rel = sorted(many_to_one_rel) - - if many_to_one_rel: - repr_str += f" {colors.italic('Relational fields')}\n" - repr_str += "".join(many_to_one_rel) many_to_many_rel = [ f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" for field in cls._meta.get_fields() if isinstance(field, ManyToManyRel) ] - many_to_many_rel = sorted(many_to_many_rel) + relational_fields = sorted(many_to_one_rel + many_to_many_rel) + + non_bionty_fields = [ + field for field in relational_fields if "bionty" not in field + ] + bionty_fields = [field for field in relational_fields if "bionty" in field] + + if non_bionty_fields: + repr_str += f" {colors.italic('Relational fields')}\n" + repr_str += "".join(non_bionty_fields) - if many_to_many_rel: - repr_str += "".join(many_to_many_rel) + if bionty_fields: + repr_str += f" {colors.italic('Bionty fields')}\n" + repr_str += "".join(bionty_fields) repr_str = repr_str.rstrip("\n") From c9cd430b609a713026f6944873290aa07c0d99f3 Mon Sep 17 00:00:00 2001 From: zethson Date: Thu, 18 Jul 2024 17:06:41 +0200 Subject: [PATCH 08/15] :art: Restore original order Signed-off-by: zethson --- lnschema_core/models.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 1a6ac34e..cc751130 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -553,7 +553,18 @@ def _get_type_for_field(field_name: str) -> str: for field in cls._meta.get_fields() if not (isinstance(field, ManyToOneRel) or isinstance(field, ManyToManyRel)) ] - non_relational_fields = sorted(non_relational_fields) + + # We prefer having the provenance fields at the end just like we do in the docs + provenance_fields = {"created_at", "created_by", "updated_at"} + non_relational_fields = non_relational_fields[:] = [ + s + for s in non_relational_fields + if not any(field in s for field in provenance_fields) + ] + [ + s + for s in non_relational_fields + if any(field in s for field in provenance_fields) + ] repr_str += f" {colors.italic('Primitive fields')}\n" if non_relational_fields: @@ -592,7 +603,7 @@ def _get_related_field_type(field) -> str: for field in cls._meta.get_fields() if isinstance(field, ManyToManyRel) ] - relational_fields = sorted(many_to_one_rel + many_to_many_rel) + relational_fields = many_to_one_rel + many_to_many_rel non_bionty_fields = [ field for field in relational_fields if "bionty" not in field From 25ace6f727767a3e29550934e1dfd07cd382fe72 Mon Sep 17 00:00:00 2001 From: zethson Date: Thu, 18 Jul 2024 17:56:24 +0200 Subject: [PATCH 09/15] :sparkles: Generalize to more schemas Signed-off-by: zethson --- lnschema_core/models.py | 50 ++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index cc751130..750a155a 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -589,34 +589,58 @@ def _get_related_field_type(field) -> str: else field_type ) - many_to_one_rel = [ - f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" + many_to_one_rel_fields = [ + field for field in cls._meta.get_fields() if isinstance(field, ManyToOneRel) and not field.name.endswith( "_links" ) # we're filtering the _links out to not clutter with duplications ] - - many_to_many_rel = [ + many_to_one_rel_formatted = [ f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" + for field in many_to_one_rel_fields + ] + + many_to_many_rel_fields = [ + field for field in cls._meta.get_fields() if isinstance(field, ManyToManyRel) ] - relational_fields = many_to_one_rel + many_to_many_rel + many_to_many_rel_formatted = [ + f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" + for field in many_to_many_rel_fields + ] + + relational_fields_formatted = ( + many_to_one_rel_formatted + many_to_many_rel_formatted + ) - non_bionty_fields = [ - field for field in relational_fields if "bionty" not in field + external_schemas = set() + for field in relational_fields_formatted: + # There is an external schema + field_type = field.split(":")[1].split()[0] + if len(field_type.split(".")) >= 2: + external_schemas.add(field_type.split(".")[0]) + + non_external_schema_fields = [ + field + for field in relational_fields_formatted + if len(field.split(":")[1].split(".")) < 2 ] - bionty_fields = [field for field in relational_fields if "bionty" in field] - if non_bionty_fields: + if relational_fields_formatted: repr_str += f" {colors.italic('Relational fields')}\n" - repr_str += "".join(non_bionty_fields) + repr_str += "".join(non_external_schema_fields) + + for ext_schema in external_schemas: + ext_schema_fields = [ + field for field in relational_fields_formatted if ext_schema in field + ] - if bionty_fields: - repr_str += f" {colors.italic('Bionty fields')}\n" - repr_str += "".join(bionty_fields) + if ext_schema_fields: + repr_str += f" {colors.italic(f'{ext_schema} fields')}\n" + repr_str += "".join(ext_schema_fields) repr_str = repr_str.rstrip("\n") From 30478545c06104c6c7b2c7fa60dff8a3357d949c Mon Sep 17 00:00:00 2001 From: zethson Date: Thu, 18 Jul 2024 18:19:42 +0200 Subject: [PATCH 10/15] :art: Include FKs and simplify Signed-off-by: zethson --- lnschema_core/models.py | 52 ++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 750a155a..1263dcb8 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -551,11 +551,15 @@ def _get_type_for_field(field_name: str) -> str: non_relational_fields = [ field.name for field in cls._meta.get_fields() - if not (isinstance(field, ManyToOneRel) or isinstance(field, ManyToManyRel)) + if not ( + isinstance(field, ManyToOneRel) + or isinstance(field, ManyToManyRel) + or isinstance(field, ForeignKey) + ) ] # We prefer having the provenance fields at the end just like we do in the docs - provenance_fields = {"created_at", "created_by", "updated_at"} + provenance_fields = {"created_at", "updated_at"} non_relational_fields = non_relational_fields[:] = [ s for s in non_relational_fields @@ -589,33 +593,24 @@ def _get_related_field_type(field) -> str: else field_type ) - many_to_one_rel_fields = [ + relational_fields = [ field for field in cls._meta.get_fields() - if isinstance(field, ManyToOneRel) + if ( + isinstance(field, ManyToOneRel) + or isinstance(field, ManyToManyRel) + or isinstance(field, ForeignKey) + ) and not field.name.endswith( "_links" ) # we're filtering the _links out to not clutter with duplications ] - many_to_one_rel_formatted = [ - f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" - for field in many_to_one_rel_fields - ] - many_to_many_rel_fields = [ - field - for field in cls._meta.get_fields() - if isinstance(field, ManyToManyRel) - ] - many_to_many_rel_formatted = [ + relational_fields_formatted = [ f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" - for field in many_to_many_rel_fields + for field in relational_fields ] - relational_fields_formatted = ( - many_to_one_rel_formatted + many_to_many_rel_formatted - ) - external_schemas = set() for field in relational_fields_formatted: # There is an external schema @@ -629,18 +624,21 @@ def _get_related_field_type(field) -> str: if len(field.split(":")[1].split(".")) < 2 ] - if relational_fields_formatted: + if non_external_schema_fields: repr_str += f" {colors.italic('Relational fields')}\n" repr_str += "".join(non_external_schema_fields) - for ext_schema in external_schemas: - ext_schema_fields = [ - field for field in relational_fields_formatted if ext_schema in field - ] + if external_schemas: + for ext_schema in external_schemas: + ext_schema_fields = [ + field + for field in relational_fields_formatted + if ext_schema in field + ] - if ext_schema_fields: - repr_str += f" {colors.italic(f'{ext_schema} fields')}\n" - repr_str += "".join(ext_schema_fields) + if ext_schema_fields: + repr_str += f" {colors.italic(f'{ext_schema} fields')}\n" + repr_str += "".join(ext_schema_fields) repr_str = repr_str.rstrip("\n") From 90ea13a2fe7b279a419f2bb6b4ff761fec977b8b Mon Sep 17 00:00:00 2001 From: zethson Date: Fri, 19 Jul 2024 09:32:42 +0200 Subject: [PATCH 11/15] :art: Reorder by base class Signed-off-by: zethson --- lnschema_core/models.py | 84 ++++++++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 18 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 1263dcb8..28c94ee2 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -1,7 +1,6 @@ from __future__ import annotations import builtins -import re from datetime import datetime from typing import ( TYPE_CHECKING, @@ -13,7 +12,7 @@ ) from django.db import models -from django.db.models import CASCADE, PROTECT +from django.db.models import CASCADE, PROTECT, Field from django.db.models.base import ModelBase from django.db.models.fields.related import ( ForeignKey, @@ -541,6 +540,29 @@ def _get_type_for_field(field_name: str) -> str: related_model_name if related_model_name else field.get_internal_type() ) + def _get_base_class_fields(cls: models.Model) -> list[str]: + base_fields = [] + for base in cls.__bases__: + if hasattr(base, "_meta"): + base_fields.extend( + [field.name for field in base._meta.get_fields()] + ) + return base_fields + + def _reorder_fields_by_class(fields_to_order: list[Field]) -> list[Field]: + """Reorders the fields so that base class fields come last.""" + non_base_class_fields = [ + field + for field in fields_to_order + if field.name not in _get_base_class_fields(cls) + ] + found_base_class_fields = [ + field + for field in fields_to_order + if field.name in _get_base_class_fields(cls) + ] + return non_base_class_fields + found_base_class_fields + # Primitive fields fields = cls._meta.fields non_relational_fields = [] @@ -549,32 +571,23 @@ def _get_type_for_field(field_name: str) -> str: non_relational_fields.append(field.name) non_relational_fields = [ - field.name + field for field in cls._meta.get_fields() if not ( isinstance(field, ManyToOneRel) or isinstance(field, ManyToManyRel) + or isinstance(field, ManyToManyField) or isinstance(field, ForeignKey) ) ] - # We prefer having the provenance fields at the end just like we do in the docs - provenance_fields = {"created_at", "updated_at"} - non_relational_fields = non_relational_fields[:] = [ - s - for s in non_relational_fields - if not any(field in s for field in provenance_fields) - ] + [ - s - for s in non_relational_fields - if any(field in s for field in provenance_fields) - ] + non_relational_fields = _reorder_fields_by_class(non_relational_fields) repr_str += f" {colors.italic('Primitive fields')}\n" if non_relational_fields: related_msg = "".join( [ - f" .{field_name}: {_get_type_for_field(field_name)}\n" + f" .{field_name.name}: {_get_type_for_field(field_name.name)}\n" for field_name in non_relational_fields ] ) @@ -593,22 +606,49 @@ def _get_related_field_type(field) -> str: else field_type ) - relational_fields = [ + class_specific_relational_fields = [ + field + for field in cls._meta.fields + cls._meta.many_to_many + if ( + isinstance(field, ManyToOneRel) + or isinstance(field, ManyToManyRel) + or isinstance(field, ManyToManyField) + or isinstance(field, ForeignKey) + ) + and not field.name.endswith( + "_links" + ) # we're filtering the _links out to not clutter with duplications + ] + non_class_specific_relational_fields = [ field for field in cls._meta.get_fields() if ( isinstance(field, ManyToOneRel) or isinstance(field, ManyToManyRel) + or isinstance(field, ManyToManyField) or isinstance(field, ForeignKey) ) and not field.name.endswith( "_links" ) # we're filtering the _links out to not clutter with duplications ] + non_class_specific_relational_fields = _reorder_fields_by_class( + non_class_specific_relational_fields + ) + + # Ensure that class specific fields (e.g. Artifact) come before non-class specific fields (e.g. collection) + filtered_non_class_specific = [ + field + for field in non_class_specific_relational_fields + if field not in class_specific_relational_fields + ] + ordered_relational_fields = ( + class_specific_relational_fields + filtered_non_class_specific + ) relational_fields_formatted = [ f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" - for field in relational_fields + for field in ordered_relational_fields ] external_schemas = set() @@ -629,6 +669,12 @@ def _get_related_field_type(field) -> str: repr_str += "".join(non_external_schema_fields) if external_schemas: + # We want Bionty to show up before other schemas + external_schemas = ( + ["bionty"] + sorted(external_schemas - {"bionty"}) # type: ignore + if "bionty" in external_schemas + else sorted(external_schemas) + ) for ext_schema in external_schemas: ext_schema_fields = [ field @@ -637,7 +683,9 @@ def _get_related_field_type(field) -> str: ] if ext_schema_fields: - repr_str += f" {colors.italic(f'{ext_schema} fields')}\n" + repr_str += ( + f" {colors.italic(f'{ext_schema.capitalize()} fields')}\n" + ) repr_str += "".join(ext_schema_fields) repr_str = repr_str.rstrip("\n") From 14b3871a4df50f2f1015ce94ec8f0d8d93e509be Mon Sep 17 00:00:00 2001 From: zethson Date: Fri, 19 Jul 2024 09:37:41 +0200 Subject: [PATCH 12/15] :art: Rename to Basic fields Signed-off-by: zethson --- lnschema_core/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 28c94ee2..dc645e24 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -583,7 +583,7 @@ def _reorder_fields_by_class(fields_to_order: list[Field]) -> list[Field]: non_relational_fields = _reorder_fields_by_class(non_relational_fields) - repr_str += f" {colors.italic('Primitive fields')}\n" + repr_str += f" {colors.italic('Basic fields')}\n" if non_relational_fields: related_msg = "".join( [ From d62061a1a5492ecd25db4ea136dc75fb9795d73d Mon Sep 17 00:00:00 2001 From: zethson Date: Fri, 19 Jul 2024 10:26:51 +0200 Subject: [PATCH 13/15] :art: Refactor Signed-off-by: zethson --- lnschema_core/models.py | 110 +++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 57 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index dc645e24..74a7b2a6 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -527,7 +527,14 @@ def include_attribute(attr_name, attr_value): return result def __repr__(cls) -> str: - repr_str = f"{colors.green(cls.__name__)}\n" + """Designed to show schema version when Records are called directly. + + Divided into several parts: + 1. Header + 2. Basic fields where we first show class specific fields and then base class fields + 3. Relational fields where we show class specific relational fields and then base class fields + 4. External schema fields where we show loaded schemas such as Bionty, wetlab and others + """ def _get_type_for_field(field_name: str) -> str: field = cls._meta.get_field(field_name) @@ -541,13 +548,12 @@ def _get_type_for_field(field_name: str) -> str: ) def _get_base_class_fields(cls: models.Model) -> list[str]: - base_fields = [] - for base in cls.__bases__: - if hasattr(base, "_meta"): - base_fields.extend( - [field.name for field in base._meta.get_fields()] - ) - return base_fields + return [ + field.name + for base in cls.__bases__ + if hasattr(base, "_meta") + for field in base._meta.get_fields() + ] def _reorder_fields_by_class(fields_to_order: list[Field]) -> list[Field]: """Reorders the fields so that base class fields come last.""" @@ -563,14 +569,11 @@ def _reorder_fields_by_class(fields_to_order: list[Field]) -> list[Field]: ] return non_base_class_fields + found_base_class_fields - # Primitive fields - fields = cls._meta.fields - non_relational_fields = [] - for field in fields: - if field.is_relation: - non_relational_fields.append(field.name) + # Header + repr_str = f"{colors.green(cls.__name__)}\n" - non_relational_fields = [ + # Basic fields + basic_fields = [ field for field in cls._meta.get_fields() if not ( @@ -580,57 +583,32 @@ def _reorder_fields_by_class(fields_to_order: list[Field]) -> list[Field]: or isinstance(field, ForeignKey) ) ] - - non_relational_fields = _reorder_fields_by_class(non_relational_fields) + basic_fields = _reorder_fields_by_class(basic_fields) repr_str += f" {colors.italic('Basic fields')}\n" - if non_relational_fields: - related_msg = "".join( + if basic_fields: + repr_str += "".join( [ f" .{field_name.name}: {_get_type_for_field(field_name.name)}\n" - for field_name in non_relational_fields + for field_name in basic_fields ] ) - repr_str += related_msg # Relational fields - def _get_related_field_type(field) -> str: - field_type = ( - field.related_model.__get_name_with_schema__() - .replace("Artifact", "") - .replace("Collection", "") - ) - return ( - _get_type_for_field(field.name) - if not field_type.strip() - else field_type - ) + relational_fields = (ManyToOneRel, ManyToManyRel, ManyToManyField, ForeignKey) class_specific_relational_fields = [ field for field in cls._meta.fields + cls._meta.many_to_many - if ( - isinstance(field, ManyToOneRel) - or isinstance(field, ManyToManyRel) - or isinstance(field, ManyToManyField) - or isinstance(field, ForeignKey) - ) - and not field.name.endswith( - "_links" - ) # we're filtering the _links out to not clutter with duplications + if isinstance(field, relational_fields) + and not field.name.endswith("_links") ] + non_class_specific_relational_fields = [ field for field in cls._meta.get_fields() - if ( - isinstance(field, ManyToOneRel) - or isinstance(field, ManyToManyRel) - or isinstance(field, ManyToManyField) - or isinstance(field, ForeignKey) - ) - and not field.name.endswith( - "_links" - ) # we're filtering the _links out to not clutter with duplications + if isinstance(field, relational_fields) + and not field.name.endswith("_links") ] non_class_specific_relational_fields = _reorder_fields_by_class( non_class_specific_relational_fields @@ -646,18 +624,28 @@ def _get_related_field_type(field) -> str: class_specific_relational_fields + filtered_non_class_specific ) + def _get_related_field_type(field) -> str: + field_type = ( + field.related_model.__get_name_with_schema__() + .replace( + "Artifact", "" + ) # some fields have an unnecessary 'Artifact' in their name + .replace( + "Collection", "" + ) # some fields have an unnecessary 'Collection' in their name + ) + return ( + _get_type_for_field(field.name) + if not field_type.strip() + else field_type + ) + relational_fields_formatted = [ f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" for field in ordered_relational_fields ] - external_schemas = set() - for field in relational_fields_formatted: - # There is an external schema - field_type = field.split(":")[1].split()[0] - if len(field_type.split(".")) >= 2: - external_schemas.add(field_type.split(".")[0]) - + # Non-external relational fields non_external_schema_fields = [ field for field in relational_fields_formatted @@ -668,6 +656,14 @@ def _get_related_field_type(field) -> str: repr_str += f" {colors.italic('Relational fields')}\n" repr_str += "".join(non_external_schema_fields) + # External relational fields + external_schemas = set() + for field in relational_fields_formatted: + field_type = field.split(":")[1].split()[0] + # External schemas have a prefix -> the split has at least two values + if len(field_type.split(".")) >= 2: + external_schemas.add(field_type.split(".")[0]) + if external_schemas: # We want Bionty to show up before other schemas external_schemas = ( From 619f516283cf322c65921f055c00dc1f823dd72d Mon Sep 17 00:00:00 2001 From: zethson Date: Fri, 19 Jul 2024 10:42:54 +0200 Subject: [PATCH 14/15] :art: Refactor Signed-off-by: zethson --- lnschema_core/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 74a7b2a6..836cb754 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -569,10 +569,10 @@ def _reorder_fields_by_class(fields_to_order: list[Field]) -> list[Field]: ] return non_base_class_fields + found_base_class_fields - # Header + # ---Header--- repr_str = f"{colors.green(cls.__name__)}\n" - # Basic fields + # ---Basic fields--- basic_fields = [ field for field in cls._meta.get_fields() @@ -594,7 +594,7 @@ def _reorder_fields_by_class(fields_to_order: list[Field]) -> list[Field]: ] ) - # Relational fields + # ---Relational fields--- relational_fields = (ManyToOneRel, ManyToManyRel, ManyToManyField, ForeignKey) class_specific_relational_fields = [ @@ -649,6 +649,7 @@ def _get_related_field_type(field) -> str: non_external_schema_fields = [ field for field in relational_fields_formatted + # Non-external schemas do not have a prefix -> the split has less than if len(field.split(":")[1].split(".")) < 2 ] From bc04b71d6bfd5135e0a0af3ca302cacf19031d3c Mon Sep 17 00:00:00 2001 From: zethson Date: Fri, 19 Jul 2024 11:18:41 +0200 Subject: [PATCH 15/15] :art: Refactoring Signed-off-by: zethson --- lnschema_core/models.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/lnschema_core/models.py b/lnschema_core/models.py index 836cb754..06ea83f2 100644 --- a/lnschema_core/models.py +++ b/lnschema_core/models.py @@ -624,6 +624,15 @@ def _reorder_fields_by_class(fields_to_order: list[Field]) -> list[Field]: class_specific_relational_fields + filtered_non_class_specific ) + non_external_schema_fields = [] + external_schema_fields = [] + for field in ordered_relational_fields: + field_name = repr(field).split(": ")[1][:-1] + if field_name.count(".") == 1 and "lnschema_core" not in field_name: + external_schema_fields.append(field) + else: + non_external_schema_fields.append(field) + def _get_related_field_type(field) -> str: field_type = ( field.related_model.__get_name_with_schema__() @@ -640,30 +649,25 @@ def _get_related_field_type(field) -> str: else field_type ) - relational_fields_formatted = [ + non_external_schema_fields_formatted = [ f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" - for field in ordered_relational_fields + for field in non_external_schema_fields ] - - # Non-external relational fields - non_external_schema_fields = [ - field - for field in relational_fields_formatted - # Non-external schemas do not have a prefix -> the split has less than - if len(field.split(":")[1].split(".")) < 2 + external_schema_fields_formatted = [ + f" .{field.name.replace('_links', '')}: {_get_related_field_type(field)}\n" + for field in external_schema_fields ] + # Non-external relational fields if non_external_schema_fields: repr_str += f" {colors.italic('Relational fields')}\n" - repr_str += "".join(non_external_schema_fields) + repr_str += "".join(non_external_schema_fields_formatted) # External relational fields external_schemas = set() - for field in relational_fields_formatted: + for field in external_schema_fields_formatted: field_type = field.split(":")[1].split()[0] - # External schemas have a prefix -> the split has at least two values - if len(field_type.split(".")) >= 2: - external_schemas.add(field_type.split(".")[0]) + external_schemas.add(field_type.split(".")[0]) if external_schemas: # We want Bionty to show up before other schemas @@ -675,7 +679,7 @@ def _get_related_field_type(field) -> str: for ext_schema in external_schemas: ext_schema_fields = [ field - for field in relational_fields_formatted + for field in external_schema_fields_formatted if ext_schema in field ]