From b73c2c91fc27e62ffd08262f5beceabd5ebcef2b Mon Sep 17 00:00:00 2001 From: root Date: Tue, 15 Jun 2021 08:16:49 +0000 Subject: [PATCH 01/55] added MsetReplacementEvent model --- datameta/models/db.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/datameta/models/db.py b/datameta/models/db.py index 935f9d18..732b53d7 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -214,17 +214,31 @@ class MetaDataSet(Base): uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) user_id = Column(Integer, ForeignKey('users.id'), nullable=False) submission_id = Column(Integer, ForeignKey('submissions.id'), nullable=True) - is_deprecated = Column(Boolean, default=False) - deprecated_label = Column(String, nullable=True) - replaced_by_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=True) + # Relationships user = relationship('User', back_populates='metadatasets') submission = relationship('Submission', back_populates='metadatasets') metadatumrecords = relationship('MetaDatumRecord', back_populates='metadataset') - replaces = relationship('MetaDataSet', backref=backref('replaced_by', remote_side=[id])) service_executions = relationship('ServiceExecution', back_populates = 'metadataset') +class MsetReplacementEvent(Base): + """ Stores information about an mset replacement event """ + __tablename__ = 'msetreplacements' + id = Column(Integer, primary_key=True) + uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) + user_id = Column(Integer, ForeignKey('users.id'), nullable=False) + datetime = Column(DateTime, nullable=False) + # former 'is_deprecated' label from MetaDataSet + reason = Column(String(140), nullable=False) + # the id of the new mset + metadataset_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=False) + + # Relationships + user = relationship('User', back_populates='metadatasets') + # replaced msets can then backref to the even and the replacing mset can be obtained via event.metadataset_id + replaced_msets = relationship('MetaDataSet', backref=backref('replaced_through', remote_side=[id])) + class ApplicationSetting(Base): __tablename__ = 'appsettings' id = Column(Integer, primary_key=True) From c61ae1d3394cb879a4f891f792291b760127c761 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 15 Jun 2021 08:34:53 +0000 Subject: [PATCH 02/55] fixed alembic complaint --- datameta/models/db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datameta/models/db.py b/datameta/models/db.py index 732b53d7..f86fd14e 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -90,6 +90,7 @@ class User(Base): apikeys = relationship('ApiKey', back_populates='user') services = relationship('Service', secondary=user_service_table, back_populates='users') service_executions = relationship('ServiceExecution', back_populates='user') + mset_replacements = relationship('MsetReplacementEvent', back_populates='user') class ApiKey(Base): @@ -235,9 +236,9 @@ class MsetReplacementEvent(Base): metadataset_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=False) # Relationships - user = relationship('User', back_populates='metadatasets') + user = relationship('User', back_populates='mset_replacements') # replaced msets can then backref to the even and the replacing mset can be obtained via event.metadataset_id - replaced_msets = relationship('MetaDataSet', backref=backref('replaced_through', remote_side=[id])) + replaced_msets = relationship('MetaDataSet', backref=backref('replaced_through', remote_side=[metadataset_id])) class ApplicationSetting(Base): __tablename__ = 'appsettings' From 723ec8bfa00ff5bd36b9b677f6d814720dcee078 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 15 Jun 2021 09:11:21 +0000 Subject: [PATCH 03/55] added alembic script for revision related to b73c2c91 --- .../alembic/versions/20210615_37ecf07b7838.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 datameta/alembic/versions/20210615_37ecf07b7838.py diff --git a/datameta/alembic/versions/20210615_37ecf07b7838.py b/datameta/alembic/versions/20210615_37ecf07b7838.py new file mode 100644 index 00000000..9a36f764 --- /dev/null +++ b/datameta/alembic/versions/20210615_37ecf07b7838.py @@ -0,0 +1,45 @@ +"""added msetreplacements table, moved update-related fields from mset + +Revision ID: 37ecf07b7838 +Revises: 7fdc829db18d +Create Date: 2021-06-15 09:09:20.305816 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '37ecf07b7838' +down_revision = '7fdc829db18d' +branch_labels = None +depends_on = None + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('msetreplacements', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('datetime', sa.DateTime(), nullable=False), + sa.Column('reason', sa.String(length=140), nullable=False), + sa.Column('metadataset_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_metadataset_id_metadatasets')), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), + sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) + ) + op.drop_constraint('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', type_='foreignkey') + op.drop_column('metadatasets', 'deprecated_label') + op.drop_column('metadatasets', 'is_deprecated') + op.drop_column('metadatasets', 'replaced_by_id') + # ### end Alembic commands ### + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('metadatasets', sa.Column('replaced_by_id', sa.INTEGER(), autoincrement=False, nullable=True)) + op.add_column('metadatasets', sa.Column('is_deprecated', sa.BOOLEAN(), autoincrement=False, nullable=True)) + op.add_column('metadatasets', sa.Column('deprecated_label', sa.VARCHAR(), autoincrement=False, nullable=True)) + op.create_foreign_key('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', 'metadatasets', ['replaced_by_id'], ['id']) + op.drop_table('msetreplacements') + # ### end Alembic commands ### From e53527b8fe007a48141d70d1d6aa0b474faf9247 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 15 Jun 2021 09:16:59 +0000 Subject: [PATCH 04/55] pleasing flake8 --- .../alembic/versions/20210615_37ecf07b7838.py | 22 ++++++++++--------- datameta/models/db.py | 9 ++++---- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/datameta/alembic/versions/20210615_37ecf07b7838.py b/datameta/alembic/versions/20210615_37ecf07b7838.py index 9a36f764..62c7c337 100644 --- a/datameta/alembic/versions/20210615_37ecf07b7838.py +++ b/datameta/alembic/versions/20210615_37ecf07b7838.py @@ -15,19 +15,20 @@ branch_labels = None depends_on = None + def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table('msetreplacements', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('datetime', sa.DateTime(), nullable=False), - sa.Column('reason', sa.String(length=140), nullable=False), - sa.Column('metadataset_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_metadataset_id_metadatasets')), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), - sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), - sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('datetime', sa.DateTime(), nullable=False), + sa.Column('reason', sa.String(length=140), nullable=False), + sa.Column('metadataset_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_metadataset_id_metadatasets')), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), + sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) ) op.drop_constraint('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', type_='foreignkey') op.drop_column('metadatasets', 'deprecated_label') @@ -35,6 +36,7 @@ def upgrade(): op.drop_column('metadatasets', 'replaced_by_id') # ### end Alembic commands ### + def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.add_column('metadatasets', sa.Column('replaced_by_id', sa.INTEGER(), autoincrement=False, nullable=True)) diff --git a/datameta/models/db.py b/datameta/models/db.py index f86fd14e..fa49a8c2 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -215,7 +215,7 @@ class MetaDataSet(Base): uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) user_id = Column(Integer, ForeignKey('users.id'), nullable=False) submission_id = Column(Integer, ForeignKey('submissions.id'), nullable=True) - + # Relationships user = relationship('User', back_populates='metadatasets') submission = relationship('Submission', back_populates='metadatasets') @@ -230,16 +230,17 @@ class MsetReplacementEvent(Base): uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) user_id = Column(Integer, ForeignKey('users.id'), nullable=False) datetime = Column(DateTime, nullable=False) - # former 'is_deprecated' label from MetaDataSet + # former 'is_deprecated' label from MetaDataSet reason = Column(String(140), nullable=False) - # the id of the new mset + # the id of the new mset metadataset_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=False) - + # Relationships user = relationship('User', back_populates='mset_replacements') # replaced msets can then backref to the even and the replacing mset can be obtained via event.metadataset_id replaced_msets = relationship('MetaDataSet', backref=backref('replaced_through', remote_side=[metadataset_id])) + class ApplicationSetting(Base): __tablename__ = 'appsettings' id = Column(Integer, primary_key=True) From 3dbc0f412fc130a1fd5db83de57dabad498d67db Mon Sep 17 00:00:00 2001 From: root Date: Thu, 24 Jun 2021 08:49:11 +0000 Subject: [PATCH 05/55] fixed MsetReplacement/MetaDataSet key issues (with @lkuchenb) --- ...ecf07b7838.py => 20210624_2735e05ff276.py} | 40 ++++++++++--------- datameta/models/db.py | 38 +++++++++--------- 2 files changed, 40 insertions(+), 38 deletions(-) rename datameta/alembic/versions/{20210615_37ecf07b7838.py => 20210624_2735e05ff276.py} (51%) diff --git a/datameta/alembic/versions/20210615_37ecf07b7838.py b/datameta/alembic/versions/20210624_2735e05ff276.py similarity index 51% rename from datameta/alembic/versions/20210615_37ecf07b7838.py rename to datameta/alembic/versions/20210624_2735e05ff276.py index 62c7c337..8a7163a8 100644 --- a/datameta/alembic/versions/20210615_37ecf07b7838.py +++ b/datameta/alembic/versions/20210624_2735e05ff276.py @@ -1,8 +1,8 @@ """added msetreplacements table, moved update-related fields from mset -Revision ID: 37ecf07b7838 +Revision ID: 2735e05ff276 Revises: 7fdc829db18d -Create Date: 2021-06-15 09:09:20.305816 +Create Date: 2021-06-24 08:35:38.370074 """ from alembic import op @@ -10,38 +10,40 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = '37ecf07b7838' +revision = '2735e05ff276' down_revision = '7fdc829db18d' branch_labels = None depends_on = None - def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table('msetreplacements', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('datetime', sa.DateTime(), nullable=False), - sa.Column('reason', sa.String(length=140), nullable=False), - sa.Column('metadataset_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_metadataset_id_metadatasets')), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), - sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), - sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('datetime', sa.DateTime(), nullable=False), + sa.Column('label', sa.String(length=140), nullable=False), + sa.Column('new_metadataset_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_new_metadataset_id_metadatasets')), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), + sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) ) + op.add_column('metadatasets', sa.Column('replaced_via_event_id', sa.Integer(), nullable=True)) op.drop_constraint('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', type_='foreignkey') - op.drop_column('metadatasets', 'deprecated_label') - op.drop_column('metadatasets', 'is_deprecated') + op.create_foreign_key(op.f('fk_metadatasets_replaced_via_event_id_msetreplacements'), 'metadatasets', 'msetreplacements', ['replaced_via_event_id'], ['id'], use_alter=True) op.drop_column('metadatasets', 'replaced_by_id') + op.drop_column('metadatasets', 'is_deprecated') + op.drop_column('metadatasets', 'deprecated_label') # ### end Alembic commands ### - def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column('metadatasets', sa.Column('replaced_by_id', sa.INTEGER(), autoincrement=False, nullable=True)) - op.add_column('metadatasets', sa.Column('is_deprecated', sa.BOOLEAN(), autoincrement=False, nullable=True)) op.add_column('metadatasets', sa.Column('deprecated_label', sa.VARCHAR(), autoincrement=False, nullable=True)) + op.add_column('metadatasets', sa.Column('is_deprecated', sa.BOOLEAN(), autoincrement=False, nullable=True)) + op.add_column('metadatasets', sa.Column('replaced_by_id', sa.INTEGER(), autoincrement=False, nullable=True)) + op.drop_constraint(op.f('fk_metadatasets_replaced_via_event_id_msetreplacements'), 'metadatasets', type_='foreignkey') op.create_foreign_key('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', 'metadatasets', ['replaced_by_id'], ['id']) + op.drop_column('metadatasets', 'replaced_via_event_id') op.drop_table('msetreplacements') # ### end Alembic commands ### diff --git a/datameta/models/db.py b/datameta/models/db.py index fa49a8c2..13336ae3 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -210,35 +210,35 @@ class MetaDatumRecord(Base): class MetaDataSet(Base): """A MetaDataSet represents all metadata associated with *one* record""" __tablename__ = 'metadatasets' - id = Column(Integer, primary_key=True) - site_id = Column(String(50), unique=True, nullable=False, index=True) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) - user_id = Column(Integer, ForeignKey('users.id'), nullable=False) - submission_id = Column(Integer, ForeignKey('submissions.id'), nullable=True) - + id = Column(Integer, primary_key=True) + site_id = Column(String(50), unique=True, nullable=False, index=True) + uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) + user_id = Column(Integer, ForeignKey('users.id'), nullable=False) + submission_id = Column(Integer, ForeignKey('submissions.id'), nullable=True) + replaced_via_event_id = Column(Integer, ForeignKey('msetreplacements.id', use_alter=True), nullable=True) + # Relationships user = relationship('User', back_populates='metadatasets') submission = relationship('Submission', back_populates='metadatasets') metadatumrecords = relationship('MetaDatumRecord', back_populates='metadataset') - service_executions = relationship('ServiceExecution', back_populates = 'metadataset') + service_executions = relationship('ServiceExecution', back_populates ='metadataset') + + replaced_via_event = relationship('MsetReplacementEvent', primaryjoin='MetaDataSet.replaced_via_event_id==MsetReplacementEvent.id') + replaces_via_event = relationship('MsetReplacementEvent', primaryjoin='MetaDataSet.id==MsetReplacementEvent.new_metadataset_id') class MsetReplacementEvent(Base): """ Stores information about an mset replacement event """ __tablename__ = 'msetreplacements' - id = Column(Integer, primary_key=True) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) - user_id = Column(Integer, ForeignKey('users.id'), nullable=False) - datetime = Column(DateTime, nullable=False) - # former 'is_deprecated' label from MetaDataSet - reason = Column(String(140), nullable=False) - # the id of the new mset - metadataset_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=False) - + id = Column(Integer, primary_key=True) + uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) + user_id = Column(Integer, ForeignKey('users.id'), nullable=False) + datetime = Column(DateTime, nullable=False) + label = Column(String(140), nullable=False) + new_metadataset_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=False) + # Relationships user = relationship('User', back_populates='mset_replacements') - # replaced msets can then backref to the even and the replacing mset can be obtained via event.metadataset_id - replaced_msets = relationship('MetaDataSet', backref=backref('replaced_through', remote_side=[metadataset_id])) class ApplicationSetting(Base): @@ -260,7 +260,7 @@ class Service(Base): site_id = Column(String(50), unique=True, nullable=False, index=True) name = Column(Text, nullable=True, unique=True) # Relationships - users = relationship('User', secondary=user_service_table, back_populates='services') + users = relationship('User', secondary=user_service_table, back_populates='services') # unfortunately, 'metadata' is a reserved keyword for sqlalchemy classes service_executions = relationship('ServiceExecution', back_populates = 'service') target_metadata = relationship('MetaDatum', back_populates = 'service') From 1b3a3fcd9c770cef3bf0d4c91ea5e9ae78184de3 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 24 Jun 2021 08:55:31 +0000 Subject: [PATCH 06/55] pleasing flake8 --- .../alembic/versions/20210624_2735e05ff276.py | 25 +++++++++++-------- datameta/models/db.py | 8 +++--- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/datameta/alembic/versions/20210624_2735e05ff276.py b/datameta/alembic/versions/20210624_2735e05ff276.py index 8a7163a8..8a480688 100644 --- a/datameta/alembic/versions/20210624_2735e05ff276.py +++ b/datameta/alembic/versions/20210624_2735e05ff276.py @@ -15,19 +15,21 @@ branch_labels = None depends_on = None + def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_table('msetreplacements', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('datetime', sa.DateTime(), nullable=False), - sa.Column('label', sa.String(length=140), nullable=False), - sa.Column('new_metadataset_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_new_metadataset_id_metadatasets')), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), - sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), - sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) + op.create_table( + 'msetreplacements', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('datetime', sa.DateTime(), nullable=False), + sa.Column('label', sa.String(length=140), nullable=False), + sa.Column('new_metadataset_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_new_metadataset_id_metadatasets')), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), + sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) ) op.add_column('metadatasets', sa.Column('replaced_via_event_id', sa.Integer(), nullable=True)) op.drop_constraint('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', type_='foreignkey') @@ -37,6 +39,7 @@ def upgrade(): op.drop_column('metadatasets', 'deprecated_label') # ### end Alembic commands ### + def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.add_column('metadatasets', sa.Column('deprecated_label', sa.VARCHAR(), autoincrement=False, nullable=True)) diff --git a/datameta/models/db.py b/datameta/models/db.py index 13336ae3..0158ddbc 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -28,7 +28,7 @@ Table ) from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import relationship from .meta import Base @@ -215,8 +215,8 @@ class MetaDataSet(Base): uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) user_id = Column(Integer, ForeignKey('users.id'), nullable=False) submission_id = Column(Integer, ForeignKey('submissions.id'), nullable=True) - replaced_via_event_id = Column(Integer, ForeignKey('msetreplacements.id', use_alter=True), nullable=True) - + replaced_via_event_id = Column(Integer, ForeignKey('msetreplacements.id', use_alter=True), nullable=True) + # Relationships user = relationship('User', back_populates='metadatasets') submission = relationship('Submission', back_populates='metadatasets') @@ -236,7 +236,7 @@ class MsetReplacementEvent(Base): datetime = Column(DateTime, nullable=False) label = Column(String(140), nullable=False) new_metadataset_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=False) - + # Relationships user = relationship('User', back_populates='mset_replacements') From 729ddcc4917e83e300c49015e012cf75b68355f9 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 24 Jun 2021 14:34:52 +0000 Subject: [PATCH 07/55] added initial mset replacement logic --- datameta/api/metadatasets.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index e131a8a1..c6a4d3da 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -13,7 +13,8 @@ # limitations under the License. from dataclasses import dataclass -from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound, HTTPNoContent +from datameta.models.db import MsetReplacementEvent +from pyramid.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPNotFound, HTTPNoContent from pyramid.view import view_config from pyramid.request import Request from sqlalchemy.orm import joinedload @@ -24,7 +25,7 @@ from ..models import MetaDatum, MetaDataSet, ServiceExecution, Service, MetaDatumRecord, Submission, File from ..security import authz import datetime -from datetime import timezone +from datetime import timezone, datetime from ..resource import resource_by_id, resource_query_by_id, get_identifier from ..utils import get_record_from_metadataset from . import DataHolderBase @@ -50,6 +51,7 @@ class MetaDataSetResponse(DataHolderBase): submission_id : Optional[str] = None service_executions : Optional[Dict[str, Optional[MetaDataSetServiceExecution]]] = None + @staticmethod def from_metadataset(metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): """Creates a MetaDataSetResponse from a metadataset database object and @@ -150,6 +152,12 @@ def post(request: Request) -> MetaDataSetResponse: # Obtain string converted version of the record record = record_to_strings(request.openapi_validated.body["record"]) + replaces = request.openapi_validated.body.get("replaces") + replaces_label = request.openapi_validated.body.get("replaces_label") + + if replaces and not replaces_label: + return HTTPBadRequest() + # Query the configured metadata. We're only considering and allowing # non-service metadata when creating a new metadataset. metadata = get_all_metadata(db, include_service_metadata=False) @@ -167,6 +175,24 @@ def post(request: Request) -> MetaDataSetResponse: submission_id = None ) db.add(mdata_set) + + if replaces: + mset_repl_evt = MsetReplacementEvent( + user_id = auth_user.id, + datetime = datetime.utcnow(), + label = replaces_label, + new_metadataset_id = mdata_set.id + ) + db.add(mset_repl_evt) + + for mset_id in replaces: + + target_mset = resource_by_id(db, MetaDataSet, mset_id) + if target_mset is None: + raise HTTPForbidden() + + target_mset.replaced_via_event_id = mset_repl_evt.id + db.flush() # Add NULL values for service metadata From 2bd9b75fc2496906e59025880d1ff6bc181de398 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 24 Jun 2021 14:39:34 +0000 Subject: [PATCH 08/55] pleasing flake8 and mypy --- datameta/api/metadatasets.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index c6a4d3da..6f4f5cd4 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -24,7 +24,6 @@ from .. import security, siteid, resource, validation from ..models import MetaDatum, MetaDataSet, ServiceExecution, Service, MetaDatumRecord, Submission, File from ..security import authz -import datetime from datetime import timezone, datetime from ..resource import resource_by_id, resource_query_by_id, get_identifier from ..utils import get_record_from_metadataset @@ -51,7 +50,6 @@ class MetaDataSetResponse(DataHolderBase): submission_id : Optional[str] = None service_executions : Optional[Dict[str, Optional[MetaDataSetServiceExecution]]] = None - @staticmethod def from_metadataset(metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): """Creates a MetaDataSetResponse from a metadataset database object and @@ -90,7 +88,7 @@ def render_record_values(metadata: Dict[str, MetaDatum], record: dict) -> dict: continue elif record_rendered[field] and metadata[field].datetimefmt: # if MetaDatum is a datetime field, render the value in isoformat - record_rendered[field] = datetime.datetime.strptime( + record_rendered[field] = datetime.strptime( record_rendered[field], metadata[field].datetimefmt ).isoformat() @@ -470,7 +468,7 @@ def set_metadata_via_service(request: Request) -> MetaDataSetResponse: service = service, metadataset = metadataset, user = auth_user, - datetime = datetime.datetime.utcnow() + datetime = datetime.utcnow() ) db.add(sexec) From 523208fa59c20f8946c860c75b28e43eef92dbee Mon Sep 17 00:00:00 2001 From: root Date: Fri, 25 Jun 2021 15:03:18 +0000 Subject: [PATCH 09/55] addressing @lkuchenb's code review --- datameta/api/metadatasets.py | 20 ++++++++++++++------ datameta/api/openapi.yaml | 10 ++++++++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 6f4f5cd4..3ca7884e 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -154,7 +154,9 @@ def post(request: Request) -> MetaDataSetResponse: replaces_label = request.openapi_validated.body.get("replaces_label") if replaces and not replaces_label: - return HTTPBadRequest() + raise errors.get_validation_error(messages=['No reason (label) given for Metadataset replacement.']) # maybe label should be reason. + if not replaces and replaces_label: + raise errors.get_validation_error(messages=["No metadataset ids specified (replacement reason (label) is given."]) # Query the configured metadata. We're only considering and allowing # non-service metadata when creating a new metadataset. @@ -172,6 +174,7 @@ def post(request: Request) -> MetaDataSetResponse: user_id = auth_user.id, submission_id = None ) + db.add(mdata_set) if replaces: @@ -183,12 +186,17 @@ def post(request: Request) -> MetaDataSetResponse: ) db.add(mset_repl_evt) - for mset_id in replaces: + msets = [ + (mset_id, resource_by_id(db, MetaDataSet, mset_id)) + for mset_id in replaces + ] - target_mset = resource_by_id(db, MetaDataSet, mset_id) - if target_mset is None: - raise HTTPForbidden() + missing_msets = [("Invalid metadataset id.", mset_id) for mset_id, target_mset in msets if target_mset is None] + if missing_msets: + messages, entities = zip(*missing_msets) + raise errors.get_validation_error(messages=messages, entities=entities) + for target_mset in msets: target_mset.replaced_via_event_id = mset_repl_evt.id db.flush() @@ -303,7 +311,7 @@ def get_metadatasets(request: Request) -> List[MetaDataSetResponse]: query = query.filter(Submission.date < submitted_before.astimezone(timezone.utc)) if awaiting_service is not None: if awaiting_service not in readable_services_by_id: - raise errors.get_validation_error(messages=['Invalid service ID specified'], fields=['awaitingServices']) + raise errors.get_validation_error(messages=['Invalid service ID specified'], fields=['awaitingService']) query = query.outerjoin(ServiceExecution, and_( MetaDataSet.id == ServiceExecution.metadataset_id, ServiceExecution.service_id == readable_services_by_id[awaiting_service].id diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index 253f9ed6..9d2d00d4 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -15,7 +15,7 @@ openapi: 3.0.0 info: description: DataMeta - version: 1.0.1 + version: 1.0.2 title: DataMeta servers: @@ -528,7 +528,7 @@ paths: format: date-time - name: awaitingService in: query - description: Identifier for a service. Restricts the result to metadatsets for which the specified service has not been executed yet. + description: Identifier for a service. Restricts the result to metadatasets for which the specified service has not been executed yet. schema: type: string responses: @@ -1371,6 +1371,12 @@ components: additionalProperties: true # a free-form object, # any property is allowed + replaces: + type: array + items: + type: string + replaces_label: + type: string required: - record additionalProperties: false From 9068be36bee5874bc200c05cc5ba63aff0830b08 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 25 Jun 2021 15:30:00 +0000 Subject: [PATCH 10/55] added user.can_update flag --- .../alembic/versions/20210624_2735e05ff276.py | 52 ------------------- datameta/api/openapi.yaml | 5 ++ datameta/api/users.py | 10 ++++ datameta/models/db.py | 3 ++ datameta/security/authz.py | 4 ++ tests/integration/fixtures/users.yaml | 8 +++ 6 files changed, 30 insertions(+), 52 deletions(-) delete mode 100644 datameta/alembic/versions/20210624_2735e05ff276.py diff --git a/datameta/alembic/versions/20210624_2735e05ff276.py b/datameta/alembic/versions/20210624_2735e05ff276.py deleted file mode 100644 index 8a480688..00000000 --- a/datameta/alembic/versions/20210624_2735e05ff276.py +++ /dev/null @@ -1,52 +0,0 @@ -"""added msetreplacements table, moved update-related fields from mset - -Revision ID: 2735e05ff276 -Revises: 7fdc829db18d -Create Date: 2021-06-24 08:35:38.370074 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = '2735e05ff276' -down_revision = '7fdc829db18d' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - 'msetreplacements', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('datetime', sa.DateTime(), nullable=False), - sa.Column('label', sa.String(length=140), nullable=False), - sa.Column('new_metadataset_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_new_metadataset_id_metadatasets')), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), - sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), - sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) - ) - op.add_column('metadatasets', sa.Column('replaced_via_event_id', sa.Integer(), nullable=True)) - op.drop_constraint('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', type_='foreignkey') - op.create_foreign_key(op.f('fk_metadatasets_replaced_via_event_id_msetreplacements'), 'metadatasets', 'msetreplacements', ['replaced_via_event_id'], ['id'], use_alter=True) - op.drop_column('metadatasets', 'replaced_by_id') - op.drop_column('metadatasets', 'is_deprecated') - op.drop_column('metadatasets', 'deprecated_label') - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('metadatasets', sa.Column('deprecated_label', sa.VARCHAR(), autoincrement=False, nullable=True)) - op.add_column('metadatasets', sa.Column('is_deprecated', sa.BOOLEAN(), autoincrement=False, nullable=True)) - op.add_column('metadatasets', sa.Column('replaced_by_id', sa.INTEGER(), autoincrement=False, nullable=True)) - op.drop_constraint(op.f('fk_metadatasets_replaced_via_event_id_msetreplacements'), 'metadatasets', type_='foreignkey') - op.create_foreign_key('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', 'metadatasets', ['replaced_by_id'], ['id']) - op.drop_column('metadatasets', 'replaced_via_event_id') - op.drop_table('msetreplacements') - # ### end Alembic commands ### diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index 9d2d00d4..8c0f2dc7 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -1654,6 +1654,8 @@ components: type: boolean siteRead: type: boolean + canUpdate: + type: boolean email: type: string group: @@ -1669,6 +1671,7 @@ components: - groupAdmin - siteAdmin - siteRead + - canUpdate - email - group additionalProperties: false @@ -1688,6 +1691,8 @@ components: type: boolean enabled: type: boolean + canUpdate: + type: boolean MetaDataResponse: type: object diff --git a/datameta/api/users.py b/datameta/api/users.py index 93cbbd89..e080c98d 100644 --- a/datameta/api/users.py +++ b/datameta/api/users.py @@ -33,6 +33,7 @@ class UserUpdateRequest(DataHolderBase): siteAdmin: bool enabled: bool siteRead: bool + canUpdate: bool @dataclass @@ -43,6 +44,7 @@ class UserResponseElement(DataHolderBase): group_admin: bool site_admin: bool site_read: bool + can_update: bool email: str group: dict @@ -63,6 +65,7 @@ def get_whoami(request: Request) -> UserResponseElement: group_admin = auth_user.group_admin, site_admin = auth_user.site_admin, site_read = auth_user.site_read, + can_update = auth_user.can_update, email = auth_user.email, group = {"id": get_identifier(auth_user.group), "name": auth_user.group.name} ) @@ -84,6 +87,7 @@ def put(request: Request): site_admin = request.openapi_validated.body.get("siteAdmin") enabled = request.openapi_validated.body.get("enabled") site_read = request.openapi_validated.body.get("siteRead") + can_update = request.openapi_validated.body.get("canUpdate") db = request.dbsession @@ -122,6 +126,10 @@ def put(request: Request): if name is not None and not authz.update_user_name(auth_user, target_user): raise HTTPForbidden() + # The user has to be site admin to change update privileges for a user + if can_update is not None and not authz.update_user_can_update(auth_user): + raise HTTPForbidden() + # Now, make the corresponding changes if group_id is not None: new_group = resource_by_id(db, Group, group_id) @@ -138,5 +146,7 @@ def put(request: Request): target_user.enabled = enabled if name is not None: target_user.fullname = name + if can_update is not None: + target_user.can_update = can_update return HTTPNoContent() diff --git a/datameta/models/db.py b/datameta/models/db.py index 0158ddbc..37d61b3b 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -29,6 +29,7 @@ ) from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship +from sqlalchemy.sql.expression import null from .meta import Base @@ -82,6 +83,8 @@ class User(Base): site_admin = Column(Boolean(create_constraint=False), nullable=False) group_admin = Column(Boolean(create_constraint=False), nullable=False) site_read = Column(Boolean(create_constraint=False), nullable=False) + can_update = Column(Boolean(create_constraint=False), nullable=False) + # Relationships group = relationship('Group', back_populates='user') metadatasets = relationship('MetaDataSet', back_populates='user') diff --git a/datameta/security/authz.py b/datameta/security/authz.py index b41f9f74..8a9b7533 100644 --- a/datameta/security/authz.py +++ b/datameta/security/authz.py @@ -150,6 +150,10 @@ def update_user_name(user, target_user): )) +def update_user_can_update(user): + return user.site_admin + + def create_service(user): return user.site_admin diff --git a/tests/integration/fixtures/users.yaml b/tests/integration/fixtures/users.yaml index 51e89508..a41f63a3 100644 --- a/tests/integration/fixtures/users.yaml +++ b/tests/integration/fixtures/users.yaml @@ -25,6 +25,7 @@ admin: group_admin: true site_admin: true site_read: true + can_update: true enabled: true references: group: @@ -44,6 +45,7 @@ group_x_admin: group_admin: true site_admin: false site_read: false + can_update: true enabled: true references: group: @@ -63,6 +65,7 @@ user_a: group_admin: false site_admin: false site_read: false + can_update: false enabled: true references: group: @@ -82,6 +85,7 @@ user_b: group_admin: false site_admin: false site_read: false + can_update: false enabled: true references: group: @@ -101,6 +105,7 @@ group_y_admin: group_admin: true site_admin: false site_read: false + can_update: true enabled: true references: group: @@ -120,6 +125,7 @@ user_c: group_admin: false site_admin: false site_read: false + can_update: false enabled: true references: group: @@ -139,6 +145,7 @@ user_site_read: group_admin: false site_admin: false site_read: true + can_update: false enabled: true references: group: @@ -158,6 +165,7 @@ service_user_0: group_admin: false site_admin: false site_read: true + can_update: false enabled: true references: group: From 2392560302e74d1b42fd3f775329226389b9f601 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 25 Jun 2021 15:31:06 +0000 Subject: [PATCH 11/55] generated alembic script pertaining to 9068be3 --- .../alembic/versions/20210625_f6a70402119f.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 datameta/alembic/versions/20210625_f6a70402119f.py diff --git a/datameta/alembic/versions/20210625_f6a70402119f.py b/datameta/alembic/versions/20210625_f6a70402119f.py new file mode 100644 index 00000000..dff6911f --- /dev/null +++ b/datameta/alembic/versions/20210625_f6a70402119f.py @@ -0,0 +1,51 @@ +"""added msetreplacements table, moved update-related fields from mset, added user.can_update flag + +Revision ID: f6a70402119f +Revises: 7fdc829db18d +Create Date: 2021-06-25 15:27:37.638601 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'f6a70402119f' +down_revision = '7fdc829db18d' +branch_labels = None +depends_on = None + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('msetreplacements', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('datetime', sa.DateTime(), nullable=False), + sa.Column('label', sa.String(length=140), nullable=False), + sa.Column('new_metadataset_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_new_metadataset_id_metadatasets')), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), + sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) + ) + op.add_column('metadatasets', sa.Column('replaced_via_event_id', sa.Integer(), nullable=True)) + op.drop_constraint('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', type_='foreignkey') + op.create_foreign_key(op.f('fk_metadatasets_replaced_via_event_id_msetreplacements'), 'metadatasets', 'msetreplacements', ['replaced_via_event_id'], ['id'], use_alter=True) + op.drop_column('metadatasets', 'deprecated_label') + op.drop_column('metadatasets', 'replaced_by_id') + op.drop_column('metadatasets', 'is_deprecated') + op.add_column('users', sa.Column('can_update', sa.Boolean(create_constraint=False), nullable=False)) + # ### end Alembic commands ### + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('users', 'can_update') + op.add_column('metadatasets', sa.Column('is_deprecated', sa.BOOLEAN(), autoincrement=False, nullable=True)) + op.add_column('metadatasets', sa.Column('replaced_by_id', sa.INTEGER(), autoincrement=False, nullable=True)) + op.add_column('metadatasets', sa.Column('deprecated_label', sa.VARCHAR(), autoincrement=False, nullable=True)) + op.drop_constraint(op.f('fk_metadatasets_replaced_via_event_id_msetreplacements'), 'metadatasets', type_='foreignkey') + op.create_foreign_key('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', 'metadatasets', ['replaced_by_id'], ['id']) + op.drop_column('metadatasets', 'replaced_via_event_id') + op.drop_table('msetreplacements') + # ### end Alembic commands ### From 54d3b639f48c85bf5a0da23626ad19fa764a7676 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 25 Jun 2021 16:22:25 +0000 Subject: [PATCH 12/55] pleasing flake8 --- .../alembic/versions/20210625_f6a70402119f.py | 24 ++++++++++--------- datameta/api/metadatasets.py | 4 ++-- datameta/models/db.py | 1 - 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/datameta/alembic/versions/20210625_f6a70402119f.py b/datameta/alembic/versions/20210625_f6a70402119f.py index dff6911f..7ff0b047 100644 --- a/datameta/alembic/versions/20210625_f6a70402119f.py +++ b/datameta/alembic/versions/20210625_f6a70402119f.py @@ -17,17 +17,18 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_table('msetreplacements', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('datetime', sa.DateTime(), nullable=False), - sa.Column('label', sa.String(length=140), nullable=False), - sa.Column('new_metadataset_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_new_metadataset_id_metadatasets')), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), - sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), - sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) + op.create_table( + 'msetreplacements', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('datetime', sa.DateTime(), nullable=False), + sa.Column('label', sa.String(length=140), nullable=False), + sa.Column('new_metadataset_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_new_metadataset_id_metadatasets')), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), + sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) ) op.add_column('metadatasets', sa.Column('replaced_via_event_id', sa.Integer(), nullable=True)) op.drop_constraint('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', type_='foreignkey') @@ -38,6 +39,7 @@ def upgrade(): op.add_column('users', sa.Column('can_update', sa.Boolean(create_constraint=False), nullable=False)) # ### end Alembic commands ### + def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.drop_column('users', 'can_update') diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 3ca7884e..c442cc8e 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from datameta.models.db import MsetReplacementEvent -from pyramid.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPNotFound, HTTPNoContent +from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound, HTTPNoContent from pyramid.view import view_config from pyramid.request import Request from sqlalchemy.orm import joinedload @@ -154,7 +154,7 @@ def post(request: Request) -> MetaDataSetResponse: replaces_label = request.openapi_validated.body.get("replaces_label") if replaces and not replaces_label: - raise errors.get_validation_error(messages=['No reason (label) given for Metadataset replacement.']) # maybe label should be reason. + raise errors.get_validation_error(messages=['No reason (label) given for Metadataset replacement.']) # maybe label should be reason. if not replaces and replaces_label: raise errors.get_validation_error(messages=["No metadataset ids specified (replacement reason (label) is given."]) diff --git a/datameta/models/db.py b/datameta/models/db.py index 37d61b3b..f2691f8f 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -29,7 +29,6 @@ ) from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship -from sqlalchemy.sql.expression import null from .meta import Base From 1c9a66890f9c6afd4e9373ba19c11cfddc66e10a Mon Sep 17 00:00:00 2001 From: root Date: Fri, 25 Jun 2021 16:33:38 +0000 Subject: [PATCH 13/55] pleasing flake8 --- datameta/alembic/versions/20210625_f6a70402119f.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datameta/alembic/versions/20210625_f6a70402119f.py b/datameta/alembic/versions/20210625_f6a70402119f.py index 7ff0b047..41f348d2 100644 --- a/datameta/alembic/versions/20210625_f6a70402119f.py +++ b/datameta/alembic/versions/20210625_f6a70402119f.py @@ -15,6 +15,7 @@ branch_labels = None depends_on = None + def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table( From bf20b419df749617e3d312db891eac2f1383f006 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 29 Jun 2021 12:54:09 +0000 Subject: [PATCH 14/55] fixing the fixtures --- datameta/api/users.py | 4 +++- datameta/security/authz.py | 2 +- tests/integration/fixtures/holders.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/datameta/api/users.py b/datameta/api/users.py index 4b2d413b..e9b0a732 100644 --- a/datameta/api/users.py +++ b/datameta/api/users.py @@ -49,6 +49,7 @@ class UserResponseElement(DataHolderBase): site_admin: Optional[bool] = None site_read: Optional[bool] = None email: Optional[str] = None + can_update: Optional[bool] = None @classmethod def from_user(cls, target_user, requesting_user): @@ -59,7 +60,8 @@ def from_user(cls, target_user, requesting_user): "group_admin": target_user.group_admin, "site_admin": target_user.site_admin, "site_read": target_user.site_read, - "email": target_user.email + "email": target_user.email, + "can_update": target_user.can_update }) return cls(id=get_identifier(target_user), name=target_user.fullname, group=get_identifier(target_user.group), **restricted_fields) diff --git a/datameta/security/authz.py b/datameta/security/authz.py index 01a30eed..d64fecaa 100644 --- a/datameta/security/authz.py +++ b/datameta/security/authz.py @@ -183,7 +183,7 @@ def update_user_can_update(user): def view_restricted_user_info(user, target_user): return has_group_rights(user, target_user.group) - + def create_service(user): return user.site_admin diff --git a/tests/integration/fixtures/holders.py b/tests/integration/fixtures/holders.py index e1464711..9e8a3a30 100644 --- a/tests/integration/fixtures/holders.py +++ b/tests/integration/fixtures/holders.py @@ -42,6 +42,7 @@ class UserFixture(Entity): site_admin : bool site_read : bool enabled : bool + can_update : bool @dataclass From 67fe823a0541feb1780c3ff208dc7235491a416c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 29 Jun 2021 13:28:39 +0000 Subject: [PATCH 15/55] fixing the fixtures --- datameta/api/openapi.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index d17a80b2..948a7394 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -1784,6 +1784,9 @@ components: $ref: "#/components/schemas/Identifier" name: type: string + canUpdate: + type: boolean + nullable: true required: - id - name From 9accbedc0cc7d521f065c3f5ba0a63c27a327a11 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 29 Jun 2021 13:36:22 +0000 Subject: [PATCH 16/55] addressing mypy error --- datameta/api/metadatasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index c442cc8e..74a0a415 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -196,7 +196,7 @@ def post(request: Request) -> MetaDataSetResponse: messages, entities = zip(*missing_msets) raise errors.get_validation_error(messages=messages, entities=entities) - for target_mset in msets: + for _, target_mset in msets: target_mset.replaced_via_event_id = mset_repl_evt.id db.flush() From 5db3d832f2059a4e610eb09e0d7752bce2176fd8 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Jun 2021 13:12:21 +0000 Subject: [PATCH 17/55] implemented code review requests (@lkuchenb) --- datameta/api/metadatasets.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 74a0a415..82e4994c 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -156,7 +156,7 @@ def post(request: Request) -> MetaDataSetResponse: if replaces and not replaces_label: raise errors.get_validation_error(messages=['No reason (label) given for Metadataset replacement.']) # maybe label should be reason. if not replaces and replaces_label: - raise errors.get_validation_error(messages=["No metadataset ids specified (replacement reason (label) is given."]) + raise errors.get_validation_error(messages=["No metadataset IDs specified but replacement reason (label) is given."]) # Query the configured metadata. We're only considering and allowing # non-service metadata when creating a new metadataset. @@ -192,10 +192,21 @@ def post(request: Request) -> MetaDataSetResponse: ] missing_msets = [("Invalid metadataset id.", mset_id) for mset_id, target_mset in msets if target_mset is None] + if missing_msets: messages, entities = zip(*missing_msets) raise errors.get_validation_error(messages=messages, entities=entities) + already_replaced = [ + (f"MetaDataSet was already replaced via event {target_mset.replaced_via_event_id}", mset_id) + for mset_id, target_mset in msets + if target_mset.replaced_via_event_id is not None + ] + + if already_replaced: + messages, entities = zip(*already_replaced) + raise errors.get_validation_error(messages=messages, entities=entities) + for _, target_mset in msets: target_mset.replaced_via_event_id = mset_repl_evt.id From b92e67ed6f70c385750e96b7e625b3f2fbe72f2c Mon Sep 17 00:00:00 2001 From: root Date: Wed, 30 Jun 2021 13:43:17 +0000 Subject: [PATCH 18/55] patched exposed event id --- datameta/api/metadatasets.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 82e4994c..fb9d9f66 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -197,11 +197,21 @@ def post(request: Request) -> MetaDataSetResponse: messages, entities = zip(*missing_msets) raise errors.get_validation_error(messages=messages, entities=entities) - already_replaced = [ - (f"MetaDataSet was already replaced via event {target_mset.replaced_via_event_id}", mset_id) - for mset_id, target_mset in msets - if target_mset.replaced_via_event_id is not None - ] + already_replaced = list() + for mset_id, target_mset in msets: + if target_mset.replaced_via_event_id is not None: + + replacement_event = resource_by_id(db, MsetReplacementEvent, target_mset.replaced_via_event_id) + if replacement_event is None: + message = "Metadataset is flagged as having been replaced, but cannot find replacement." + else: + replacing_mset = resource_by_id(db, MetaDataSet, replacement_event.new_metadataset_id) + if replacing_mset is None: + message = "Metadataset is flagged as having been replaced, but cannot find replacement." + else: + message = f"Metadataset was already replaced by {get_identifier(replacement_event)}." + + already_replaced.append((message, mset_id)) if already_replaced: messages, entities = zip(*already_replaced) From 307353f136e729987735631666db5fd0da9c6271 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Jul 2021 08:41:23 +0000 Subject: [PATCH 19/55] actually fixed exposed event id --- datameta/api/metadatasets.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index fb9d9f66..23f74267 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -201,15 +201,15 @@ def post(request: Request) -> MetaDataSetResponse: for mset_id, target_mset in msets: if target_mset.replaced_via_event_id is not None: - replacement_event = resource_by_id(db, MsetReplacementEvent, target_mset.replaced_via_event_id) - if replacement_event is None: + event_unknown = target_mset.replaced_via_event is None + + if any(( + event_unknown, + not event_unknown and target_mset.replaced_via_event.new_metadataset_id is None + )): message = "Metadataset is flagged as having been replaced, but cannot find replacement." else: - replacing_mset = resource_by_id(db, MetaDataSet, replacement_event.new_metadataset_id) - if replacing_mset is None: - message = "Metadataset is flagged as having been replaced, but cannot find replacement." - else: - message = f"Metadataset was already replaced by {get_identifier(replacement_event)}." + message = f"Metadataset was already replaced by {target_mset.replaced_via_event.new_metadataset_id}." already_replaced.append((message, mset_id)) From f5d2483f83370a04502387903b57062c76944ba7 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Jul 2021 08:44:01 +0000 Subject: [PATCH 20/55] added can_update to restricted fields, updated info request integration test, restricted_fields now accessed via staticmethod --- datameta/api/users.py | 11 ++++++----- tests/integration/test_information_requests.py | 6 +++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/datameta/api/users.py b/datameta/api/users.py index e9b0a732..30ccbada 100644 --- a/datameta/api/users.py +++ b/datameta/api/users.py @@ -51,17 +51,18 @@ class UserResponseElement(DataHolderBase): email: Optional[str] = None can_update: Optional[bool] = None + @staticmethod + def get_restricted_fields(): + return ["group_admin", "site_admin", "site_read", "email", "can_update"] + @classmethod def from_user(cls, target_user, requesting_user): restricted_fields = dict() if authz.view_restricted_user_info(requesting_user, target_user): restricted_fields.update({ - "group_admin": target_user.group_admin, - "site_admin": target_user.site_admin, - "site_read": target_user.site_read, - "email": target_user.email, - "can_update": target_user.can_update + field: getattr(target_user, field) + for field in UserResponseElement.get_restricted_fields() }) return cls(id=get_identifier(target_user), name=target_user.fullname, group=get_identifier(target_user.group), **restricted_fields) diff --git a/tests/integration/test_information_requests.py b/tests/integration/test_information_requests.py index ebcc9f74..238a918d 100644 --- a/tests/integration/test_information_requests.py +++ b/tests/integration/test_information_requests.py @@ -18,6 +18,7 @@ from . import BaseIntegrationTest from datameta.api import base_url +from datameta.api.users import UserResponseElement class TestUserInformationRequest(BaseIntegrationTest): @@ -46,6 +47,9 @@ def do_request(self, user_uuid, **params): ("site_admin_query_wrong_id" , "admin" , "flopsy" , 404), ]) def test_user_query(self, testname: str, executing_user: str, target_user: str, expected_response: int): + def snake_to_camel(s): + return ''.join(x.capitalize() if i else x for i, x in enumerate(s.split('_'))) + user = self.fixture_manager.get_fixture('users', executing_user) if testname == "site_admin_query_wrong_id": @@ -63,7 +67,7 @@ def test_user_query(self, testname: str, executing_user: str, target_user: str, successful_request = expected_response == 200 privileged_request = "admin" in executing_user - restricted_fields = ["groupAdmin", "siteAdmin", "siteRead", "email"] + restricted_fields = map(snake_to_camel, UserResponseElement.get_restricted_fields()) assert any(( not successful_request, From 7a4c6de0c0ae05dd5edcdc2e58a6dd0a6e196aa8 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Jul 2021 10:04:09 +0000 Subject: [PATCH 21/55] added metadataset->msetreplacementevent relationships --- datameta/models/db.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datameta/models/db.py b/datameta/models/db.py index f2691f8f..b0db7c3c 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -240,7 +240,9 @@ class MsetReplacementEvent(Base): new_metadataset_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=False) # Relationships - user = relationship('User', back_populates='mset_replacements') + user = relationship('User', back_populates='mset_replacements') + new_metadataset = relationship("MetaDataSet", primaryjoin='MetaDataSet.id==MsetReplacementEvent.new_metadataset_id') + replaced_metadatasets = relationship("MetaDataSet", primaryjoin='MetaDataSet.replaced_via_event_id==MsetReplacementEvent.id') class ApplicationSetting(Base): From effc7d76531dbe1bc786d22138e3ed70547d27b9 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Jul 2021 10:04:33 +0000 Subject: [PATCH 22/55] simplified check for already replaced metadatasets --- datameta/api/metadatasets.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 23f74267..dfef3e65 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -197,21 +197,11 @@ def post(request: Request) -> MetaDataSetResponse: messages, entities = zip(*missing_msets) raise errors.get_validation_error(messages=messages, entities=entities) - already_replaced = list() - for mset_id, target_mset in msets: - if target_mset.replaced_via_event_id is not None: - - event_unknown = target_mset.replaced_via_event is None - - if any(( - event_unknown, - not event_unknown and target_mset.replaced_via_event.new_metadataset_id is None - )): - message = "Metadataset is flagged as having been replaced, but cannot find replacement." - else: - message = f"Metadataset was already replaced by {target_mset.replaced_via_event.new_metadataset_id}." - - already_replaced.append((message, mset_id)) + already_replaced = [ + (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset)}", mset_id) + for mset_id, target_mset in msets + if target_mset.replaced_via_event_id is not None + ] if already_replaced: messages, entities = zip(*already_replaced) From db6efbcd1566b76b39014c24bf5774a8bac197c4 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Jul 2021 10:34:43 +0000 Subject: [PATCH 23/55] added back_populates args --- datameta/models/db.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datameta/models/db.py b/datameta/models/db.py index b0db7c3c..91373f0e 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -225,8 +225,8 @@ class MetaDataSet(Base): metadatumrecords = relationship('MetaDatumRecord', back_populates='metadataset') service_executions = relationship('ServiceExecution', back_populates ='metadataset') - replaced_via_event = relationship('MsetReplacementEvent', primaryjoin='MetaDataSet.replaced_via_event_id==MsetReplacementEvent.id') - replaces_via_event = relationship('MsetReplacementEvent', primaryjoin='MetaDataSet.id==MsetReplacementEvent.new_metadataset_id') + replaced_via_event = relationship('MsetReplacementEvent', primaryjoin='MetaDataSet.replaced_via_event_id==MsetReplacementEvent.id', back_populates='new_metadataset') + replaces_via_event = relationship('MsetReplacementEvent', primaryjoin='MetaDataSet.id==MsetReplacementEvent.new_metadataset_id', back_populates='replaced_metadatasets') class MsetReplacementEvent(Base): @@ -241,8 +241,8 @@ class MsetReplacementEvent(Base): # Relationships user = relationship('User', back_populates='mset_replacements') - new_metadataset = relationship("MetaDataSet", primaryjoin='MetaDataSet.id==MsetReplacementEvent.new_metadataset_id') - replaced_metadatasets = relationship("MetaDataSet", primaryjoin='MetaDataSet.replaced_via_event_id==MsetReplacementEvent.id') + new_metadataset = relationship("MetaDataSet", primaryjoin='MetaDataSet.id==MsetReplacementEvent.new_metadataset_id', back_populates='replaces_via_event') + replaced_metadatasets = relationship("MetaDataSet", primaryjoin='MetaDataSet.replaced_via_event_id==MsetReplacementEvent.id', back_populates='replaced_via_event') class ApplicationSetting(Base): From 6c7199bfa753ec89d5154de47324aea71de59cba Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Jul 2021 16:37:51 +0000 Subject: [PATCH 24/55] fixed issue with introducing can_update column for existing users --- datameta/alembic/versions/20210625_f6a70402119f.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datameta/alembic/versions/20210625_f6a70402119f.py b/datameta/alembic/versions/20210625_f6a70402119f.py index 41f348d2..6d0d3389 100644 --- a/datameta/alembic/versions/20210625_f6a70402119f.py +++ b/datameta/alembic/versions/20210625_f6a70402119f.py @@ -37,7 +37,9 @@ def upgrade(): op.drop_column('metadatasets', 'deprecated_label') op.drop_column('metadatasets', 'replaced_by_id') op.drop_column('metadatasets', 'is_deprecated') - op.add_column('users', sa.Column('can_update', sa.Boolean(create_constraint=False), nullable=False)) + op.add_column('users', sa.Column('can_update', sa.Boolean(create_constraint=False), nullable=True)) + op.execute('UPDATE users SET can_update = site_admin;') + op.alter_column('users', 'can_update', nullable=False) # ### end Alembic commands ### From 55a11f7abe226745a5a1dcc4e61c523900115b78 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Jul 2021 16:48:22 +0000 Subject: [PATCH 25/55] openapi version bump (1.4.0) --- datameta/api/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index bf5e18ce..0cf0d869 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -15,7 +15,7 @@ openapi: 3.0.0 info: description: DataMeta - version: 1.3.0 + version: 1.4.0 title: DataMeta servers: From c6a340fa593946ff6bb1a4dfee544eb9b45a3bfd Mon Sep 17 00:00:00 2001 From: root Date: Mon, 5 Jul 2021 15:36:12 +0000 Subject: [PATCH 26/55] added/registered new api route; renamed information request operation ids --- datameta/api/__init__.py | 1 + datameta/api/openapi.yaml | 65 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/datameta/api/__init__.py b/datameta/api/__init__.py index 0ffef4ad..74aae972 100644 --- a/datameta/api/__init__.py +++ b/datameta/api/__init__.py @@ -62,6 +62,7 @@ def includeme(config: Configurator) -> None: config.add_route("groups_id", base_url + "/groups/{id}") config.add_route("rpc_delete_files", base_url + "/rpc/delete-files") config.add_route("rpc_delete_metadatasets", base_url + "/rpc/delete-metadatasets") + config.add_route("rpc_update_metadatasets", base_url + "/rpc/update-metadatasets") config.add_route("rpc_get_file_url", base_url + "/rpc/get-file-url/{id}") config.add_route('register_submit', base_url + "/registrations") config.add_route("register_settings", base_url + "/registrationsettings") diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index 0cf0d869..fd0fab61 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -104,7 +104,7 @@ paths: [Attention this endpoint is not RESTful, the result should not be cached.] tags: - Remote Procedure Calls - operationId: GetUserInformation + operationId: GetOwnUserInfo responses: "200": description: OK @@ -156,6 +156,7 @@ paths: '500': description: Internal Server Error + /rpc/delete-metadatasets: post: summary: Bulk-delete Staged MetaDataSets @@ -189,6 +190,43 @@ paths: '500': description: Internal Server Error + /rpc/update-metadatasets: + post: + summary: Update a range of metadatasets. + description: >- + Update metadatasets. + tags: + - Remote Procedure Calls + operationId: UpdateMetaDataSets + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ReplacementMetaDataSet" + description: >- + Provide all properties for one MetaDataSet and a list of MetaDataSets to be replaced. + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/MetaDataSetResponse" + '401': + description: Unauthorized + '403': + description: Forbidden + '404': + description: File not found + '400': + description: Validation Error + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorModel" + '500': + description: Internal Server Error + /rpc/get-file-url/{id}: get: @@ -347,7 +385,7 @@ paths: Get information about a user. tags: - Authentication and Users - operationId: UserInformationRequest + operationId: GetUserInfo parameters: - name: id in: path @@ -998,7 +1036,7 @@ paths: Get information about a group. tags: - Groups - operationId: GroupInformationRequest + operationId: GetGroupInfo parameters: - name: id in: path @@ -1437,6 +1475,18 @@ components: additionalProperties: false MetaDataSet: + type: object + properties: + record: + type: object + additionalProperties: true + # a free-form object, + # any property is allowed + required: + - record + additionalProperties: false + + ReplacementMetaDataSet: type: object properties: record: @@ -1448,10 +1498,17 @@ components: type: array items: type: string - replaces_label: + replacesLabel: type: string + fileIds: + type: array + items: + type: string required: - record + - replaces + - replacesLabel + - fileIds additionalProperties: false ServiceExecution: From e0dfe33c4a5f4add0d791c0e62cf8bc293171875 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Jul 2021 08:15:02 +0000 Subject: [PATCH 27/55] added update_metadatasets protoype --- datameta/api/metadatasets.py | 199 +++++++++++++++++++++++++++-------- 1 file changed, 158 insertions(+), 41 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index dfef3e65..7f8b48cd 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -30,6 +30,7 @@ from . import DataHolderBase from .. import errors from .metadata import get_all_metadata, get_service_metadata, get_metadata_with_access +from datameta.api.submissions import SubmissionResponse @dataclass @@ -39,6 +40,37 @@ class MetaDataSetServiceExecution(DataHolderBase): service_id : dict user_id : dict +@dataclass +class ReplacementMsetResponse(DataHolderBase): + id : dict + record : Dict[str, Optional[str]] + file_ids : Dict[str, Optional[Dict[str, str]]] + user_id : str + replaces : List[str] + submission_id : Optional[str] = None + service_executions : Optional[Dict[str, Optional[MetaDataSetServiceExecution]]] = None + + @classmethod + def from_metadataset(cls, metadataset: MetaDataSet, replaces: List[MetaDataSet], metadata_with_access: Dict[str, MetaDatum]): + return cls( + id = get_identifier(metadataset), + record = get_record_from_metadataset(metadataset, metadata_with_access), + file_ids = get_mset_associated_files(metadataset, metadata_with_access), + user_id = get_identifier(metadataset.user), + replaces = {get_identifier(mset) for mset in replaces}, + submission_id = get_identifier(metadataset.submission) if metadataset.submission else None, + service_executions = collect_service_executions(metadata_with_access, metadataset) + ) + + +def get_mset_associated_files(metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): + # Identify file ids associated with this metadataset for metadata with access + + return { + mdrec.metadatum.name : resource.get_identifier_or_none(mdrec.file) + for mdrec in metadataset.metadatumrecords + if mdrec.metadatum.isfile and mdrec.metadatum.name in metadata_with_access + } @dataclass class MetaDataSetResponse(DataHolderBase): @@ -56,15 +88,11 @@ def from_metadataset(metadataset: MetaDataSet, metadata_with_access: Dict[str, M a dictionary of metadata [MetaDatum.name, MetaDatum] that the receiving user has read access to.""" - # Identify file ids associated with this metadataset for metadata with access - file_ids = { mdrec.metadatum.name : resource.get_identifier_or_none(mdrec.file) - for mdrec in metadataset.metadatumrecords - if mdrec.metadatum.isfile and mdrec.metadatum.name in metadata_with_access} # Build the metadataset response return MetaDataSetResponse( id = get_identifier(metadataset), record = get_record_from_metadataset(metadataset, metadata_with_access), - file_ids = file_ids, + file_ids = get_mset_associated_files(metadataset, metadata_with_access), user_id = get_identifier(metadataset.user), submission_id = get_identifier(metadataset.submission) if metadataset.submission else None, service_executions = collect_service_executions(metadata_with_access, metadataset) @@ -137,26 +165,20 @@ def delete_metadatasets(request: Request) -> HTTPNoContent: @view_config( - route_name="metadatasets", - renderer='json', + route_name="rpc_update_metadatasets", + renderer="json", request_method="POST", openapi=True ) -def post(request: Request) -> MetaDataSetResponse: - """Create new metadataset""" +def update_metadatasets(request: Request) -> SubmissionResponse: auth_user = security.revalidate_user(request) db = request.dbsession # Obtain string converted version of the record record = record_to_strings(request.openapi_validated.body["record"]) - replaces = request.openapi_validated.body.get("replaces") - replaces_label = request.openapi_validated.body.get("replaces_label") - - if replaces and not replaces_label: - raise errors.get_validation_error(messages=['No reason (label) given for Metadataset replacement.']) # maybe label should be reason. - if not replaces and replaces_label: - raise errors.get_validation_error(messages=["No metadataset IDs specified but replacement reason (label) is given."]) + replaces = request.openapi_validated.body["replaces"] + replaces_label = request.openapi_validated.body["replacesLabel"] # Query the configured metadata. We're only considering and allowing # non-service metadata when creating a new metadataset. @@ -177,39 +199,134 @@ def post(request: Request) -> MetaDataSetResponse: db.add(mdata_set) - if replaces: - mset_repl_evt = MsetReplacementEvent( - user_id = auth_user.id, - datetime = datetime.utcnow(), - label = replaces_label, - new_metadataset_id = mdata_set.id + mset_repl_evt = MsetReplacementEvent( + user_id = auth_user.id, + datetime = datetime.utcnow(), + label = replaces_label, + new_metadataset_id = mdata_set.id + ) + db.add(mset_repl_evt) + + msets = [ + (mset_id, resource_by_id(db, MetaDataSet, mset_id)) + for mset_id in replaces + ] + + missing_msets = [("Invalid metadataset id.", mset_id) for mset_id, target_mset in msets if target_mset is None] + + if missing_msets: + messages, entities = zip(*missing_msets) + raise errors.get_validation_error(messages=messages, entities=entities) + + already_replaced = [ + (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset)}", mset_id) + for mset_id, target_mset in msets + if target_mset.replaced_via_event_id is not None + ] + + if already_replaced: + messages, entities = zip(*already_replaced) + raise errors.get_validation_error(messages=messages, entities=entities) + + for _, target_mset in msets: + target_mset.replaced_via_event_id = mset_repl_evt.id + + # Add NULL values for service metadata + service_metadata = get_service_metadata(db) + for s_mdatum in service_metadata.values(): + mdatum_rec = MetaDatumRecord( + metadatum_id = s_mdatum.id, + metadataset_id = mdata_set.id, + file_id = None, + value = None + ) + db.add(mdatum_rec) + + # Add the non-service metadata as specified in the request body + for name, value in record.items(): + mdatum_rec = MetaDatumRecord( + metadatum_id = metadata[name].id, + metadataset_id = mdata_set.id, + file_id = None, + value = value ) - db.add(mset_repl_evt) + db.add(mdatum_rec) + + # Collect files, drop duplicates + file_ids = set(request.openapi_validated.body['fileIds']) + db_files = { file_id : resource.resource_query_by_id(db, File, file_id).options(joinedload(File.metadatumrecord)).one_or_none() for file_id in file_ids } + + # Validate submission access to the specified files + validation.validate_submission_access(db, db_files, {}, auth_user) + + # Validate the provided records + validate_metadataset_record(service_metadata, record, return_err_message=False, rendered=False) + + # Validate the associations between files and records + fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { mdata_set.site_id : mdata_set }, ignore_submitted_metadatasets=True) + + # If there were any validation errors, return 400 + if val_errors: + entities, fields, messages = zip(*val_errors) + raise errors.get_validation_error(messages=messages, fields=fields, entities=entities) + + # Given that validation hasn't failed, we know that file names are unique. Flatten the dict. + fnames = { k : v[0] for k, v in fnames.items() } + + # Associate the files with the metadata + for fname, mdatrec in ref_fnames.items(): + mdatrec.file = fnames[fname] + db.add(mdatrec) + + # Add a submission + submission = Submission( + site_id = siteid.generate(request, Submission), + label = replaces_label, + date = datetime.utcnow(), + metadatasets = list(mdata_set), + group_id = auth_user.group.id + ) + db.add(submission) + + + # Check which metadata of this metadataset the user is allowed to view + metadata_with_access = get_metadata_with_access(db, auth_user) + + return ReplacementMsetResponse.from_metadataset(mdata_set, msets, metadata_with_access) - msets = [ - (mset_id, resource_by_id(db, MetaDataSet, mset_id)) - for mset_id in replaces - ] - missing_msets = [("Invalid metadataset id.", mset_id) for mset_id, target_mset in msets if target_mset is None] +@view_config( + route_name="metadatasets", + renderer='json', + request_method="POST", + openapi=True +) +def post(request: Request) -> MetaDataSetResponse: + """Create new metadataset""" + auth_user = security.revalidate_user(request) + db = request.dbsession - if missing_msets: - messages, entities = zip(*missing_msets) - raise errors.get_validation_error(messages=messages, entities=entities) + # Obtain string converted version of the record + record = record_to_strings(request.openapi_validated.body["record"]) - already_replaced = [ - (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset)}", mset_id) - for mset_id, target_mset in msets - if target_mset.replaced_via_event_id is not None - ] + # Query the configured metadata. We're only considering and allowing + # non-service metadata when creating a new metadataset. + metadata = get_all_metadata(db, include_service_metadata=False) - if already_replaced: - messages, entities = zip(*already_replaced) - raise errors.get_validation_error(messages=messages, entities=entities) + # prevalidate (raises 400 in case of validation failure): + validate_metadataset_record(metadata, record) - for _, target_mset in msets: - target_mset.replaced_via_event_id = mset_repl_evt.id + # Render records according to MetaDatum constraints. + record = render_record_values(metadata, record) + # construct new MetaDataSet: + mdata_set = MetaDataSet( + site_id = siteid.generate(request, MetaDataSet), + user_id = auth_user.id, + submission_id = None + ) + + db.add(mdata_set) db.flush() # Add NULL values for service metadata From 4fecd28218b006d99e65a22df9e65b477586a69e Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Jul 2021 08:29:01 +0000 Subject: [PATCH 28/55] pleasing flake8 and mypy --- datameta/api/metadatasets.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 7f8b48cd..8fe12be6 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -19,7 +19,7 @@ from pyramid.request import Request from sqlalchemy.orm import joinedload from sqlalchemy import and_ -from typing import Optional, Dict, List +from typing import Optional, Dict, List, Set, Any, Tuple from ..linting import validate_metadataset_record from .. import security, siteid, resource, validation from ..models import MetaDatum, MetaDataSet, ServiceExecution, Service, MetaDatumRecord, Submission, File @@ -40,18 +40,19 @@ class MetaDataSetServiceExecution(DataHolderBase): service_id : dict user_id : dict + @dataclass class ReplacementMsetResponse(DataHolderBase): id : dict record : Dict[str, Optional[str]] file_ids : Dict[str, Optional[Dict[str, str]]] user_id : str - replaces : List[str] + replaces : Set[Any] submission_id : Optional[str] = None service_executions : Optional[Dict[str, Optional[MetaDataSetServiceExecution]]] = None @classmethod - def from_metadataset(cls, metadataset: MetaDataSet, replaces: List[MetaDataSet], metadata_with_access: Dict[str, MetaDatum]): + def from_metadataset(cls, metadataset: MetaDataSet, replaces: List[Tuple[Any, MetaDataSet]], metadata_with_access: Dict[str, MetaDatum]): return cls( id = get_identifier(metadataset), record = get_record_from_metadataset(metadataset, metadata_with_access), @@ -61,7 +62,7 @@ def from_metadataset(cls, metadataset: MetaDataSet, replaces: List[MetaDataSet], submission_id = get_identifier(metadataset.submission) if metadataset.submission else None, service_executions = collect_service_executions(metadata_with_access, metadataset) ) - + def get_mset_associated_files(metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): # Identify file ids associated with this metadataset for metadata with access @@ -72,6 +73,7 @@ def get_mset_associated_files(metadataset: MetaDataSet, metadata_with_access: Di if mdrec.metadatum.isfile and mdrec.metadatum.name in metadata_with_access } + @dataclass class MetaDataSetResponse(DataHolderBase): """MetaDataSetResponse container for OpenApi communication""" @@ -278,7 +280,7 @@ def update_metadatasets(request: Request) -> SubmissionResponse: mdatrec.file = fnames[fname] db.add(mdatrec) - # Add a submission + # Add a submission submission = Submission( site_id = siteid.generate(request, Submission), label = replaces_label, @@ -288,7 +290,6 @@ def update_metadatasets(request: Request) -> SubmissionResponse: ) db.add(submission) - # Check which metadata of this metadataset the user is allowed to view metadata_with_access = get_metadata_with_access(db, auth_user) From 115eff66d88d6a7000d60c792de0abc0c0b85203 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Jul 2021 08:32:19 +0000 Subject: [PATCH 29/55] package bump (pleasing versioning checks) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 91ab6366..86b2bde5 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ setup( name = 'datameta', - version = '1.0.5', + version = '1.0.6', description = 'DataMeta - submission server for data and associated metadata', long_description = README + '\n\n' + CHANGES, author = 'Leon Kuchenbecker', From 4912fdbfc64aa6485abf41b8c64da6bccbedb7a3 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Jul 2021 08:38:28 +0000 Subject: [PATCH 30/55] merged openapi.yaml with upstream changes --- datameta/api/openapi.yaml | 40 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index 3703fcd1..7263c624 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -15,7 +15,7 @@ openapi: 3.0.0 info: description: DataMeta - version: 1.4.0 + version: 1.5.0 title: DataMeta servers: @@ -211,7 +211,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/MetaDataSetResponse" + $ref: "#/components/schemas/ReplacementMsetResponse" '401': description: Unauthorized '403': @@ -1530,6 +1530,42 @@ components: - fileIds additionalProperties: false + ReplacementMsetResponse: + type: object + properties: + record: + type: object + additionalProperties: true + # a free-form object, any property is allowed + replaces: + type: array + items: + type: string + fileIds: + type: object + additionalProperties: true + # a free-form object mapping the field names to file IDs + serviceExecutions: + type: object + # nullable as a whole for responses to users who cannot see service executions + nullable: true + additionalProperties: + # Maps record field names to service executions. The individual + # fields are also nullable for services that haven't been executed + # yet. + $ref: "#/components/schemas/MetaDataSetServiceExecution" + id: + $ref: "#/components/schemas/Identifier" + submissionId: + $ref: "#/components/schemas/NullableIdentifier" + userId: + $ref: "#/components/schemas/Identifier" + required: + - record + - replaces + - fileIds + additionalProperties: false + MetaDataSetResponse: type: object properties: From 36cd8ec3cd7ba025ffda96e59e4bf1e12d9d45c6 Mon Sep 17 00:00:00 2001 From: Christian Schudoma Date: Tue, 6 Jul 2021 15:05:04 +0200 Subject: [PATCH 31/55] Apply suggestions from code review rename update -> replace Co-authored-by: Leon Kuchenbecker --- datameta/api/__init__.py | 2 +- datameta/api/metadatasets.py | 4 ++-- datameta/api/openapi.yaml | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datameta/api/__init__.py b/datameta/api/__init__.py index 74aae972..4120783e 100644 --- a/datameta/api/__init__.py +++ b/datameta/api/__init__.py @@ -62,7 +62,7 @@ def includeme(config: Configurator) -> None: config.add_route("groups_id", base_url + "/groups/{id}") config.add_route("rpc_delete_files", base_url + "/rpc/delete-files") config.add_route("rpc_delete_metadatasets", base_url + "/rpc/delete-metadatasets") - config.add_route("rpc_update_metadatasets", base_url + "/rpc/update-metadatasets") + config.add_route("rpc_replace_metadatasets", base_url + "/rpc/replace-metadatasets") config.add_route("rpc_get_file_url", base_url + "/rpc/get-file-url/{id}") config.add_route('register_submit', base_url + "/registrations") config.add_route("register_settings", base_url + "/registrationsettings") diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 8fe12be6..e32aa5ac 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -167,7 +167,7 @@ def delete_metadatasets(request: Request) -> HTTPNoContent: @view_config( - route_name="rpc_update_metadatasets", + route_name="rpc_replace_metadatasets", renderer="json", request_method="POST", openapi=True @@ -265,7 +265,7 @@ def update_metadatasets(request: Request) -> SubmissionResponse: validate_metadataset_record(service_metadata, record, return_err_message=False, rendered=False) # Validate the associations between files and records - fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { mdata_set.site_id : mdata_set }, ignore_submitted_metadatasets=True) + fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { mdata_set.site_id : mdata_set }) # If there were any validation errors, return 400 if val_errors: diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index 7263c624..dcccbc85 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -190,14 +190,14 @@ paths: '500': description: Internal Server Error - /rpc/update-metadatasets: + /rpc/replace-metadatasets: post: - summary: Update a range of metadatasets. + summary: Replace a range of metadatasets. description: >- - Update metadatasets. + Replace metadatasets. tags: - Remote Procedure Calls - operationId: UpdateMetaDataSets + operationId: ReplaceMetaDataSets requestBody: content: application/json: From ac112b7f737c1a75763c02c8f60f6dd33ddf9f83 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Jul 2021 16:47:10 +0000 Subject: [PATCH 32/55] works --- datameta/api/metadatasets.py | 22 +++++++++++++--------- datameta/api/openapi.yaml | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index e32aa5ac..746ff55d 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -19,7 +19,7 @@ from pyramid.request import Request from sqlalchemy.orm import joinedload from sqlalchemy import and_ -from typing import Optional, Dict, List, Set, Any, Tuple +from typing import Optional, Dict, List, Any, Tuple from ..linting import validate_metadataset_record from .. import security, siteid, resource, validation from ..models import MetaDatum, MetaDataSet, ServiceExecution, Service, MetaDatumRecord, Submission, File @@ -47,7 +47,7 @@ class ReplacementMsetResponse(DataHolderBase): record : Dict[str, Optional[str]] file_ids : Dict[str, Optional[Dict[str, str]]] user_id : str - replaces : Set[Any] + replaces : List[Dict[str, Any]] submission_id : Optional[str] = None service_executions : Optional[Dict[str, Optional[MetaDataSetServiceExecution]]] = None @@ -58,7 +58,7 @@ def from_metadataset(cls, metadataset: MetaDataSet, replaces: List[Tuple[Any, Me record = get_record_from_metadataset(metadataset, metadata_with_access), file_ids = get_mset_associated_files(metadataset, metadata_with_access), user_id = get_identifier(metadataset.user), - replaces = {get_identifier(mset) for mset in replaces}, + replaces = [get_identifier(mset) for _, mset in replaces], submission_id = get_identifier(metadataset.submission) if metadataset.submission else None, service_executions = collect_service_executions(metadata_with_access, metadataset) ) @@ -172,7 +172,7 @@ def delete_metadatasets(request: Request) -> HTTPNoContent: request_method="POST", openapi=True ) -def update_metadatasets(request: Request) -> SubmissionResponse: +def replace_metadatasets(request: Request) -> SubmissionResponse: auth_user = security.revalidate_user(request) db = request.dbsession @@ -200,6 +200,7 @@ def update_metadatasets(request: Request) -> SubmissionResponse: ) db.add(mdata_set) + db.flush() mset_repl_evt = MsetReplacementEvent( user_id = auth_user.id, @@ -208,20 +209,21 @@ def update_metadatasets(request: Request) -> SubmissionResponse: new_metadataset_id = mdata_set.id ) db.add(mset_repl_evt) + db.flush() msets = [ (mset_id, resource_by_id(db, MetaDataSet, mset_id)) for mset_id in replaces ] - missing_msets = [("Invalid metadataset id.", mset_id) for mset_id, target_mset in msets if target_mset is None] + missing_msets = [("Invalid metadataset id.", target_mset) for mset_id, target_mset in msets if target_mset is None] if missing_msets: messages, entities = zip(*missing_msets) raise errors.get_validation_error(messages=messages, entities=entities) already_replaced = [ - (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset)}", mset_id) + (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset)}", target_mset) for mset_id, target_mset in msets if target_mset.replaced_via_event_id is not None ] @@ -262,7 +264,7 @@ def update_metadatasets(request: Request) -> SubmissionResponse: validation.validate_submission_access(db, db_files, {}, auth_user) # Validate the provided records - validate_metadataset_record(service_metadata, record, return_err_message=False, rendered=False) + validate_metadataset_record(metadata, record, return_err_message=False, rendered=True) # Validate the associations between files and records fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { mdata_set.site_id : mdata_set }) @@ -280,12 +282,14 @@ def update_metadatasets(request: Request) -> SubmissionResponse: mdatrec.file = fnames[fname] db.add(mdatrec) + _mset = resource.resource_query_by_id(db, MetaDataSet, mdata_set.site_id).options(joinedload(MetaDataSet.metadatumrecords).joinedload(MetaDatumRecord.metadatum)).one_or_none() + # Add a submission submission = Submission( site_id = siteid.generate(request, Submission), label = replaces_label, date = datetime.utcnow(), - metadatasets = list(mdata_set), + metadatasets = [_mset], group_id = auth_user.group.id ) db.add(submission) @@ -293,7 +297,7 @@ def update_metadatasets(request: Request) -> SubmissionResponse: # Check which metadata of this metadataset the user is allowed to view metadata_with_access = get_metadata_with_access(db, auth_user) - return ReplacementMsetResponse.from_metadataset(mdata_set, msets, metadata_with_access) + return ReplacementMsetResponse.from_metadataset(_mset, msets, metadata_with_access) @view_config( diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index dcccbc85..fbd35af9 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -1540,7 +1540,7 @@ components: replaces: type: array items: - type: string + type: object fileIds: type: object additionalProperties: true From 4a35bcead1445eac121a710ed97e9fc8c38307c5 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Jul 2021 17:16:27 +0000 Subject: [PATCH 33/55] weird. --- datameta/api/metadatasets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 746ff55d..182c9c00 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -223,14 +223,14 @@ def replace_metadatasets(request: Request) -> SubmissionResponse: raise errors.get_validation_error(messages=messages, entities=entities) already_replaced = [ - (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset)}", target_mset) + (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset).get('site')}", target_mset) for mset_id, target_mset in msets if target_mset.replaced_via_event_id is not None ] if already_replaced: messages, entities = zip(*already_replaced) - raise errors.get_validation_error(messages=messages, entities=entities) + raise errors.get_validation_error(messages=list(messages), entities=list(entities)) for _, target_mset in msets: target_mset.replaced_via_event_id = mset_repl_evt.id From e0803da857984a6ff45cbf860e2d016e845f2688 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Jul 2021 17:18:36 +0000 Subject: [PATCH 34/55] weird. --- datameta/api/metadatasets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 182c9c00..cace24b2 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -220,7 +220,7 @@ def replace_metadatasets(request: Request) -> SubmissionResponse: if missing_msets: messages, entities = zip(*missing_msets) - raise errors.get_validation_error(messages=messages, entities=entities) + raise errors.get_validation_error(messages=list(messages), entities=list(entities)) already_replaced = [ (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset).get('site')}", target_mset) @@ -272,7 +272,7 @@ def replace_metadatasets(request: Request) -> SubmissionResponse: # If there were any validation errors, return 400 if val_errors: entities, fields, messages = zip(*val_errors) - raise errors.get_validation_error(messages=messages, fields=fields, entities=entities) + raise errors.get_validation_error(messages=list(messages), fields=fields, entities=list(entities)) # Given that validation hasn't failed, we know that file names are unique. Flatten the dict. fnames = { k : v[0] for k, v in fnames.items() } From 6f751234f7c63be094b69791657884d35cec4e8e Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Jul 2021 17:23:24 +0000 Subject: [PATCH 35/55] weird --- datameta/api/metadatasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index cace24b2..38f6a6b4 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -220,7 +220,7 @@ def replace_metadatasets(request: Request) -> SubmissionResponse: if missing_msets: messages, entities = zip(*missing_msets) - raise errors.get_validation_error(messages=list(messages), entities=list(entities)) + raise errors.get_validation_error(messages=list(messages), entities=list(entities)) # @lkuchenb: any idea why these list-casts are necessary to please mypy? they're not required in e.g validation.py:63 already_replaced = [ (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset).get('site')}", target_mset) From d0b3f6b682e397f00b8061a7358b0ec6af4045c3 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 7 Jul 2021 09:07:31 +0000 Subject: [PATCH 36/55] refactored + working --- datameta/api/metadatasets.py | 288 ++++++++++++++++------------------- 1 file changed, 135 insertions(+), 153 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 38f6a6b4..b05c25a4 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -64,16 +64,6 @@ def from_metadataset(cls, metadataset: MetaDataSet, replaces: List[Tuple[Any, Me ) -def get_mset_associated_files(metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): - # Identify file ids associated with this metadataset for metadata with access - - return { - mdrec.metadatum.name : resource.get_identifier_or_none(mdrec.file) - for mdrec in metadataset.metadatumrecords - if mdrec.metadatum.isfile and mdrec.metadatum.name in metadata_with_access - } - - @dataclass class MetaDataSetResponse(DataHolderBase): """MetaDataSetResponse container for OpenApi communication""" @@ -101,6 +91,15 @@ def from_metadataset(metadataset: MetaDataSet, metadata_with_access: Dict[str, M ) +def get_mset_associated_files(metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): + """ Identify file ids associated with this metadataset for metadata with access """ + return { + mdrec.metadatum.name : resource.get_identifier_or_none(mdrec.file) + for mdrec in metadataset.metadatumrecords + if mdrec.metadatum.isfile and mdrec.metadatum.name in metadata_with_access + } + + def record_to_strings(record: Dict[str, str]): return { k: str(v) if v is not None else None @@ -166,54 +165,64 @@ def delete_metadatasets(request: Request) -> HTTPNoContent: return HTTPNoContent() -@view_config( - route_name="rpc_replace_metadatasets", - renderer="json", - request_method="POST", - openapi=True -) -def replace_metadatasets(request: Request) -> SubmissionResponse: - auth_user = security.revalidate_user(request) - db = request.dbsession +def initialize_service_metadata(db, mset_id): + service_metadata = get_service_metadata(db) + for s_mdatum in service_metadata.values(): + mdatum_rec = MetaDatumRecord( + metadatum_id = s_mdatum.id, + metadataset_id = mset_id, + file_id = None, + value = None + ) + db.add(mdatum_rec) - # Obtain string converted version of the record - record = record_to_strings(request.openapi_validated.body["record"]) - replaces = request.openapi_validated.body["replaces"] - replaces_label = request.openapi_validated.body["replacesLabel"] +def add_metadata_from_request(db, record, metadata, mset_id): + # Add the non-service metadata as specified in the request body + for name, value in record.items(): + mdatum_rec = MetaDatumRecord( + metadatum_id = metadata[name].id, + metadataset_id = mset_id, + file_id = None, + value = value + ) + db.add(mdatum_rec) - # Query the configured metadata. We're only considering and allowing - # non-service metadata when creating a new metadataset. - metadata = get_all_metadata(db, include_service_metadata=False) - # prevalidate (raises 400 in case of validation failure): - validate_metadataset_record(metadata, record) +def validate_associated_files(db, file_ids, auth_user): + # Collect files, drop duplicates + db_files = { file_id : resource.resource_query_by_id(db, File, file_id).options(joinedload(File.metadatumrecord)).one_or_none() for file_id in set(file_ids) } - # Render records according to MetaDatum constraints. - record = render_record_values(metadata, record) + # Validate submission access to the specified files + validation.validate_submission_access(db, db_files, {}, auth_user) - # construct new MetaDataSet: - mdata_set = MetaDataSet( - site_id = siteid.generate(request, MetaDataSet), - user_id = auth_user.id, - submission_id = None - ) + return db_files - db.add(mdata_set) - db.flush() - mset_repl_evt = MsetReplacementEvent( - user_id = auth_user.id, - datetime = datetime.utcnow(), - label = replaces_label, - new_metadataset_id = mdata_set.id - ) - db.add(mset_repl_evt) - db.flush() +def link_files(db, mdata_set, db_files, ignore_submitted=False): + + # Validate the associations between files and records + fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { mdata_set.site_id : mdata_set }, ignore_submitted_metadatasets=ignore_submitted) + + # If there were any validation errors, return 400 + if val_errors: + entities, fields, messages = zip(*val_errors) + raise errors.get_validation_error(messages=list(messages), fields=fields, entities=list(entities)) + + # Given that validation hasn't failed, we know that file names are unique. Flatten the dict. + fnames = { k : v[0] for k, v in fnames.items() } + + # Associate the files with the metadata + for fname, mdatrec in ref_fnames.items(): + mdatrec.file = fnames[fname] + db.add(mdatrec) + + +def execute_mset_replacement(db, new_mset_id, replaced_msets, replaces_label, user_id): msets = [ (mset_id, resource_by_id(db, MetaDataSet, mset_id)) - for mset_id in replaces + for mset_id in replaced_msets ] missing_msets = [("Invalid metadataset id.", target_mset) for mset_id, target_mset in msets if target_mset is None] @@ -232,72 +241,92 @@ def replace_metadatasets(request: Request) -> SubmissionResponse: messages, entities = zip(*already_replaced) raise errors.get_validation_error(messages=list(messages), entities=list(entities)) + mset_repl_evt = MsetReplacementEvent( + user_id = user_id, + datetime = datetime.utcnow(), + label = replaces_label, + new_metadataset_id = new_mset_id + ) + db.add(mset_repl_evt) + db.flush() + for _, target_mset in msets: target_mset.replaced_via_event_id = mset_repl_evt.id - # Add NULL values for service metadata - service_metadata = get_service_metadata(db) - for s_mdatum in service_metadata.values(): - mdatum_rec = MetaDatumRecord( - metadatum_id = s_mdatum.id, - metadataset_id = mdata_set.id, - file_id = None, - value = None - ) - db.add(mdatum_rec) + return msets - # Add the non-service metadata as specified in the request body - for name, value in record.items(): - mdatum_rec = MetaDatumRecord( - metadatum_id = metadata[name].id, - metadataset_id = mdata_set.id, - file_id = None, - value = value - ) - db.add(mdatum_rec) - # Collect files, drop duplicates - file_ids = set(request.openapi_validated.body['fileIds']) - db_files = { file_id : resource.resource_query_by_id(db, File, file_id).options(joinedload(File.metadatumrecord)).one_or_none() for file_id in file_ids } +def prepare_record(db, request): + # Obtain string converted version of the record + record = record_to_strings(request.openapi_validated.body["record"]) - # Validate submission access to the specified files - validation.validate_submission_access(db, db_files, {}, auth_user) + # Query the configured metadata. We're only considering and allowing + # non-service metadata when creating a new metadataset. + metadata = get_all_metadata(db, include_service_metadata=False) - # Validate the provided records - validate_metadataset_record(metadata, record, return_err_message=False, rendered=True) + # prevalidate (raises 400 in case of validation failure): + validate_metadataset_record(metadata, record) - # Validate the associations between files and records - fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { mdata_set.site_id : mdata_set }) + # Render records according to MetaDatum constraints. + record = render_record_values(metadata, record) - # If there were any validation errors, return 400 - if val_errors: - entities, fields, messages = zip(*val_errors) - raise errors.get_validation_error(messages=list(messages), fields=fields, entities=list(entities)) + return record, metadata - # Given that validation hasn't failed, we know that file names are unique. Flatten the dict. - fnames = { k : v[0] for k, v in fnames.items() } - # Associate the files with the metadata - for fname, mdatrec in ref_fnames.items(): - mdatrec.file = fnames[fname] - db.add(mdatrec) +@view_config( + route_name="rpc_replace_metadatasets", + renderer="json", + request_method="POST", + openapi=True +) +def replace_metadatasets(request: Request) -> SubmissionResponse: + auth_user = security.revalidate_user(request) + db = request.dbsession + + record, metadata = prepare_record(db, request) + + # construct new MetaDataSet: + mdata_set = MetaDataSet( + site_id = siteid.generate(request, MetaDataSet), + user_id = auth_user.id, + submission_id = None + ) + + db.add(mdata_set) + db.flush() + + replaces = request.openapi_validated.body["replaces"] + replaces_label = request.openapi_validated.body["replacesLabel"] + + replaced_msets = execute_mset_replacement(db, mdata_set.id, replaces, replaces_label, auth_user.id) + + initialize_service_metadata(db, mdata_set.id) + + add_metadata_from_request(db, record, metadata, mdata_set.id) - _mset = resource.resource_query_by_id(db, MetaDataSet, mdata_set.site_id).options(joinedload(MetaDataSet.metadatumrecords).joinedload(MetaDatumRecord.metadatum)).one_or_none() + db_files = validate_associated_files(db, set(request.openapi_validated.body['fileIds']), auth_user) + + # Validate the provided records + validate_metadataset_record(metadata, record, return_err_message=False, rendered=True) + + link_files(db, mdata_set, db_files, ignore_submitted=False) + + mdata_set = resource.resource_query_by_id(db, MetaDataSet, mdata_set.site_id).options(joinedload(MetaDataSet.metadatumrecords).joinedload(MetaDatumRecord.metadatum)).one_or_none() # Add a submission submission = Submission( - site_id = siteid.generate(request, Submission), - label = replaces_label, - date = datetime.utcnow(), - metadatasets = [_mset], - group_id = auth_user.group.id - ) + site_id = siteid.generate(request, Submission), + label = replaces_label, + date = datetime.utcnow(), + metadatasets = [mdata_set], + group_id = auth_user.group.id + ) db.add(submission) # Check which metadata of this metadataset the user is allowed to view metadata_with_access = get_metadata_with_access(db, auth_user) - return ReplacementMsetResponse.from_metadataset(_mset, msets, metadata_with_access) + return ReplacementMsetResponse.from_metadataset(mdata_set, replaced_msets, metadata_with_access) @view_config( @@ -311,18 +340,7 @@ def post(request: Request) -> MetaDataSetResponse: auth_user = security.revalidate_user(request) db = request.dbsession - # Obtain string converted version of the record - record = record_to_strings(request.openapi_validated.body["record"]) - - # Query the configured metadata. We're only considering and allowing - # non-service metadata when creating a new metadataset. - metadata = get_all_metadata(db, include_service_metadata=False) - - # prevalidate (raises 400 in case of validation failure): - validate_metadataset_record(metadata, record) - - # Render records according to MetaDatum constraints. - record = render_record_values(metadata, record) + record, metadata = prepare_record(db, request) # construct new MetaDataSet: mdata_set = MetaDataSet( @@ -334,26 +352,9 @@ def post(request: Request) -> MetaDataSetResponse: db.add(mdata_set) db.flush() - # Add NULL values for service metadata - service_metadata = get_service_metadata(db) - for s_mdatum in service_metadata.values(): - mdatum_rec = MetaDatumRecord( - metadatum_id = s_mdatum.id, - metadataset_id = mdata_set.id, - file_id = None, - value = None - ) - db.add(mdatum_rec) + initialize_service_metadata(db, mdata_set.id) - # Add the non-service metadata as specified in the request body - for name, value in record.items(): - mdatum_rec = MetaDatumRecord( - metadatum_id = metadata[name].id, - metadataset_id = mdata_set.id, - file_id = None, - value = value - ) - db.add(mdatum_rec) + add_metadata_from_request(db, record, metadata, mdata_set.id) return MetaDataSetResponse( id = get_identifier(mdata_set), @@ -458,9 +459,9 @@ def get_metadatasets(request: Request) -> List[MetaDataSetResponse]: raise HTTPNotFound() return [ - MetaDataSetResponse.from_metadataset(mdata_set, metadata_with_access) - for mdata_set in mdata_sets - ] + MetaDataSetResponse.from_metadataset(mdata_set, metadata_with_access) + for mdata_set in mdata_sets + ] @view_config( @@ -571,12 +572,7 @@ def set_metadata_via_service(request: Request) -> MetaDataSetResponse: messages, fields = zip(*val_errors) raise errors.get_validation_error(messages, fields) - # Collect files, drop duplicates - file_ids = set(request.openapi_validated.body['fileIds']) - db_files = { file_id : resource.resource_query_by_id(db, File, file_id).options(joinedload(File.metadatumrecord)).one_or_none() for file_id in file_ids } - - # Validate submission access to the specified files - validation.validate_submission_access(db, db_files, {}, auth_user) + db_files = validate_associated_files(db, set(request.openapi_validated.body['fileIds']), auth_user) # Validate the provided records validate_metadataset_record(service_metadata, records, return_err_message=False, rendered=False) @@ -588,29 +584,15 @@ def set_metadata_via_service(request: Request) -> MetaDataSetResponse: db_rec.value = record_value db.add(db_rec) - # Validate the associations between files and records - fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { metadataset.site_id : metadataset }, ignore_submitted_metadatasets=True) - - # If there were any validation errors, return 400 - if val_errors: - entities, fields, messages = zip(*val_errors) - raise errors.get_validation_error(messages=messages, fields=fields, entities=entities) - - # Given that validation hasn't failed, we know that file names are unique. Flatten the dict. - fnames = { k : v[0] for k, v in fnames.items() } - - # Associate the files with the metadata - for fname, mdatrec in ref_fnames.items(): - mdatrec.file = fnames[fname] - db.add(mdatrec) + link_files(db, metadataset, db_files, ignore_submitted=True) # Create a service execution sexec = ServiceExecution( - service = service, - metadataset = metadataset, - user = auth_user, - datetime = datetime.utcnow() - ) + service = service, + metadataset = metadataset, + user = auth_user, + datetime = datetime.utcnow() + ) db.add(sexec) From e77bf56132f7d87e9123c97058fcf6f46f003473 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 7 Jul 2021 09:38:41 +0000 Subject: [PATCH 37/55] added preliminary test for metadataset replacement --- .../fixtures/files/group_x_file_1.txt | 1 + .../fixtures/files/group_x_file_2.txt | 1 + .../fixtures/files_independent.yaml | 24 +++++++ tests/integration/test_mset_replacement.py | 67 +++++++++++++++++++ 4 files changed, 93 insertions(+) create mode 100644 tests/integration/fixtures/files/group_x_file_1.txt create mode 100644 tests/integration/fixtures/files/group_x_file_2.txt create mode 100644 tests/integration/test_mset_replacement.py diff --git a/tests/integration/fixtures/files/group_x_file_1.txt b/tests/integration/fixtures/files/group_x_file_1.txt new file mode 100644 index 00000000..f226f2ae --- /dev/null +++ b/tests/integration/fixtures/files/group_x_file_1.txt @@ -0,0 +1 @@ +user_a_file_1 diff --git a/tests/integration/fixtures/files/group_x_file_2.txt b/tests/integration/fixtures/files/group_x_file_2.txt new file mode 100644 index 00000000..b2a72147 --- /dev/null +++ b/tests/integration/fixtures/files/group_x_file_2.txt @@ -0,0 +1 @@ +user_a_file_2 diff --git a/tests/integration/fixtures/files_independent.yaml b/tests/integration/fixtures/files_independent.yaml index 3f669ae9..58aa84e1 100644 --- a/tests/integration/fixtures/files_independent.yaml +++ b/tests/integration/fixtures/files_independent.yaml @@ -35,3 +35,27 @@ user_a_file_2: user: fixtureset: users name: user_a + +group_x_file_1: + class: File + attributes: + site_id: group_x_file_1 + name: group_x_file_1.txt + content_uploaded: True + checksum: bbe230ca2f18d04af27f03f7f13631af + references: + user: + fixtureset: users + name: group_x_admin + +group_x_file_2: + class: File + attributes: + site_id: group_x_file_2 + name: group_x_file_2.txt + content_uploaded: True + checksum: f7a3deca77966ae714b0e9d73402659b + references: + user: + fixtureset: users + name: group_x_admin diff --git a/tests/integration/test_mset_replacement.py b/tests/integration/test_mset_replacement.py new file mode 100644 index 00000000..85526530 --- /dev/null +++ b/tests/integration/test_mset_replacement.py @@ -0,0 +1,67 @@ +# Copyright 2021 Universität Tübingen, DKFZ and EMBL for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from parameterized import parameterized + +from datameta.api import base_url + +from . import BaseIntegrationTest + + +class MsetReplacementTest(BaseIntegrationTest): + def setUp(self): + super().setUp() + self.fixture_manager.load_fixtureset('groups') + self.fixture_manager.load_fixtureset('users') + self.fixture_manager.load_fixtureset('apikeys') + self.fixture_manager.load_fixtureset('services') + self.fixture_manager.load_fixtureset('metadata') + self.fixture_manager.load_fixtureset('files_msets') + self.fixture_manager.load_fixtureset('files_independent') + self.fixture_manager.load_fixtureset('submissions') + self.fixture_manager.load_fixtureset('metadatasets') + self.fixture_manager.load_fixtureset('serviceexecutions') + self.fixture_manager.copy_files_to_storage() + self.fixture_manager.populate_metadatasets() + + @parameterized.expand([ + ("success", "group_x_admin", 200), + ]) + def test_mset_replacement(self, testname: str, executing_user: str, expected_status: int): + user = self.fixture_manager.get_fixture("users", executing_user) if executing_user else None + auth_headers = self.apikey_auth(user) if user else {} + + request_body = { + "record": { + "Date": "2021-07-01", + "ZIP Code": "108", + "ID": "XC13", # this is weird: when using the swagger interface, you can use the key "#ID", is that # stripped off somewhere? + "FileR2": "group_x_file_1.txt", + "FileR1": "group_x_file_2.txt" + }, + "replaces": [ + "mset_a" + ], + "replacesLabel": "because i can", + "fileIds": [ + "group_x_file_1", "group_x_file_2" + ] + } + + self.testapp.post_json( + url = f"{base_url}/rpc/replace-metadatasets", + headers = auth_headers, + status = expected_status, + params = request_body + ) From 0289f2a22c77b05bf33618f6e4dd6209bc53f22d Mon Sep 17 00:00:00 2001 From: root Date: Wed, 7 Jul 2021 10:07:14 +0000 Subject: [PATCH 38/55] workaround to re-enable bulk-deletion test --- tests/integration/test_bulkdelete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_bulkdelete.py b/tests/integration/test_bulkdelete.py index 67837c84..211eeb0c 100644 --- a/tests/integration/test_bulkdelete.py +++ b/tests/integration/test_bulkdelete.py @@ -20,7 +20,7 @@ def test_file_deletion(self): auth_headers = self.apikey_auth(user) files = self.fixture_manager.get_fixtureset('files_independent') - file_ids = [ file.site_id for file in files.values() ] + file_ids = [ file.site_id for file in files.values() if not file.site_id.startswith("group") ] self.testapp.post_json( f"{base_url}/rpc/delete-files", From ac2b43153f20104f86fddab9e39070f9018086e6 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 14 Jul 2021 07:11:34 +0000 Subject: [PATCH 39/55] removed redundant label from msetreplacementevents (label is attached to submission) --- datameta/alembic/versions/20210625_f6a70402119f.py | 1 - datameta/api/metadatasets.py | 5 ++--- datameta/models/db.py | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/datameta/alembic/versions/20210625_f6a70402119f.py b/datameta/alembic/versions/20210625_f6a70402119f.py index 6d0d3389..5f05a0d4 100644 --- a/datameta/alembic/versions/20210625_f6a70402119f.py +++ b/datameta/alembic/versions/20210625_f6a70402119f.py @@ -24,7 +24,6 @@ def upgrade(): sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), sa.Column('user_id', sa.Integer(), nullable=False), sa.Column('datetime', sa.DateTime(), nullable=False), - sa.Column('label', sa.String(length=140), nullable=False), sa.Column('new_metadataset_id', sa.Integer(), nullable=False), sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_new_metadataset_id_metadatasets')), sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index b05c25a4..a3c77138 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -218,7 +218,7 @@ def link_files(db, mdata_set, db_files, ignore_submitted=False): db.add(mdatrec) -def execute_mset_replacement(db, new_mset_id, replaced_msets, replaces_label, user_id): +def execute_mset_replacement(db, new_mset_id, replaced_msets, user_id): msets = [ (mset_id, resource_by_id(db, MetaDataSet, mset_id)) @@ -244,7 +244,6 @@ def execute_mset_replacement(db, new_mset_id, replaced_msets, replaces_label, us mset_repl_evt = MsetReplacementEvent( user_id = user_id, datetime = datetime.utcnow(), - label = replaces_label, new_metadataset_id = new_mset_id ) db.add(mset_repl_evt) @@ -298,7 +297,7 @@ def replace_metadatasets(request: Request) -> SubmissionResponse: replaces = request.openapi_validated.body["replaces"] replaces_label = request.openapi_validated.body["replacesLabel"] - replaced_msets = execute_mset_replacement(db, mdata_set.id, replaces, replaces_label, auth_user.id) + replaced_msets = execute_mset_replacement(db, mdata_set.id, replaces, auth_user.id) initialize_service_metadata(db, mdata_set.id) diff --git a/datameta/models/db.py b/datameta/models/db.py index 91373f0e..3f1894f8 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -236,7 +236,6 @@ class MsetReplacementEvent(Base): uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) user_id = Column(Integer, ForeignKey('users.id'), nullable=False) datetime = Column(DateTime, nullable=False) - label = Column(String(140), nullable=False) new_metadataset_id = Column(Integer, ForeignKey('metadatasets.id'), nullable=False) # Relationships From 3e1bfcd08590177a3d0b9c503f96524e850e836d Mon Sep 17 00:00:00 2001 From: root Date: Wed, 14 Jul 2021 11:57:12 +0000 Subject: [PATCH 40/55] cleaning --- tests/integration/test_mset_replacement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_mset_replacement.py b/tests/integration/test_mset_replacement.py index 85526530..21e75134 100644 --- a/tests/integration/test_mset_replacement.py +++ b/tests/integration/test_mset_replacement.py @@ -46,7 +46,7 @@ def test_mset_replacement(self, testname: str, executing_user: str, expected_sta "record": { "Date": "2021-07-01", "ZIP Code": "108", - "ID": "XC13", # this is weird: when using the swagger interface, you can use the key "#ID", is that # stripped off somewhere? + "ID": "XC13", "FileR2": "group_x_file_1.txt", "FileR1": "group_x_file_2.txt" }, From c6f22a0dee233a45806c96cd4ce0080bdd5e8d4a Mon Sep 17 00:00:00 2001 From: root Date: Wed, 14 Jul 2021 11:57:51 +0000 Subject: [PATCH 41/55] added can_update=True to initial user --- datameta/scripts/initialize_db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datameta/scripts/initialize_db.py b/datameta/scripts/initialize_db.py index a1a16ca4..c2b3df3a 100644 --- a/datameta/scripts/initialize_db.py +++ b/datameta/scripts/initialize_db.py @@ -68,7 +68,8 @@ def create_initial_user(request, email, fullname, password, groupname): group=init_group, group_admin=True, site_admin=True, - site_read=True + site_read=True, + can_update=True ) db.add(root) From 9e88373c304db78445e370d997d701a43d594287 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 14 Jul 2021 11:58:41 +0000 Subject: [PATCH 42/55] metadatasetresponses now include replaced/replaces information (#462) --- datameta/api/metadatasets.py | 9 +++++++-- datameta/api/openapi.yaml | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index a3c77138..c07ac63f 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -53,6 +53,7 @@ class ReplacementMsetResponse(DataHolderBase): @classmethod def from_metadataset(cls, metadataset: MetaDataSet, replaces: List[Tuple[Any, MetaDataSet]], metadata_with_access: Dict[str, MetaDatum]): + return cls( id = get_identifier(metadataset), record = get_record_from_metadataset(metadataset, metadata_with_access), @@ -60,7 +61,7 @@ def from_metadataset(cls, metadataset: MetaDataSet, replaces: List[Tuple[Any, Me user_id = get_identifier(metadataset.user), replaces = [get_identifier(mset) for _, mset in replaces], submission_id = get_identifier(metadataset.submission) if metadataset.submission else None, - service_executions = collect_service_executions(metadata_with_access, metadataset) + service_executions = collect_service_executions(metadata_with_access, metadataset), ) @@ -73,6 +74,8 @@ class MetaDataSetResponse(DataHolderBase): user_id : str submission_id : Optional[str] = None service_executions : Optional[Dict[str, Optional[MetaDataSetServiceExecution]]] = None + replaces : Optional[List[str]] = None + replaced_by : Optional[str] = None @staticmethod def from_metadataset(metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): @@ -87,7 +90,9 @@ def from_metadataset(metadataset: MetaDataSet, metadata_with_access: Dict[str, M file_ids = get_mset_associated_files(metadataset, metadata_with_access), user_id = get_identifier(metadataset.user), submission_id = get_identifier(metadataset.submission) if metadataset.submission else None, - service_executions = collect_service_executions(metadata_with_access, metadataset) + service_executions = collect_service_executions(metadata_with_access, metadataset), + replaces = [get_identifier(mset) for event in metadataset.replaces_via_event for mset in event.replaced_metadatasets] if metadataset.replaces_via_event else None, + replaced_by = get_identifier(metadataset.replaced_via_event.new_metadataset) if metadataset.replaced_via_event else None ) diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index fbd35af9..90462f5b 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -1592,6 +1592,14 @@ components: $ref: "#/components/schemas/NullableIdentifier" userId: $ref: "#/components/schemas/Identifier" + replaces: + type: array + items: + type: object + nullable: true + replacedBy: + type: object + nullable: true required: - record additionalProperties: false From d882ffef408f9d09a0149e4c859a5e440a0f60ae Mon Sep 17 00:00:00 2001 From: root Date: Wed, 28 Jul 2021 11:56:16 +0000 Subject: [PATCH 43/55] added more testcases --- .../fixtures/files/group_x_file_3.txt | 1 + .../fixtures/files/group_x_file_4.txt | 1 + .../fixtures/files_independent.yaml | 24 +++++++++++++ tests/integration/test_mset_replacement.py | 36 ++++++++++++++++--- 4 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 tests/integration/fixtures/files/group_x_file_3.txt create mode 100644 tests/integration/fixtures/files/group_x_file_4.txt diff --git a/tests/integration/fixtures/files/group_x_file_3.txt b/tests/integration/fixtures/files/group_x_file_3.txt new file mode 100644 index 00000000..f226f2ae --- /dev/null +++ b/tests/integration/fixtures/files/group_x_file_3.txt @@ -0,0 +1 @@ +user_a_file_1 diff --git a/tests/integration/fixtures/files/group_x_file_4.txt b/tests/integration/fixtures/files/group_x_file_4.txt new file mode 100644 index 00000000..b2a72147 --- /dev/null +++ b/tests/integration/fixtures/files/group_x_file_4.txt @@ -0,0 +1 @@ +user_a_file_2 diff --git a/tests/integration/fixtures/files_independent.yaml b/tests/integration/fixtures/files_independent.yaml index 58aa84e1..c9c64331 100644 --- a/tests/integration/fixtures/files_independent.yaml +++ b/tests/integration/fixtures/files_independent.yaml @@ -59,3 +59,27 @@ group_x_file_2: user: fixtureset: users name: group_x_admin + +group_x_file_3: + class: File + attributes: + site_id: group_x_file_3 + name: group_x_file_3.txt + content_uploaded: True + checksum: bbe230ca2f18d04af27f03f7f13631af + references: + user: + fixtureset: users + name: group_x_admin + +group_x_file_4: + class: File + attributes: + site_id: group_x_file_4 + name: group_x_file_4.txt + content_uploaded: True + checksum: f7a3deca77966ae714b0e9d73402659b + references: + user: + fixtureset: users + name: group_x_admin diff --git a/tests/integration/test_mset_replacement.py b/tests/integration/test_mset_replacement.py index 21e75134..07f9e22d 100644 --- a/tests/integration/test_mset_replacement.py +++ b/tests/integration/test_mset_replacement.py @@ -35,10 +35,19 @@ def setUp(self): self.fixture_manager.copy_files_to_storage() self.fixture_manager.populate_metadatasets() + """ + Erweiterung der Tests auf die wesentlichen Szenarien, die auftreten können (Replacement of already replaced, non-existing replaced / replacing / etc) + """ + + @parameterized.expand([ - ("success", "group_x_admin", 200), + ("success", "group_x_admin", "mset_a", 200), + ("insufficient_access_rights", "user_a", "mset_a", 400), + ("insufficient_access_rights_other_group", "user_b", "mset_a", 400), + ("target_mset_does_not_exist", "group_x_admin", "blob", 400), + ("target_mset_already_replaced", "group_x_admin", "mset_a", 400) ]) - def test_mset_replacement(self, testname: str, executing_user: str, expected_status: int): + def test_mset_replacement(self, testname: str, executing_user: str, replaced_mset: str, expected_status: int): user = self.fixture_manager.get_fixture("users", executing_user) if executing_user else None auth_headers = self.apikey_auth(user) if user else {} @@ -51,7 +60,7 @@ def test_mset_replacement(self, testname: str, executing_user: str, expected_sta "FileR1": "group_x_file_2.txt" }, "replaces": [ - "mset_a" + replaced_mset ], "replacesLabel": "because i can", "fileIds": [ @@ -62,6 +71,25 @@ def test_mset_replacement(self, testname: str, executing_user: str, expected_sta self.testapp.post_json( url = f"{base_url}/rpc/replace-metadatasets", headers = auth_headers, - status = expected_status, + status = expected_status if testname != "target_mset_already_replaced" else 200, params = request_body ) + + if testname == "target_mset_already_replaced": + + request_body["record"].update({ + "ID": "blargh", + "FileR1": "group_x_file_3.txt", + "FileR2": "group_x_file_4.txt", + }) + request_body["fileIds"] = ["group_x_file_3", "group_x_file_4"] + + self.testapp.post_json( + url = f"{base_url}/rpc/replace-metadatasets", + headers = auth_headers, + status = expected_status, + params = request_body + ) + + + From d39538407148af682e1994e04daf8494478f13ad Mon Sep 17 00:00:00 2001 From: root Date: Wed, 28 Jul 2021 11:57:45 +0000 Subject: [PATCH 44/55] pleasing flake8 --- tests/integration/test_mset_replacement.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/integration/test_mset_replacement.py b/tests/integration/test_mset_replacement.py index 07f9e22d..b69aa964 100644 --- a/tests/integration/test_mset_replacement.py +++ b/tests/integration/test_mset_replacement.py @@ -35,11 +35,6 @@ def setUp(self): self.fixture_manager.copy_files_to_storage() self.fixture_manager.populate_metadatasets() - """ - Erweiterung der Tests auf die wesentlichen Szenarien, die auftreten können (Replacement of already replaced, non-existing replaced / replacing / etc) - """ - - @parameterized.expand([ ("success", "group_x_admin", "mset_a", 200), ("insufficient_access_rights", "user_a", "mset_a", 400), @@ -71,7 +66,7 @@ def test_mset_replacement(self, testname: str, executing_user: str, replaced_mse self.testapp.post_json( url = f"{base_url}/rpc/replace-metadatasets", headers = auth_headers, - status = expected_status if testname != "target_mset_already_replaced" else 200, + status = expected_status if testname != "target_mset_already_replaced" else 200, params = request_body ) @@ -90,6 +85,3 @@ def test_mset_replacement(self, testname: str, executing_user: str, replaced_mse status = expected_status, params = request_body ) - - - From b7cf1edd66d36064231bae34e95662be35fdf27b Mon Sep 17 00:00:00 2001 From: root Date: Wed, 28 Jul 2021 12:00:38 +0000 Subject: [PATCH 45/55] pleasing mypy --- tests/integration/test_mset_replacement.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_mset_replacement.py b/tests/integration/test_mset_replacement.py index b69aa964..24feaada 100644 --- a/tests/integration/test_mset_replacement.py +++ b/tests/integration/test_mset_replacement.py @@ -72,11 +72,9 @@ def test_mset_replacement(self, testname: str, executing_user: str, replaced_mse if testname == "target_mset_already_replaced": - request_body["record"].update({ - "ID": "blargh", - "FileR1": "group_x_file_3.txt", - "FileR2": "group_x_file_4.txt", - }) + request_body["record"]["ID"] = "blargh" + request_body["record"]["FileR1"] = "group_x_file_3.txt" + request_body["record"]["FileR2"] = "group_x_file_4.txt" request_body["fileIds"] = ["group_x_file_3", "group_x_file_4"] self.testapp.post_json( From e47a56d0f40324e988294224385ac89cc3367b71 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 28 Jul 2021 12:04:28 +0000 Subject: [PATCH 46/55] pleasing mypy --- tests/integration/test_mset_replacement.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_mset_replacement.py b/tests/integration/test_mset_replacement.py index 24feaada..c2e92154 100644 --- a/tests/integration/test_mset_replacement.py +++ b/tests/integration/test_mset_replacement.py @@ -72,10 +72,22 @@ def test_mset_replacement(self, testname: str, executing_user: str, replaced_mse if testname == "target_mset_already_replaced": - request_body["record"]["ID"] = "blargh" - request_body["record"]["FileR1"] = "group_x_file_3.txt" - request_body["record"]["FileR2"] = "group_x_file_4.txt" - request_body["fileIds"] = ["group_x_file_3", "group_x_file_4"] + request_body = { + "record": { + "Date": "2021-07-01", + "ZIP Code": "108", + "ID": "blargh", + "FileR2": "group_x_file_3.txt", + "FileR1": "group_x_file_4.txt" + }, + "replaces": [ + replaced_mset + ], + "replacesLabel": "because i can", + "fileIds": [ + "group_x_file_3", "group_x_file_4" + ] + } self.testapp.post_json( url = f"{base_url}/rpc/replace-metadatasets", From b87e481f0da5a5188efb2407ad36af5f2fe84b5e Mon Sep 17 00:00:00 2001 From: root Date: Wed, 28 Jul 2021 15:09:56 +0000 Subject: [PATCH 47/55] removed comment --- datameta/api/metadatasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index c07ac63f..530a6b19 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -234,7 +234,7 @@ def execute_mset_replacement(db, new_mset_id, replaced_msets, user_id): if missing_msets: messages, entities = zip(*missing_msets) - raise errors.get_validation_error(messages=list(messages), entities=list(entities)) # @lkuchenb: any idea why these list-casts are necessary to please mypy? they're not required in e.g validation.py:63 + raise errors.get_validation_error(messages=list(messages), entities=list(entities)) already_replaced = [ (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset).get('site')}", target_mset) From 455ee05f86ff8dd9877b4c7c8942d8a55384dfb5 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 28 Jul 2021 15:10:29 +0000 Subject: [PATCH 48/55] added information about replaced metadatasets to datatable --- datameta/api/ui/view.py | 1 + datameta/static/js/view.js | 52 +++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/datameta/api/ui/view.py b/datameta/api/ui/view.py index 6812edb4..a574b139 100644 --- a/datameta/api/ui/view.py +++ b/datameta/api/ui/view.py @@ -224,6 +224,7 @@ def post(request: Request): submission_datetime = mdata_set.submission.date.isoformat(), submission_label = mdata_set.submission.label, service_executions = service_executions, + replaced_by = get_identifier(mdata_set.replaced_via_event.new_metadataset) if mdata_set.replaced_via_event else None ) for (mdata_set, _), service_executions in zip(mdata_sets, service_executions_all) ] diff --git a/datameta/static/js/view.js b/datameta/static/js/view.js index 6f69fe24..293b3cc1 100644 --- a/datameta/static/js/view.js +++ b/datameta/static/js/view.js @@ -23,25 +23,32 @@ DataMeta.view.buildColumns = function(mdata) { return { title : mdatum.serviceId === null ? mdatum.name : ' '+mdatum.name, data : null, - render : function(mdataset, type, row, meta) { + render : function(mdataset, type, row, meta) { // We don't have access if (!(mdatum.name in mdataset.record)) return ''; // Special case NULL and service metadatum with access but not run yet if (mdataset.serviceExecutions !== null && mdatum.name in mdataset.serviceExecutions - && mdataset.serviceExecutions[mdatum.name]===null) + && mdataset.serviceExecutions[mdatum.name] === null) return ' pending'; // Special case NULL if (mdataset.record[mdatum.name] === null) return 'empty'; - // Speical case file - if (mdataset.fileIds[mdatum.name]) return ' '+mdataset.record[mdatum.name]+''; + // Special case file + if (mdataset.fileIds[mdatum.name]) { + var record_str = mdataset.replacedBy === null ? mdataset.record[mdatum.name] : '' + mdataset.record[mdatum.name] + ''; + return '' + record_str + ''; + } // All other cases - return mdataset.record[mdatum.name]; + return mdataset.replacedBy === null ? mdataset.record[mdatum.name] : '' + mdataset.record[mdatum.name] + ''; } }; }); } +function format_cell(data, is_replaced) { + return is_replaced ? '
' + data + '
' : '
' + '
' + data + '
' +} + DataMeta.view.initTable = function() { // Fetch metadata information from the API @@ -57,23 +64,44 @@ DataMeta.view.initTable = function() { // Extract field names from the metadata information var mdata = json; + + //var cell_style_start = data.replacedBy === null ? '
' : '
'; + //var cell_style_end = data.replacedBy === null ? '
' : '
'; + var cell_style_start = ''; + var cell_style_end = ''; + + + var columns = [ { title: ' Submission', data: null, className: "id_col", render: function(data) { - var label = data.submissionLabel ? data.submissionLabel : 'empty'; - return '
' + label + '
' + data.submissionId.site + '
' + var label = data.submissionLabel ? data.submissionLabel : 'empty'; //'empty'; + return '
' + format_cell(label, data.replacedBy !== null) + '
' + data.submissionId.site + '
'; }}, { title: ' Submission Time', data: null, className: "id_col", render: function(data) { var d = moment(new Date(data.submissionDatetime)); - return '
' + d.format('YYYY-MM-DD HH:mm:ss') + - '
' + 'GMT' + d.format('ZZ') + '
' + return '
' + format_cell(d.format('YYYY-MM-DD HH:mm:ss'), data.replacedBy !== null) + '
' + 'GMT' + d.format('ZZ') + '
'; }}, { title: ' User', data: null, className: "id_col", render: data => - '
'+data.userName+'
'+data.userId.site+'
' + '
' + format_cell(data.userName, data.replacedBy !== null) + '
' + data.userId.site + '
' }, { title: ' Group', data: null, className: "id_col", render: data => - '
'+data.groupName+'
'+data.groupId.site+'
' + '
' + format_cell(data.groupName, data.replacedBy !== null) + '
' + data.groupId.site + '
' }, - { title: ' ID', data: "id.site", className: "id_col", render: data => '' + data + ''} + { title: ' ID', data: null, className: "id_col", render: function(data) { + + var return_str = format_cell(data.id.site, data.replacedBy !== null); + return '
' + return_str + (data.replacedBy === null ? '' : '
replaced by ' + data.replacedBy.site + '
') + '
'; + if (data.replacedBy !== null) { + return_str += '
' + data.id.site + '
'; + return_str += '
replaced by ' + data.replacedBy.site + '
'; + } else { + return_str += '
' + data.id.site + '
'; + } + + return return_str + '' + + } + }, ].concat(DataMeta.view.buildColumns(mdata)) // Build table based on field names From 27163f5e4864833a0d2da013fee03f3f8b48534c Mon Sep 17 00:00:00 2001 From: Christian Schudoma Date: Tue, 3 Aug 2021 22:33:46 +0200 Subject: [PATCH 49/55] Apply suggestions from @lkuchenb's code review Co-authored-by: Leon Kuchenbecker --- datameta/api/metadatasets.py | 5 ++--- datameta/api/openapi.yaml | 3 +-- datameta/api/users.py | 2 +- datameta/models/db.py | 2 +- datameta/static/js/view.js | 4 ++-- tests/integration/test_bulkdelete.py | 2 +- 6 files changed, 8 insertions(+), 10 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 530a6b19..4137944d 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -230,11 +230,10 @@ def execute_mset_replacement(db, new_mset_id, replaced_msets, user_id): for mset_id in replaced_msets ] - missing_msets = [("Invalid metadataset id.", target_mset) for mset_id, target_mset in msets if target_mset is None] + missing_msets = [ f"Invalid metadataset id: {mset_id}" for mset_id, target_mset in msets if target_mset is None] if missing_msets: - messages, entities = zip(*missing_msets) - raise errors.get_validation_error(messages=list(messages), entities=list(entities)) + raise errors.get_validation_error(messages=missing_msets) already_replaced = [ (f"Metadataset already replaced by {get_identifier(target_mset.replaced_via_event.new_metadataset).get('site')}", target_mset) diff --git a/datameta/api/openapi.yaml b/datameta/api/openapi.yaml index 90462f5b..870bdc29 100644 --- a/datameta/api/openapi.yaml +++ b/datameta/api/openapi.yaml @@ -156,7 +156,6 @@ paths: '500': description: Internal Server Error - /rpc/delete-metadatasets: post: summary: Bulk-delete Staged MetaDataSets @@ -217,7 +216,7 @@ paths: '403': description: Forbidden '404': - description: File not found + description: Not found '400': description: Validation Error content: diff --git a/datameta/api/users.py b/datameta/api/users.py index 495a41a0..e3c26e1b 100644 --- a/datameta/api/users.py +++ b/datameta/api/users.py @@ -62,7 +62,7 @@ def from_user(cls, target_user, requesting_user): if authz.view_restricted_user_info(requesting_user, target_user): restricted_fields.update({ field: getattr(target_user, field) - for field in UserResponseElement.get_restricted_fields() + for field in cls.get_restricted_fields() }) return cls(id=get_identifier(target_user), name=target_user.fullname, group_id=get_identifier(target_user.group), **restricted_fields) diff --git a/datameta/models/db.py b/datameta/models/db.py index 3f1894f8..9e89c614 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -231,7 +231,7 @@ class MetaDataSet(Base): class MsetReplacementEvent(Base): """ Stores information about an mset replacement event """ - __tablename__ = 'msetreplacements' + __tablename__ = 'msetreplacementevents' id = Column(Integer, primary_key=True) uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) user_id = Column(Integer, ForeignKey('users.id'), nullable=False) diff --git a/datameta/static/js/view.js b/datameta/static/js/view.js index 293b3cc1..277b387d 100644 --- a/datameta/static/js/view.js +++ b/datameta/static/js/view.js @@ -23,7 +23,7 @@ DataMeta.view.buildColumns = function(mdata) { return { title : mdatum.serviceId === null ? mdatum.name : ' '+mdatum.name, data : null, - render : function(mdataset, type, row, meta) { + render : function(mdataset, type, row, meta) { // We don't have access if (!(mdatum.name in mdataset.record)) return ''; // Special case NULL and service metadatum with access but not run yet @@ -36,7 +36,7 @@ DataMeta.view.buildColumns = function(mdata) { // Special case file if (mdataset.fileIds[mdatum.name]) { var record_str = mdataset.replacedBy === null ? mdataset.record[mdatum.name] : '' + mdataset.record[mdatum.name] + ''; - return '' + record_str + ''; + return ' ' + record_str + ''; } // All other cases return mdataset.replacedBy === null ? mdataset.record[mdatum.name] : '' + mdataset.record[mdatum.name] + ''; diff --git a/tests/integration/test_bulkdelete.py b/tests/integration/test_bulkdelete.py index 211eeb0c..7a2de085 100644 --- a/tests/integration/test_bulkdelete.py +++ b/tests/integration/test_bulkdelete.py @@ -20,7 +20,7 @@ def test_file_deletion(self): auth_headers = self.apikey_auth(user) files = self.fixture_manager.get_fixtureset('files_independent') - file_ids = [ file.site_id for file in files.values() if not file.site_id.startswith("group") ] + file_ids = ["user_a_file_1", "user_a_file_2"] self.testapp.post_json( f"{base_url}/rpc/delete-files", From 306cf353e9ffc7f5d4272c3d942c9116a394610a Mon Sep 17 00:00:00 2001 From: root Date: Tue, 3 Aug 2021 22:03:22 +0000 Subject: [PATCH 50/55] implemented suggestions from @lkuchenb's code review --- datameta/api/metadatasets.py | 42 +++++++++++++++++++++++++++--------- datameta/api/ui/view.py | 2 ++ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 530a6b19..e05efc84 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -19,7 +19,7 @@ from pyramid.request import Request from sqlalchemy.orm import joinedload from sqlalchemy import and_ -from typing import Optional, Dict, List, Any, Tuple +from typing import Optional, Dict, List, Any from ..linting import validate_metadataset_record from .. import security, siteid, resource, validation from ..models import MetaDatum, MetaDataSet, ServiceExecution, Service, MetaDatumRecord, Submission, File @@ -52,14 +52,15 @@ class ReplacementMsetResponse(DataHolderBase): service_executions : Optional[Dict[str, Optional[MetaDataSetServiceExecution]]] = None @classmethod - def from_metadataset(cls, metadataset: MetaDataSet, replaces: List[Tuple[Any, MetaDataSet]], metadata_with_access: Dict[str, MetaDatum]): + def from_metadataset(cls, metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): + replaces = [get_identifier(mset) for event in metadataset.replaces_via_event for mset in event.replaced_metadatasets] if metadataset.replaces_via_event else None return cls( id = get_identifier(metadataset), record = get_record_from_metadataset(metadataset, metadata_with_access), file_ids = get_mset_associated_files(metadataset, metadata_with_access), user_id = get_identifier(metadataset.user), - replaces = [get_identifier(mset) for _, mset in replaces], + replaces = replaces, submission_id = get_identifier(metadataset.submission) if metadataset.submission else None, service_executions = collect_service_executions(metadata_with_access, metadataset), ) @@ -171,30 +172,41 @@ def delete_metadatasets(request: Request) -> HTTPNoContent: def initialize_service_metadata(db, mset_id): + """ + For the specified metadataset, set initial metadatumrecords + for all service-provided metadata fields. + """ service_metadata = get_service_metadata(db) for s_mdatum in service_metadata.values(): - mdatum_rec = MetaDatumRecord( + _ = MetaDatumRecord( metadatum_id = s_mdatum.id, metadataset_id = mset_id, file_id = None, value = None ) - db.add(mdatum_rec) + # db.add(mdatum_rec) def add_metadata_from_request(db, record, metadata, mset_id): - # Add the non-service metadata as specified in the request body + """ + For the specified metadataset, set non-service metadatum records + as supplied in the request body. + """ for name, value in record.items(): - mdatum_rec = MetaDatumRecord( + _ = MetaDatumRecord( metadatum_id = metadata[name].id, metadataset_id = mset_id, file_id = None, value = value ) - db.add(mdatum_rec) + # db.add(mdatum_rec) def validate_associated_files(db, file_ids, auth_user): + """ + Check that a list of files is accessible by an authenticated user. + """ + # Collect files, drop duplicates db_files = { file_id : resource.resource_query_by_id(db, File, file_id).options(joinedload(File.metadatumrecord)).one_or_none() for file_id in set(file_ids) } @@ -205,6 +217,10 @@ def validate_associated_files(db, file_ids, auth_user): def link_files(db, mdata_set, db_files, ignore_submitted=False): + """ + Perform association checks between a specified metadataset and list of files + and, upon success, link the files to the metadataset. + """ # Validate the associations between files and records fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { mdata_set.site_id : mdata_set }, ignore_submitted_metadatasets=ignore_submitted) @@ -224,6 +240,12 @@ def link_files(db, mdata_set, db_files, ignore_submitted=False): def execute_mset_replacement(db, new_mset_id, replaced_msets, user_id): + """ + Evaluate if a replacement of a list of metadatasets is possible + - validate target metadataset ids + - ensure target metadatasets have not already been replaced + Upon success, generate a replacement event and mark the target datasets as replaced. + """ msets = [ (mset_id, resource_by_id(db, MetaDataSet, mset_id)) @@ -302,7 +324,7 @@ def replace_metadatasets(request: Request) -> SubmissionResponse: replaces = request.openapi_validated.body["replaces"] replaces_label = request.openapi_validated.body["replacesLabel"] - replaced_msets = execute_mset_replacement(db, mdata_set.id, replaces, auth_user.id) + execute_mset_replacement(db, mdata_set.id, replaces, auth_user.id) initialize_service_metadata(db, mdata_set.id) @@ -330,7 +352,7 @@ def replace_metadatasets(request: Request) -> SubmissionResponse: # Check which metadata of this metadataset the user is allowed to view metadata_with_access = get_metadata_with_access(db, auth_user) - return ReplacementMsetResponse.from_metadataset(mdata_set, replaced_msets, metadata_with_access) + return ReplacementMsetResponse.from_metadataset(mdata_set, metadata_with_access) @view_config( diff --git a/datameta/api/ui/view.py b/datameta/api/ui/view.py index a574b139..18ba3fd8 100644 --- a/datameta/api/ui/view.py +++ b/datameta/api/ui/view.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datameta.models.db import MsetReplacementEvent from sqlalchemy.orm import joinedload, aliased from sqlalchemy import func, and_, or_, desc, asc @@ -182,6 +183,7 @@ def post(request: Request): .options(joinedload(MetaDataSet.user))\ .options(joinedload(MetaDataSet.service_executions).joinedload(ServiceExecution.user))\ .options(joinedload(MetaDataSet.service_executions).joinedload(ServiceExecution.service).joinedload(Service.target_metadata))\ + .options(joinedload(MetaDataSet.replaced_via_event).joinedload(MsetReplacementEvent.new_metadataset))\ .offset(start)\ .limit(length)\ From 455626a11faf5abaf200a4f2fd2145ca816983c2 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 3 Aug 2021 22:13:46 +0000 Subject: [PATCH 51/55] dealt with fallout from renaming msetreplacements-table --- .../alembic/versions/20210625_f6a70402119f.py | 16 ++++++++-------- datameta/models/db.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/datameta/alembic/versions/20210625_f6a70402119f.py b/datameta/alembic/versions/20210625_f6a70402119f.py index 5f05a0d4..6d6ec4ff 100644 --- a/datameta/alembic/versions/20210625_f6a70402119f.py +++ b/datameta/alembic/versions/20210625_f6a70402119f.py @@ -19,20 +19,20 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table( - 'msetreplacements', + 'msetreplacementevents', sa.Column('id', sa.Integer(), nullable=False), sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False), sa.Column('user_id', sa.Integer(), nullable=False), sa.Column('datetime', sa.DateTime(), nullable=False), sa.Column('new_metadataset_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_new_metadataset_id_metadatasets')), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')), - sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')), - sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid')) + sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacementevents_new_metadataset_id_metadatasets')), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacementevents_user_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacementevents')), + sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacementevents_uuid')) ) op.add_column('metadatasets', sa.Column('replaced_via_event_id', sa.Integer(), nullable=True)) op.drop_constraint('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', type_='foreignkey') - op.create_foreign_key(op.f('fk_metadatasets_replaced_via_event_id_msetreplacements'), 'metadatasets', 'msetreplacements', ['replaced_via_event_id'], ['id'], use_alter=True) + op.create_foreign_key(op.f('fk_metadatasets_replaced_via_event_id_msetreplacementevents'), 'metadatasets', 'msetreplacementevents', ['replaced_via_event_id'], ['id'], use_alter=True) op.drop_column('metadatasets', 'deprecated_label') op.drop_column('metadatasets', 'replaced_by_id') op.drop_column('metadatasets', 'is_deprecated') @@ -48,8 +48,8 @@ def downgrade(): op.add_column('metadatasets', sa.Column('is_deprecated', sa.BOOLEAN(), autoincrement=False, nullable=True)) op.add_column('metadatasets', sa.Column('replaced_by_id', sa.INTEGER(), autoincrement=False, nullable=True)) op.add_column('metadatasets', sa.Column('deprecated_label', sa.VARCHAR(), autoincrement=False, nullable=True)) - op.drop_constraint(op.f('fk_metadatasets_replaced_via_event_id_msetreplacements'), 'metadatasets', type_='foreignkey') + op.drop_constraint(op.f('fk_metadatasets_replaced_via_event_id_msetreplacementevents'), 'metadatasets', type_='foreignkey') op.create_foreign_key('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', 'metadatasets', ['replaced_by_id'], ['id']) op.drop_column('metadatasets', 'replaced_via_event_id') - op.drop_table('msetreplacements') + op.drop_table('msetreplacementevents') # ### end Alembic commands ### diff --git a/datameta/models/db.py b/datameta/models/db.py index 9e89c614..a97ab656 100644 --- a/datameta/models/db.py +++ b/datameta/models/db.py @@ -217,7 +217,7 @@ class MetaDataSet(Base): uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False) user_id = Column(Integer, ForeignKey('users.id'), nullable=False) submission_id = Column(Integer, ForeignKey('submissions.id'), nullable=True) - replaced_via_event_id = Column(Integer, ForeignKey('msetreplacements.id', use_alter=True), nullable=True) + replaced_via_event_id = Column(Integer, ForeignKey('msetreplacementevents.id', use_alter=True), nullable=True) # Relationships user = relationship('User', back_populates='metadatasets') From 00738a1d313838443e0bff80c9a646f81cca8287 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 4 Aug 2021 06:49:25 +0000 Subject: [PATCH 52/55] fixed bulk deletion test --- tests/integration/test_bulkdelete.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/test_bulkdelete.py b/tests/integration/test_bulkdelete.py index 7a2de085..6372cacc 100644 --- a/tests/integration/test_bulkdelete.py +++ b/tests/integration/test_bulkdelete.py @@ -18,8 +18,7 @@ def test_file_deletion(self): user = self.fixture_manager.get_fixture('users', 'user_a') auth_headers = self.apikey_auth(user) - files = self.fixture_manager.get_fixtureset('files_independent') - + file_ids = ["user_a_file_1", "user_a_file_2"] self.testapp.post_json( From ea308547692974181f2641ab4e24df8df181b2c2 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 4 Aug 2021 08:16:56 +0000 Subject: [PATCH 53/55] fixed failing stage_and_submit test --- datameta/api/metadatasets.py | 10 +++++----- tests/integration/test_bulkdelete.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index dab5a497..6e97628c 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -47,7 +47,7 @@ class ReplacementMsetResponse(DataHolderBase): record : Dict[str, Optional[str]] file_ids : Dict[str, Optional[Dict[str, str]]] user_id : str - replaces : List[Dict[str, Any]] + replaces : List[Any] submission_id : Optional[str] = None service_executions : Optional[Dict[str, Optional[MetaDataSetServiceExecution]]] = None @@ -178,13 +178,13 @@ def initialize_service_metadata(db, mset_id): """ service_metadata = get_service_metadata(db) for s_mdatum in service_metadata.values(): - _ = MetaDatumRecord( + mdatum_rec = MetaDatumRecord( metadatum_id = s_mdatum.id, metadataset_id = mset_id, file_id = None, value = None ) - # db.add(mdatum_rec) + db.add(mdatum_rec) def add_metadata_from_request(db, record, metadata, mset_id): @@ -193,13 +193,13 @@ def add_metadata_from_request(db, record, metadata, mset_id): as supplied in the request body. """ for name, value in record.items(): - _ = MetaDatumRecord( + mdatum_rec = MetaDatumRecord( metadatum_id = metadata[name].id, metadataset_id = mset_id, file_id = None, value = value ) - # db.add(mdatum_rec) + db.add(mdatum_rec) def validate_associated_files(db, file_ids, auth_user): diff --git a/tests/integration/test_bulkdelete.py b/tests/integration/test_bulkdelete.py index 6372cacc..dfb49cdc 100644 --- a/tests/integration/test_bulkdelete.py +++ b/tests/integration/test_bulkdelete.py @@ -18,7 +18,7 @@ def test_file_deletion(self): user = self.fixture_manager.get_fixture('users', 'user_a') auth_headers = self.apikey_auth(user) - + file_ids = ["user_a_file_1", "user_a_file_2"] self.testapp.post_json( From cbe39ebad1a2439005f67018090415047b643ac1 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 4 Aug 2021 08:19:48 +0000 Subject: [PATCH 54/55] pleasing mypy --- datameta/api/metadatasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datameta/api/metadatasets.py b/datameta/api/metadatasets.py index 6e97628c..a0f2f6e1 100644 --- a/datameta/api/metadatasets.py +++ b/datameta/api/metadatasets.py @@ -54,7 +54,7 @@ class ReplacementMsetResponse(DataHolderBase): @classmethod def from_metadataset(cls, metadataset: MetaDataSet, metadata_with_access: Dict[str, MetaDatum]): - replaces = [get_identifier(mset) for event in metadataset.replaces_via_event for mset in event.replaced_metadatasets] if metadataset.replaces_via_event else None + replaces = [get_identifier(mset) for event in metadataset.replaces_via_event for mset in event.replaced_metadatasets] # if metadataset.replaces_via_event else None return cls( id = get_identifier(metadataset), record = get_record_from_metadataset(metadataset, metadata_with_access), From dd036ccb2176f3ab2446449a4245df30d6bdf3ba Mon Sep 17 00:00:00 2001 From: root Date: Tue, 24 Aug 2021 09:51:49 +0000 Subject: [PATCH 55/55] fixed issue that prevented new user registration requests to be completed --- datameta/api/ui/admin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datameta/api/ui/admin.py b/datameta/api/ui/admin.py index 18e83edd..9cb005ea 100644 --- a/datameta/api/ui/admin.py +++ b/datameta/api/ui/admin.py @@ -86,6 +86,7 @@ def v_admin_put_request(request): enabled = True, site_admin = False, site_read = False, + can_update = False, group_admin = newuser_make_admin, pwhash = '!') try: