Feature/adding mset replacement functionality 20210615

datameta/alembic/versions/20210625_f6a70402119f.py

@@ -0,0 +1,54 @@
+"""added msetreplacements table, moved update-related fields from mset, added user.can_update flag
+
+Revision ID: f6a70402119f
+Revises: 7fdc829db18d
+Create Date: 2021-06-25 15:27:37.638601
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'f6a70402119f'
+down_revision = '7fdc829db18d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'msetreplacements',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('user_id', sa.Integer(), nullable=False),
+        sa.Column('datetime', sa.DateTime(), nullable=False),
+        sa.Column('label', sa.String(length=140), nullable=False),
+        sa.Column('new_metadataset_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['new_metadataset_id'], ['metadatasets.id'], name=op.f('fk_msetreplacements_new_metadataset_id_metadatasets')),
+        sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_msetreplacements_user_id_users')),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_msetreplacements')),
+        sa.UniqueConstraint('uuid', name=op.f('uq_msetreplacements_uuid'))
+    )
+    op.add_column('metadatasets', sa.Column('replaced_via_event_id', sa.Integer(), nullable=True))
+    op.drop_constraint('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', type_='foreignkey')
+    op.create_foreign_key(op.f('fk_metadatasets_replaced_via_event_id_msetreplacements'), 'metadatasets', 'msetreplacements', ['replaced_via_event_id'], ['id'], use_alter=True)
+    op.drop_column('metadatasets', 'deprecated_label')
+    op.drop_column('metadatasets', 'replaced_by_id')
+    op.drop_column('metadatasets', 'is_deprecated')
+    op.add_column('users', sa.Column('can_update', sa.Boolean(create_constraint=False), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('users', 'can_update')
+    op.add_column('metadatasets', sa.Column('is_deprecated', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    op.add_column('metadatasets', sa.Column('replaced_by_id', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.add_column('metadatasets', sa.Column('deprecated_label', sa.VARCHAR(), autoincrement=False, nullable=True))
+    op.drop_constraint(op.f('fk_metadatasets_replaced_via_event_id_msetreplacements'), 'metadatasets', type_='foreignkey')
+    op.create_foreign_key('fk_metadatasets_replaced_by_id_metadatasets', 'metadatasets', 'metadatasets', ['replaced_by_id'], ['id'])
+    op.drop_column('metadatasets', 'replaced_via_event_id')
+    op.drop_table('msetreplacements')
+    # ### end Alembic commands ###

datameta/api/metadatasets.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from dataclasses import dataclass
+from datameta.models.db import MsetReplacementEvent
 from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound, HTTPNoContent
 from pyramid.view import view_config
 from pyramid.request import Request
@@ -23,8 +24,7 @@
 from .. import security, siteid, resource, validation
 from ..models import MetaDatum, MetaDataSet, ServiceExecution, Service, MetaDatumRecord, Submission, File
 from ..security import authz
-import datetime
-from datetime import timezone
+from datetime import timezone, datetime
 from ..resource import resource_by_id, resource_query_by_id, get_identifier
 from ..utils import get_record_from_metadataset
 from . import DataHolderBase
@@ -88,7 +88,7 @@ def render_record_values(metadata: Dict[str, MetaDatum], record: dict) -> dict:
             continue
         elif record_rendered[field] and metadata[field].datetimefmt:
             # if MetaDatum is a datetime field, render the value in isoformat
-            record_rendered[field] = datetime.datetime.strptime(
+            record_rendered[field] = datetime.strptime(
                     record_rendered[field],
                     metadata[field].datetimefmt
                     ).isoformat()
@@ -150,6 +150,14 @@ def post(request: Request) -> MetaDataSetResponse:
     # Obtain string converted version of the record
     record = record_to_strings(request.openapi_validated.body["record"])
 
+    replaces = request.openapi_validated.body.get("replaces")
+    replaces_label = request.openapi_validated.body.get("replaces_label")
+
+    if replaces and not replaces_label:
+        raise errors.get_validation_error(messages=['No reason (label) given for Metadataset replacement.'])  # maybe label should be reason.
+    if not replaces and replaces_label:
+        raise errors.get_validation_error(messages=["No metadataset ids specified (replacement reason (label) is given."])
+
     # Query the configured metadata. We're only considering and allowing
     # non-service metadata when creating a new metadataset.
     metadata = get_all_metadata(db, include_service_metadata=False)
@@ -166,7 +174,31 @@ def post(request: Request) -> MetaDataSetResponse:
         user_id = auth_user.id,
         submission_id = None
     )
+
     db.add(mdata_set)
+
+    if replaces:
+        mset_repl_evt = MsetReplacementEvent(
+            user_id = auth_user.id,
+            datetime = datetime.utcnow(),
+            label = replaces_label,
+            new_metadataset_id = mdata_set.id
+        )
+        db.add(mset_repl_evt)
+
+        msets = [
+            (mset_id, resource_by_id(db, MetaDataSet, mset_id))
+            for mset_id in replaces
+        ]
+
+        missing_msets = [("Invalid metadataset id.", mset_id) for mset_id, target_mset in msets if target_mset is None]
+        if missing_msets:
+            messages, entities = zip(*missing_msets)
+            raise errors.get_validation_error(messages=messages, entities=entities)
+
+        for _, target_mset in msets:
+            target_mset.replaced_via_event_id = mset_repl_evt.id
+
     db.flush()
 
     # Add NULL values for service metadata
@@ -279,7 +311,7 @@ def get_metadatasets(request: Request) -> List[MetaDataSetResponse]:
         query = query.filter(Submission.date < submitted_before.astimezone(timezone.utc))
     if awaiting_service is not None:
         if awaiting_service not in readable_services_by_id:
-            raise errors.get_validation_error(messages=['Invalid service ID specified'], fields=['awaitingServices'])
+            raise errors.get_validation_error(messages=['Invalid service ID specified'], fields=['awaitingService'])
         query = query.outerjoin(ServiceExecution, and_(
             MetaDataSet.id == ServiceExecution.metadataset_id,
             ServiceExecution.service_id == readable_services_by_id[awaiting_service].id
@@ -444,7 +476,7 @@ def set_metadata_via_service(request: Request) -> MetaDataSetResponse:
             service       = service,
             metadataset   = metadataset,
             user          = auth_user,
-            datetime      = datetime.datetime.utcnow()
+            datetime      = datetime.utcnow()
             )
 
     db.add(sexec)

datameta/api/openapi.yaml

@@ -15,7 +15,7 @@
 openapi: 3.0.0
 info:
   description: DataMeta
-  version: 1.2.0
+  version: 1.3.0
   title: DataMeta
 
 servers:
@@ -563,7 +563,7 @@ paths:
             format: date-time
         - name: awaitingService
           in: query
-          description: Identifier for a service. Restricts the result to metadatsets for which the specified service has not been executed yet.
+          description: Identifier for a service. Restricts the result to metadatasets for which the specified service has not been executed yet.
           schema:
             type: string
       responses:
@@ -1441,6 +1441,12 @@ components:
           additionalProperties: true
           # a free-form object,
           # any property is allowed
+        replaces:
+          type: array
+          items:
+            type: string
+        replaces_label:
+          type: string
       required:
         - record
       additionalProperties: false
@@ -1730,6 +1736,8 @@ components:
           type: boolean
         siteRead:
           type: boolean
+        canUpdate:
+          type: boolean
         email:
           type: string
         group:
@@ -1745,6 +1753,7 @@ components:
         - groupAdmin
         - siteAdmin
         - siteRead
+        - canUpdate
         - email
         - group
       additionalProperties: false
@@ -1775,6 +1784,9 @@ components:
               $ref: "#/components/schemas/Identifier"
             name:
               type: string
+        canUpdate:
+          type: boolean
+          nullable: true
       required:
         - id
         - name
@@ -1796,6 +1808,8 @@ components:
           type: boolean
         enabled:
           type: boolean
+        canUpdate:
+          type: boolean
 
     MetaDataResponse:
       type: object

datameta/api/users.py

@@ -36,6 +36,7 @@ class UserUpdateRequest(DataHolderBase):
     siteAdmin: bool
     enabled: bool
     siteRead: bool
+    canUpdate: bool
 
 
 @dataclass
@@ -48,6 +49,7 @@ class UserResponseElement(DataHolderBase):
     site_admin: Optional[bool] = None
     site_read: Optional[bool] = None
     email: Optional[str] = None
+    can_update: Optional[bool] = None
 
     @classmethod
     def from_user(cls, target_user, requesting_user):
@@ -58,7 +60,8 @@ def from_user(cls, target_user, requesting_user):
                 "group_admin": target_user.group_admin,
                 "site_admin": target_user.site_admin,
                 "site_read": target_user.site_read,
-                "email": target_user.email
+                "email": target_user.email,
+                "can_update": target_user.can_update
             })
 
         return cls(id=get_identifier(target_user), name=target_user.fullname, group=get_identifier(target_user.group), **restricted_fields)
@@ -72,6 +75,7 @@ class WhoamiResponseElement(DataHolderBase):
     group_admin: bool
     site_admin: bool
     site_read: bool
+    can_update: bool
     email: str
     group: dict
 
@@ -92,6 +96,7 @@ def get_whoami(request: Request) -> WhoamiResponseElement:
         group_admin     =   auth_user.group_admin,
         site_admin      =   auth_user.site_admin,
         site_read       =   auth_user.site_read,
+        can_update      =   auth_user.can_update,
         email           =   auth_user.email,
         group           =   {"id": get_identifier(auth_user.group), "name": auth_user.group.name}
     )
@@ -138,6 +143,7 @@ def put(request: Request):
     site_admin = request.openapi_validated.body.get("siteAdmin")
     enabled = request.openapi_validated.body.get("enabled")
     site_read = request.openapi_validated.body.get("siteRead")
+    can_update = request.openapi_validated.body.get("canUpdate")
 
     db = request.dbsession
 
@@ -176,6 +182,10 @@ def put(request: Request):
     if name is not None and not authz.update_user_name(auth_user, target_user):
         raise HTTPForbidden()
 
+    # The user has to be site admin to change update privileges for a user
+    if can_update is not None and not authz.update_user_can_update(auth_user):
+        raise HTTPForbidden()
+
     # Now, make the corresponding changes
     if group_id is not None:
         new_group = resource_by_id(db, Group, group_id)
@@ -192,5 +202,7 @@ def put(request: Request):
         target_user.enabled = enabled
     if name is not None:
         target_user.fullname = name
+    if can_update is not None:
+        target_user.can_update = can_update
 
     return HTTPNoContent()

datameta/models/db.py

@@ -28,7 +28,7 @@
     Table
 )
 from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy.orm import relationship, backref
+from sqlalchemy.orm import relationship
 
 from .meta import Base
 
@@ -82,6 +82,8 @@ class User(Base):
     site_admin           = Column(Boolean(create_constraint=False), nullable=False)
     group_admin          = Column(Boolean(create_constraint=False), nullable=False)
     site_read            = Column(Boolean(create_constraint=False), nullable=False)
+    can_update           = Column(Boolean(create_constraint=False), nullable=False)
+
     # Relationships
     group                = relationship('Group', back_populates='user')
     metadatasets         = relationship('MetaDataSet', back_populates='user')
@@ -90,6 +92,7 @@ class User(Base):
     apikeys              = relationship('ApiKey', back_populates='user')
     services             = relationship('Service', secondary=user_service_table, back_populates='users')
     service_executions   = relationship('ServiceExecution', back_populates='user')
+    mset_replacements    = relationship('MsetReplacementEvent', back_populates='user')
 
 
 class ApiKey(Base):
@@ -209,20 +212,35 @@ class MetaDatumRecord(Base):
 class MetaDataSet(Base):
     """A MetaDataSet represents all metadata associated with *one* record"""
     __tablename__  = 'metadatasets'
-    id               = Column(Integer, primary_key=True)
-    site_id          = Column(String(50), unique=True, nullable=False, index=True)
-    uuid             = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False)
-    user_id          = Column(Integer, ForeignKey('users.id'), nullable=False)
-    submission_id    = Column(Integer, ForeignKey('submissions.id'), nullable=True)
-    is_deprecated    = Column(Boolean, default=False)
-    deprecated_label = Column(String, nullable=True)
-    replaced_by_id   = Column(Integer, ForeignKey('metadatasets.id'), nullable=True)
+    id                     = Column(Integer, primary_key=True)
+    site_id                = Column(String(50), unique=True, nullable=False, index=True)
+    uuid                   = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False)
+    user_id                = Column(Integer, ForeignKey('users.id'), nullable=False)
+    submission_id          = Column(Integer, ForeignKey('submissions.id'), nullable=True)
+    replaced_via_event_id  = Column(Integer, ForeignKey('msetreplacements.id', use_alter=True), nullable=True)
+
     # Relationships
     user                 = relationship('User', back_populates='metadatasets')
     submission           = relationship('Submission', back_populates='metadatasets')
     metadatumrecords     = relationship('MetaDatumRecord', back_populates='metadataset')
-    replaces             = relationship('MetaDataSet', backref=backref('replaced_by', remote_side=[id]))
-    service_executions   = relationship('ServiceExecution', back_populates = 'metadataset')
+    service_executions   = relationship('ServiceExecution', back_populates ='metadataset')
+
+    replaced_via_event   = relationship('MsetReplacementEvent', primaryjoin='MetaDataSet.replaced_via_event_id==MsetReplacementEvent.id')
+    replaces_via_event   = relationship('MsetReplacementEvent', primaryjoin='MetaDataSet.id==MsetReplacementEvent.new_metadataset_id')
+
+
+class MsetReplacementEvent(Base):
+    """ Stores information about an mset replacement event """
+    __tablename__ = 'msetreplacements'
+    id                   = Column(Integer, primary_key=True)
+    uuid                 = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4, nullable=False)
+    user_id              = Column(Integer, ForeignKey('users.id'), nullable=False)
+    datetime             = Column(DateTime, nullable=False)
+    label                = Column(String(140), nullable=False)
+    new_metadataset_id   = Column(Integer, ForeignKey('metadatasets.id'), nullable=False)
+
+    # Relationships
+    user                 = relationship('User', back_populates='mset_replacements')
 
 
 class ApplicationSetting(Base):
@@ -244,7 +262,7 @@ class Service(Base):
     site_id      = Column(String(50), unique=True, nullable=False, index=True)
     name         = Column(Text, nullable=True, unique=True)
     # Relationships
-    users           = relationship('User', secondary=user_service_table, back_populates='services')
+    users                = relationship('User', secondary=user_service_table, back_populates='services')
     # unfortunately, 'metadata' is a reserved keyword for sqlalchemy classes
     service_executions   = relationship('ServiceExecution', back_populates = 'service')
     target_metadata      = relationship('MetaDatum', back_populates = 'service')

datameta/security/authz.py

@@ -176,6 +176,10 @@ def update_user_name(user, target_user):
     ))
 
 
+def update_user_can_update(user):
+    return user.site_admin
+
+
 def view_restricted_user_info(user, target_user):
     return has_group_rights(user, target_user.group)
 

tests/integration/fixtures/holders.py

@@ -42,6 +42,7 @@ class UserFixture(Entity):
     site_admin    : bool
     site_read     : bool
     enabled       : bool
+    can_update    : bool
 
 
 @dataclass