From 324cad7f002afe77258d9d26cda34aa48df5b6ac Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Mon, 30 Jan 2023 12:48:05 +0200 Subject: [PATCH 1/4] TaskProtocol model --- alyx/actions/admin.py | 9 +++-- alyx/actions/models.py | 4 +- alyx/actions/serializers.py | 9 ++++- alyx/actions/tests_rest.py | 44 +++++++++++++++++++++- alyx/actions/views.py | 2 + alyx/experiments/models.py | 10 +++++ alyx/jobs/admin.py | 11 +++--- alyx/misc/management/commands/one_cache.py | 13 +++++-- 8 files changed, 86 insertions(+), 16 deletions(-) diff --git a/alyx/actions/admin.py b/alyx/actions/admin.py index 648d8ff8..3ec08a0a 100644 --- a/alyx/actions/admin.py +++ b/alyx/actions/admin.py @@ -477,10 +477,10 @@ def _pass_narrative_templates(context): class SessionAdmin(BaseActionAdmin): list_display = ['subject_l', 'start_time', 'number', 'lab', 'dataset_count', - 'task_protocol', 'qc', 'user_list', 'project_'] + 'task_protocol_', 'qc', 'user_list', 'project_'] list_display_links = ['start_time'] fields = BaseActionAdmin.fields + [ - 'repo_url', 'qc', 'extended_qc', 'projects', ('type', 'task_protocol', ), 'number', + 'repo_url', 'qc', 'extended_qc', 'projects', ('type', 'task_protocols', ), 'number', 'n_correct_trials', 'n_trials', 'weighing', 'auto_datetime'] list_filter = [('users', RelatedDropdownFilter), ('start_time', DateRangeFilter), @@ -488,7 +488,7 @@ class SessionAdmin(BaseActionAdmin): ('lab', RelatedDropdownFilter), ] search_fields = ('subject__nickname', 'lab__name', 'projects__name', 'users__username', - 'task_protocol', 'pk') + 'task_protocol__name', 'pk') ordering = ('-start_time', 'task_protocol', 'lab') inlines = [WaterAdminInline, DatasetInline, NoteInline] readonly_fields = ['repo_url', 'task_protocol', 'weighing', 'qc', 'extended_qc', @@ -520,6 +520,9 @@ def add_view(self, request, extra_context=None): def project_(self, obj): return [getattr(p, 'name', None) for p in obj.projects.all()] + def task_protocol_(self, obj): + return [getattr(p, 'name', None) for p in obj.task_protocols.all()] + def repo_url(self, obj): url = settings.SESSION_REPO_URL.format( lab=obj.subject.lab.name, diff --git a/alyx/actions/models.py b/alyx/actions/models.py index c9ff13c8..774c4a59 100644 --- a/alyx/actions/models.py +++ b/alyx/actions/models.py @@ -251,7 +251,9 @@ class Session(BaseAction): help_text="User-defined session type (e.g. Base, Experiment)") number = models.IntegerField(null=True, blank=True, help_text="Optional session number for this level") - task_protocol = models.CharField(max_length=1023, blank=True, default='') + task_protocol = models.CharField(max_length=1023, blank=True, default='old task protocol') + task_protocols = models.ManyToManyField('experiments.TaskProtocol', blank=True, + verbose_name='Session task protocols') n_trials = models.IntegerField(blank=True, null=True) n_correct_trials = models.IntegerField(blank=True, null=True) diff --git a/alyx/actions/serializers.py b/alyx/actions/serializers.py index 53899d0c..746abaaf 100644 --- a/alyx/actions/serializers.py +++ b/alyx/actions/serializers.py @@ -10,11 +10,12 @@ from data.models import Dataset, DatasetType from misc.models import LabLocation, Lab from experiments.serializers import ProbeInsertionListSerializer, FilterDatasetSerializer +from experiments.models import TaskProtocol from misc.serializers import NoteSerializer SESSION_FIELDS = ('subject', 'users', 'location', 'procedures', 'lab', 'projects', 'type', - 'task_protocol', 'number', 'start_time', 'end_time', 'narrative', + 'task_protocols', 'number', 'start_time', 'end_time', 'narrative', 'parent_session', 'n_correct_trials', 'n_trials', 'url', 'extended_qc', 'qc', 'wateradmin_session_related', 'data_dataset_session_related', 'auto_datetime') @@ -127,12 +128,13 @@ def setup_eager_loading(queryset): """ Perform necessary eager loading of data to avoid horrible performance.""" queryset = queryset.select_related('subject', 'lab') queryset = queryset.prefetch_related('projects') + queryset = queryset.prefetch_related('task_protocols') return queryset.order_by('-start_time') class Meta: model = Session fields = ('id', 'subject', 'start_time', 'number', 'lab', 'projects', 'url', - 'task_protocol') + 'task_protocols') class SessionDetailSerializer(BaseActionSerializer): @@ -142,6 +144,9 @@ class SessionDetailSerializer(BaseActionSerializer): probe_insertion = ProbeInsertionListSerializer(read_only=True, many=True) projects = serializers.SlugRelatedField(read_only=False, slug_field='name', many=True, queryset=Project.objects.all(), required=False) + task_protocols = serializers.SlugRelatedField( + read_only=False, slug_field='name', many=True, + queryset=TaskProtocol.objects.all(), required=False) notes = NoteSerializer(read_only=True, many=True) qc = BaseSerializerEnumField(required=False) diff --git a/alyx/actions/tests_rest.py b/alyx/actions/tests_rest.py index f7c56a64..a98bca1b 100644 --- a/alyx/actions/tests_rest.py +++ b/alyx/actions/tests_rest.py @@ -6,6 +6,7 @@ from alyx import base from alyx.base import BaseTests from subjects.models import Subject, Project +from experiments.models import TaskProtocol from misc.models import Lab, Note, ContentType from actions.models import Session, WaterType, WaterAdministration @@ -21,6 +22,8 @@ def setUp(self): self.lab02 = Lab.objects.create(name='awesomelab') self.projectX = Project.objects.create(name='projectX') self.projectY = Project.objects.create(name='projectY') + self.protocolX = TaskProtocol.objects.create(name='ephysChoiceWorld') + self.protocolY = TaskProtocol.objects.create(name='passiveChoiceWorld') # Set an implant weight. self.subject.implant_weight = 4.56 self.subject.save() @@ -187,6 +190,45 @@ def test_sessions_projects(self): d = self.ar(self.client.get(reverse('session-list') + f'?projects={self.projectY.name}')) self.assertEqual(len(d), 1) + def test_sessions_protocols(self): + ses1dict = {'subject': self.subject.nickname, + 'users': [self.superuser.username], + 'projects': [self.projectX.name], + 'start_time': '2020-07-09T12:34:56', + 'end_time': '2020-07-09T12:34:57', + 'type': 'Base', + 'number': '1', + 'lab': self.lab01.name, + 'task_protocol': [self.protocolX] + } + ses2dict = {'subject': self.subject.nickname, + 'users': [self.superuser.username, self.superuser2.username], + 'projects': [self.projectX.name], + 'start_time': '2020-07-09T12:34:56', + 'end_time': '2020-07-09T12:34:57', + 'type': 'Base', + 'number': '2', + 'lab': self.lab01.name, + 'task_protocol': [self.protocolX, self.protocolY] + } + self.ar(self.post(reverse('session-list'), data=ses1dict), 201) + self.ar(self.post(reverse('session-list'), data=ses2dict), 201) + # Test the user filter, this should return 2 sessions + q = f'?task_protocols={self.protocolX.name}' + d = self.ar(self.client.get(reverse('session-list') + q)) + self.assertEqual(len(d), 2) + # This should return only one session + q = f'?task_protocols={self.protocolY.name}' + d = self.ar(self.client.get(reverse('session-list') + q)) + self.assertEqual(len(d), 1) + # test the legacy filter that should act in the same way + q = f'?task_protocol={self.protocolX.name}' + d = self.ar(self.client.get(reverse('session-list') + q)) + self.assertEqual(len(d), 2) + q = f'?task_protocols={self.protocolY.name}' + d = self.ar(self.client.get(reverse('session-list') + q)) + self.assertEqual(len(d), 1) + def test_sessions(self): a_dict4json = {'String': 'this is not a JSON', 'Integer': 4, 'List': ['titi', 4]} ses_dict = {'subject': self.subject.nickname, @@ -201,7 +243,7 @@ def test_sessions(self): 'lab': self.lab01.name, 'n_trials': 100, 'n_correct_trials': 75, - 'task_protocol': self.test_protocol, + 'task_protocol': [self.protocolX], 'json': a_dict4json} # Test the session creation r = self.post(reverse('session-list'), data=ses_dict) diff --git a/alyx/actions/views.py b/alyx/actions/views.py index 60fcf153..030c6303 100644 --- a/alyx/actions/views.py +++ b/alyx/actions/views.py @@ -198,6 +198,8 @@ class SessionFilter(BaseFilterSet): date_range = django_filters.CharFilter(field_name='date_range', method=('filter_date_range')) type = django_filters.CharFilter(field_name='type', lookup_expr=('iexact')) lab = django_filters.CharFilter(field_name='lab__name', lookup_expr=('iexact')) + task_protocols = django_filters.CharFilter(field_name='task_protocols__name', + lookup_expr=('icontains')) task_protocol = django_filters.CharFilter(field_name='task_protocol', lookup_expr=('icontains')) qc = django_filters.CharFilter(method='enum_field_filter') diff --git a/alyx/experiments/models.py b/alyx/experiments/models.py index eb6b9be6..9dd2c04d 100644 --- a/alyx/experiments/models.py +++ b/alyx/experiments/models.py @@ -235,3 +235,13 @@ class Meta: def save(self, *args, **kwargs): super(Channel, self).save(*args, **kwargs) self.trajectory_estimate.save() # this will bump the datetime auto-update of trajectory + + +class TaskProtocol(BaseModel): + name = models.CharField(max_length=255, unique=True) + version = models.CharField(max_length=255, unique=True) # TODO Change uniques + description = models.CharField( + max_length=1023, blank=True, help_text='Description of the task protocol') + + def __str__(self): + return "" % self.name diff --git a/alyx/jobs/admin.py b/alyx/jobs/admin.py index f114d96d..3f763c8e 100644 --- a/alyx/jobs/admin.py +++ b/alyx/jobs/admin.py @@ -11,9 +11,9 @@ class TaskAdmin(BaseAdmin): exclude = ['json'] readonly_fields = ['session', 'log', 'parents'] list_display = ['name', 'graph', 'status', 'version_str', 'level', 'datetime', - 'session_str', 'session_task_protocol', 'session_projects'] + 'session_str', 'session_task_protocols', 'session_projects'] search_fields = ('session__id', 'session__lab__name', 'session__subject__nickname', - 'log', 'version', 'session__task_protocol', 'session__projects__name') + 'log', 'version', 'session__task_protocols__name', 'session__projects__name') ordering = ('-session__start_time', 'level') list_editable = ('status', ) list_filter = [('name', DropdownFilter), @@ -32,9 +32,10 @@ def session_projects(self, obj): return obj.session.projects.name session_projects.short_description = 'projects' - def session_task_protocol(self, obj): - return obj.session.task_protocol - session_task_protocol.short_description = 'task_protocol' + def session_task_protocols(self, obj): + if obj.session.task_protocols is not None: + return obj.session.task_protocols.name + session_task_protocols.short_description = 'task_protocols' def session_str(self, obj): url = get_admin_url(obj.session) diff --git a/alyx/misc/management/commands/one_cache.py b/alyx/misc/management/commands/one_cache.py index 65da2197..af726b49 100644 --- a/alyx/misc/management/commands/one_cache.py +++ b/alyx/misc/management/commands/one_cache.py @@ -311,12 +311,15 @@ def generate_sessions_frame(int_id=True, tags=None) -> pd.DataFrame: ) """ fields = ('id', 'lab__name', 'subject__nickname', 'start_time__date', - 'number', 'task_protocol', 'all_projects') + 'number', 'all_protocols', 'all_projects') + projects = ArrayAgg('projects__name') + protocols = ArrayAgg('task_protocols__name') query = (Session .objects .select_related('subject', 'lab') .prefetch_related('projects') - .annotate(all_projects=ArrayAgg('projects__name')) + .prefetch_related('task_protocols') + .annotate(all_projects=projects, all_protocols=protocols) .order_by('-start_time', 'subject__nickname', '-number')) # FIXME Ignores nickname :( if tags: if not isinstance(tags, str): @@ -327,16 +330,18 @@ def generate_sessions_frame(int_id=True, tags=None) -> pd.DataFrame: logger.debug(f'Raw session frame = {getsizeof(df) / 1024**2} MiB') # Rename, sort fields df['all_projects'] = df['all_projects'].map(lambda x: ','.join(filter(None, set(x)))) + df['all_protocols'] = df['all_protocols'].map(lambda x: ','.join(filter(None, set(x)))) + renames = {'start_time': 'date', 'all_projects': 'projects', 'all_protocols': 'task_protocols'} df = ( (df .rename(lambda x: x.split('__')[0], axis=1) - .rename({'start_time': 'date', 'all_projects': 'projects'}, axis=1) + .rename(renames, axis=1) .dropna(subset=['number', 'date', 'subject', 'lab']) # Remove dud or base sessions .sort_values(['date', 'subject', 'number'], ascending=False)) ) df['number'] = df['number'].astype(int) # After dropping nans we can convert number to int # These columns may be empty; ensure None -> '' - for col in ('task_protocol', 'projects'): + for col in ('task_protocols', 'projects'): df[col] = df[col].astype(str) if int_id: From 474c0a20ae2a79ff0b62a9fe1b6c8ac1aa10f56d Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Thu, 2 Feb 2023 16:26:21 +0200 Subject: [PATCH 2/4] name version unique contraint --- alyx/experiments/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/alyx/experiments/models.py b/alyx/experiments/models.py index 9dd2c04d..c824e2e9 100644 --- a/alyx/experiments/models.py +++ b/alyx/experiments/models.py @@ -238,10 +238,13 @@ def save(self, *args, **kwargs): class TaskProtocol(BaseModel): - name = models.CharField(max_length=255, unique=True) - version = models.CharField(max_length=255, unique=True) # TODO Change uniques + name = models.CharField(max_length=255) + version = models.CharField(max_length=255, help_text='The major version of the task protocol') description = models.CharField( max_length=1023, blank=True, help_text='Description of the task protocol') + class Meta: + unique_together = (('name', 'version'),) + def __str__(self): return "" % self.name From a199e6c1d584b765ba324d04eaad0d9cd893b9a6 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Fri, 3 Feb 2023 12:18:05 +0200 Subject: [PATCH 3/4] Add migrations --- ...k_protocols_alter_session_task_protocol.py | 24 +++++++++++++++++ .../migrations/0012_taskprotocol.py | 27 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 alyx/actions/migrations/0019_session_task_protocols_alter_session_task_protocol.py create mode 100644 alyx/experiments/migrations/0012_taskprotocol.py diff --git a/alyx/actions/migrations/0019_session_task_protocols_alter_session_task_protocol.py b/alyx/actions/migrations/0019_session_task_protocols_alter_session_task_protocol.py new file mode 100644 index 00000000..c5f5cae6 --- /dev/null +++ b/alyx/actions/migrations/0019_session_task_protocols_alter_session_task_protocol.py @@ -0,0 +1,24 @@ +# Generated by Django 4.1.5 on 2023-02-03 10:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('experiments', '0012_taskprotocol'), + ('actions', '0018_session_projects_alter_session_project'), + ] + + operations = [ + migrations.AddField( + model_name='session', + name='task_protocols', + field=models.ManyToManyField(blank=True, to='experiments.taskprotocol', verbose_name='Session task protocols'), + ), + migrations.AlterField( + model_name='session', + name='task_protocol', + field=models.CharField(blank=True, default='old task protocol', max_length=1023), + ), + ] diff --git a/alyx/experiments/migrations/0012_taskprotocol.py b/alyx/experiments/migrations/0012_taskprotocol.py new file mode 100644 index 00000000..e6fe9368 --- /dev/null +++ b/alyx/experiments/migrations/0012_taskprotocol.py @@ -0,0 +1,27 @@ +# Generated by Django 4.1.5 on 2023-02-03 10:16 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('experiments', '0011_chronic_insertion'), + ] + + operations = [ + migrations.CreateModel( + name='TaskProtocol', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('json', models.JSONField(blank=True, help_text='Structured data, formatted in a user-defined way', null=True)), + ('name', models.CharField(max_length=255)), + ('version', models.CharField(help_text='The major version of the task protocol', max_length=255)), + ('description', models.CharField(blank=True, help_text='Description of the task protocol', max_length=1023)), + ], + options={ + 'unique_together': {('name', 'version')}, + }, + ), + ] From 709bec509767543aa1b06b15a22542771e0877e2 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Fri, 3 Feb 2023 12:27:12 +0200 Subject: [PATCH 4/4] task_protocols slugfield in SessionListSerializer --- alyx/actions/serializers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/alyx/actions/serializers.py b/alyx/actions/serializers.py index 746abaaf..dfaf3a17 100644 --- a/alyx/actions/serializers.py +++ b/alyx/actions/serializers.py @@ -122,6 +122,10 @@ class SessionListSerializer(BaseActionSerializer): slug_field='name', queryset=Project.objects.all(), many=True) + task_protocols = serializers.SlugRelatedField(read_only=False, + slug_field='name', + queryset=TaskProtocol.objects.all(), + many=True) @staticmethod def setup_eager_loading(queryset):