From 45ad8c05d051c86113a7948b202eec5d4ea5123e Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Wed, 28 Sep 2022 13:22:40 -0400 Subject: [PATCH] Remove redundant "Optimized-" models (#196) * Remove redundant "Optimized-" models * Remove references in swcc project * Lint fix of migration file * Remove reference in tasks * Remove implicit references in swcc project --- shapeworks_cloud/core/filters.py | 18 +----- .../0013_remove_redundant_models.py | 50 +++++++++++++++ shapeworks_cloud/core/models.py | 53 +-------------- shapeworks_cloud/core/rest.py | 22 ------- shapeworks_cloud/core/serializers.py | 39 ----------- shapeworks_cloud/core/tasks.py | 7 +- shapeworks_cloud/urls.py | 6 -- swcc/swcc/models/__init__.py | 4 -- swcc/swcc/models/other_models.py | 50 +-------------- swcc/swcc/models/project.py | 64 +++++++------------ swcc/tests/factories.py | 20 +----- swcc/usage.ipynb | 9 ++- 12 files changed, 86 insertions(+), 256 deletions(-) create mode 100644 shapeworks_cloud/core/migrations/0013_remove_redundant_models.py diff --git a/shapeworks_cloud/core/filters.py b/shapeworks_cloud/core/filters.py index 06f35ca6..d95a44b7 100644 --- a/shapeworks_cloud/core/filters.py +++ b/shapeworks_cloud/core/filters.py @@ -67,16 +67,8 @@ class Meta: fields = ['project', 'mesh'] -class OptimizedShapeModelFilter(FilterSet): - project = ModelChoiceFilter(queryset=models.Project.objects.all()) - - class Meta: - models = models.OptimizedShapeModel - fields = ['project'] - - class OptimizedParticlesFilter(FilterSet): - shape_model = ModelChoiceFilter(queryset=models.OptimizedShapeModel.objects.all()) + project = ModelChoiceFilter(queryset=models.Project.objects.all()) original_segmentation = ModelChoiceFilter( field_name='groomed_segmentation__segmentation', queryset=models.Segmentation.objects.all(), @@ -97,11 +89,3 @@ class Meta: 'original_segmentation', 'original_mesh', ] - - -class OptimizedPCAModelFilter(FilterSet): - shape_model = ModelChoiceFilter(queryset=models.OptimizedShapeModel.objects.all()) - - class Meta: - models = models.OptimizedParticles - fields = ['shape_model'] diff --git a/shapeworks_cloud/core/migrations/0013_remove_redundant_models.py b/shapeworks_cloud/core/migrations/0013_remove_redundant_models.py new file mode 100644 index 00000000..f81055a6 --- /dev/null +++ b/shapeworks_cloud/core/migrations/0013_remove_redundant_models.py @@ -0,0 +1,50 @@ +# Generated by Django 3.2.15 on 2022-09-27 14:38 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_analysis_models'), + ] + + operations = [ + migrations.RemoveField( + model_name='optimizedshapemodel', + name='project', + ), + migrations.RemoveField( + model_name='optimizedsurfacereconstruction', + name='particles', + ), + migrations.RemoveField( + model_name='optimizedsurfacereconstructionmeta', + name='shape_model', + ), + migrations.RemoveField( + model_name='optimizedparticles', + name='shape_model', + ), + migrations.AddField( + model_name='optimizedparticles', + name='project', + field=models.ForeignKey( + default=None, on_delete=django.db.models.deletion.CASCADE, to='core.project' + ), + preserve_default=False, + ), + migrations.DeleteModel( + name='OptimizedPCAModel', + ), + migrations.DeleteModel( + name='OptimizedShapeModel', + ), + migrations.DeleteModel( + name='OptimizedSurfaceReconstruction', + ), + migrations.DeleteModel( + name='OptimizedSurfaceReconstructionMeta', + ), + ] diff --git a/shapeworks_cloud/core/models.py b/shapeworks_cloud/core/models.py index b4d48a09..142c7d0f 100644 --- a/shapeworks_cloud/core/models.py +++ b/shapeworks_cloud/core/models.py @@ -109,26 +109,13 @@ class GroomedMesh(TimeStampedModel, models.Model): project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='groomed_meshes') -class OptimizedShapeModel(TimeStampedModel, models.Model): - project = models.ForeignKey( - Project, - on_delete=models.CASCADE, - related_name='optimized_shape_models', - ) - parameters = models.JSONField(default=dict) - - class OptimizedParticles(TimeStampedModel, models.Model): + project = models.ForeignKey(Project, on_delete=models.CASCADE) world = S3FileField() local = S3FileField() transform = S3FileField() constraints = S3FileField(null=True) - shape_model = models.ForeignKey( - OptimizedShapeModel, - on_delete=models.CASCADE, - related_name='particles', - ) groomed_segmentation = models.ForeignKey( GroomedSegmentation, on_delete=models.CASCADE, @@ -143,41 +130,3 @@ class OptimizedParticles(TimeStampedModel, models.Model): blank=True, null=True, ) - - -class OptimizedSurfaceReconstructionMeta(TimeStampedModel, models.Model): - method = models.CharField(max_length=255) # TODO: Should be choices - reconstruction_params = S3FileField() - template_reconstruction = S3FileField() - - shape_model = models.ForeignKey( - OptimizedShapeModel, - on_delete=models.CASCADE, - related_name='+', - ) - - -class OptimizedSurfaceReconstruction(TimeStampedModel, models.Model): - particles = models.OneToOneField( - OptimizedParticles, - on_delete=models.CASCADE, - related_name='surface_reconstruction', - primary_key=True, - ) - sample_reconstruction = S3FileField() - - -class OptimizedPCAModel(TimeStampedModel, models.Model): - shape_model = models.OneToOneField( - OptimizedShapeModel, - on_delete=models.CASCADE, - related_name='pca_model', - primary_key=True, - ) - - mean_particles = S3FileField() - pca_modes = S3FileField() - eigen_spectrum = S3FileField() - - -# TODO: checkpoints... what about pca model reference? diff --git a/shapeworks_cloud/core/rest.py b/shapeworks_cloud/core/rest.py index a82f38f9..016663dd 100644 --- a/shapeworks_cloud/core/rest.py +++ b/shapeworks_cloud/core/rest.py @@ -116,34 +116,12 @@ class GroomedMeshViewSet(BaseViewSet): filterset_class = filters.GroomedMeshFilter -class OptimizedShapeModelViewSet(BaseViewSet): - queryset = models.OptimizedShapeModel.objects.all() - serializer_class = serializers.OptimizedShapeModelSerializer - filterset_class = filters.OptimizedShapeModelFilter - - class OptimizedParticlesViewSet(BaseViewSet): queryset = models.OptimizedParticles.objects.all() serializer_class = serializers.OptimizedParticlesSerializer filterset_class = filters.OptimizedParticlesFilter -class OptimizedSurfaceReconstructionMetaViewSet(BaseViewSet): - queryset = models.OptimizedSurfaceReconstructionMeta.objects.all() - serializer_class = serializers.OptimizedSurfaceReconstructionMetaSerializer - - -class OptimizedSurfaceReconstructionViewSet(BaseViewSet): - queryset = models.OptimizedSurfaceReconstruction.objects.all() - serializer_class = serializers.OptimizedSurfaceReconstructionSerializer - - -class OptimizedPCAModelViewSet(BaseViewSet): - queryset = models.OptimizedPCAModel.objects.all() - serializer_class = serializers.OptimizedPCAModelSerializer - filterset_class = filters.OptimizedPCAModelFilter - - class CachedAnalysisViewSet(BaseViewSet): queryset = models.CachedAnalysis.objects.all() diff --git a/shapeworks_cloud/core/serializers.py b/shapeworks_cloud/core/serializers.py index 8c446b02..d82c5a7e 100644 --- a/shapeworks_cloud/core/serializers.py +++ b/shapeworks_cloud/core/serializers.py @@ -122,14 +122,6 @@ def get_id(self, obj) -> int: return obj.mesh_id -class OptimizedShapeModelSerializer(serializers.ModelSerializer): - parameters = serializers.DictField() - - class Meta: - model = models.OptimizedShapeModel - fields = '__all__' - - class OptimizedParticlesSerializer(serializers.ModelSerializer): world = S3FileSerializerField() local = S3FileSerializerField() @@ -139,34 +131,3 @@ class OptimizedParticlesSerializer(serializers.ModelSerializer): class Meta: model = models.OptimizedParticles fields = '__all__' - - -class OptimizedSurfaceReconstructionMetaSerializer(serializers.ModelSerializer): - reconstruction_params = S3FileSerializerField() - template_reconstruction = S3FileSerializerField() - - class Meta: - model = models.OptimizedSurfaceReconstructionMeta - fields = '__all__' - - -class OptimizedSurfaceReconstructionSerializer(serializers.ModelSerializer): - sample_reconstruction = S3FileSerializerField() - - class Meta: - model = models.OptimizedSurfaceReconstruction - fields = '__all__' - - -class OptimizedPCAModelSerializer(serializers.ModelSerializer): - id = serializers.SerializerMethodField() - mean_particles = S3FileSerializerField() - pca_modes = S3FileSerializerField() - eigen_spectrum = S3FileSerializerField() - - class Meta: - model = models.OptimizedPCAModel - fields = '__all__' - - def get_id(self, obj) -> int: - return obj.shape_model.id diff --git a/shapeworks_cloud/core/tasks.py b/shapeworks_cloud/core/tasks.py index 74a532aa..14ffa924 100644 --- a/shapeworks_cloud/core/tasks.py +++ b/shapeworks_cloud/core/tasks.py @@ -189,7 +189,7 @@ def post_command_function(project, download_dir, project_dfs, project_filename): def optimize(user_id, project_id, form_data): def pre_command_function(): # delete any previous results - models.OptimizedShapeModel.objects.filter(project=project_id).delete() + models.OptimizedParticles.objects.filter(project=project_id).delete() def post_command_function(project, download_dir, project_dfs, project_filename): save_dfs_to_project_spreadsheet( @@ -211,13 +211,10 @@ def post_command_function(project, download_dir, project_dfs, project_filename): target_mesh = project_groomed_meshes.filter( file__endswith=groomed_filename, ).first() - result_shape_object = models.OptimizedShapeModel.objects.create( - project=project, parameters=form_data - ) result_particles_object = models.OptimizedParticles.objects.create( groomed_segmentation=target_segmentation, groomed_mesh=target_mesh, - shape_model=result_shape_object, + project=project, ) result_particles_object.world.save( row['world_particles_1'].split('/')[-1], diff --git a/shapeworks_cloud/urls.py b/shapeworks_cloud/urls.py index c603c476..78391a48 100644 --- a/shapeworks_cloud/urls.py +++ b/shapeworks_cloud/urls.py @@ -20,15 +20,9 @@ 'groomed-segmentations', rest.GroomedSegmentationViewSet, basename='groomed_segmentation' ) router.register('groomed-meshes', rest.GroomedMeshViewSet, basename='groomed_mesh') -router.register( - 'optimized-shape-models', rest.OptimizedShapeModelViewSet, basename='optimized_shape_model' -) router.register( 'optimized-particles', rest.OptimizedParticlesViewSet, basename='optimized_particles' ) -router.register( - 'optimized-pca-model', rest.OptimizedPCAModelViewSet, basename='optimized_pca_model' -) router.register('cached-analysis', rest.CachedAnalysisViewSet, basename='cached_analysis') router.register( 'cached-analysis-mode', rest.CachedAnalysisModeViewSet, basename='cached_analysis_mode' diff --git a/swcc/swcc/models/__init__.py b/swcc/swcc/models/__init__.py index 076e85de..ce3aa7eb 100644 --- a/swcc/swcc/models/__init__.py +++ b/swcc/swcc/models/__init__.py @@ -5,8 +5,6 @@ Image, Mesh, OptimizedParticles, - OptimizedPCAModel, - OptimizedShapeModel, Segmentation, ) from .project import Project @@ -19,8 +17,6 @@ 'Image', 'Mesh', 'OptimizedParticles', - 'OptimizedPCAModel', - 'OptimizedShapeModel', 'Project', 'Segmentation', 'Subject', diff --git a/swcc/swcc/models/other_models.py b/swcc/swcc/models/other_models.py index 9f9a4687..8613b26e 100644 --- a/swcc/swcc/models/other_models.py +++ b/swcc/swcc/models/other_models.py @@ -1,14 +1,9 @@ from __future__ import annotations -from pathlib import Path - try: - from typing import Any, Dict, Iterator, List, Literal, Optional + from typing import List, Literal, Optional except ImportError: from typing import ( - Any, - Dict, - Iterator, List, Optional, ) @@ -67,57 +62,18 @@ class GroomedMesh(ApiModel): project: Project -class OptimizedShapeModel(ApiModel): - _endpoint = 'optimized-shape-models' - - project: Project - parameters: Dict[str, Any] - - @property - def particles(self) -> Iterator[OptimizedParticles]: - return OptimizedParticles.list(shape_model=self) - - def add_particles( - self, - world: Path, - local: Path, - transform: Path, - groomed_segmentation: Optional[GroomedSegmentation], - groomed_mesh: Optional[GroomedMesh], - constraints: Path, - ) -> OptimizedParticles: - return OptimizedParticles( - world=world, - local=local, - transform=transform, - shape_model=self, - groomed_segmentation=groomed_segmentation, - groomed_mesh=groomed_mesh, - constraints=constraints, - ).create() - - class OptimizedParticles(ApiModel): _endpoint = 'optimized-particles' world: FileType[Literal['core.OptimizedParticles.world']] local: FileType[Literal['core.OptimizedParticles.local']] transform: FileType[Literal['core.OptimizedParticles.transform']] - shape_model: OptimizedShapeModel + project: Project groomed_segmentation: Optional[GroomedSegmentation] groomed_mesh: Optional[GroomedMesh] constraints: Optional[FileType[Literal['core.OptimizedParticles.constraints']]] = None -class OptimizedPCAModel(ApiModel): - _endpoint = 'optimized-pca-model' - - mean_particles: FileType[Literal['core.OptimizedPCAModel.mean_particles']] - pca_modes: FileType[Literal['core.OptimizedPCAModel.pca_modes']] - eigen_spectrum: FileType[Literal['core.OptimizedPCAModel.eigen_spectrum']] - shape_model: OptimizedShapeModel - - class CachedAnalysisModePCA(ApiModel): _endpoint = 'cached-analysis-mode-pca' @@ -152,6 +108,4 @@ class CachedAnalysis(ApiModel): Image.update_forward_refs() GroomedSegmentation.update_forward_refs() GroomedMesh.update_forward_refs() -OptimizedShapeModel.update_forward_refs() OptimizedParticles.update_forward_refs() -OptimizedPCAModel.update_forward_refs() diff --git a/swcc/swcc/models/project.py b/swcc/swcc/models/project.py index 46ffab07..d141cea5 100644 --- a/swcc/swcc/models/project.py +++ b/swcc/swcc/models/project.py @@ -5,12 +5,13 @@ from tempfile import TemporaryDirectory try: - from typing import Any, Dict, Iterator, Literal, Optional, Union + from typing import Any, Dict, Iterator, List, Literal, Optional, Union except ImportError: from typing import ( Any, Dict, Iterator, + List, Optional, Union, ) @@ -30,7 +31,7 @@ GroomedMesh, GroomedSegmentation, Mesh, - OptimizedShapeModel, + OptimizedParticles, Segmentation, ) from .subject import Subject @@ -43,7 +44,7 @@ class ProjectFileIO(BaseModel, FileIO): class Config: arbitrary_types_allowed = True - def load_data(self): + def load_data(self, interpret=True): if ( not self.project.file or not hasattr(self.project.file, 'path') @@ -51,21 +52,20 @@ def load_data(self): ): return file = self.project.file.path - data = None if str(file).endswith('xlsx') or str(file).endswith('xlsx'): raise NotImplementedError('Convert spreadsheet file to excel before parsing') elif str(file).endswith('json') or str(file).endswith('swproj'): - data, optimize = self.load_data_from_json(file) + return self.load_data_from_json(file, interpret) else: raise Exception(f'Unknown format for {file} - expected .xlsx, .xls, .swproj, or .json') - def load_data_from_json(self, file): + def load_data_from_json(self, file, interpret): contents = json.load(open(file)) - shape_model = self.project.add_shape_model(contents['optimize']) - self.interpret_data(contents['data'], shape_model) - return None, None + if interpret: + self.interpret_data(contents['data']) + return contents['data'], contents['optimize'] - def interpret_data(self, data, shape_model): + def interpret_data(self, data): segmentations = { PurePath(segmentation.file.name).stem: segmentation for segmentation in self.project.dataset.segmentations @@ -94,7 +94,6 @@ def interpret_data(self, data, shape_model): self.interpret_data_row( segmentations, meshes, - shape_model, Path(str(entry_values.get('shape'))), Path(str(entry_values.get('groomed'))), Path(str(entry_values.get('alignment'))), @@ -106,7 +105,6 @@ def interpret_data_row( self, segmentations, meshes, - shape_model, shape_file, groomed_file, alignment_file, @@ -153,14 +151,15 @@ def interpret_data_row( with transform.open('w') as f: f.write(str(project_root / alignment_file)) - shape_model.add_particles( + OptimizedParticles( world=project_root / world, local=project_root / local, transform=transform, groomed_segmentation=groomed_segmentation, groomed_mesh=groomed_mesh, constraints=None, - ) + project=self.project, + ).create() def load_analysis_from_json(self, file_path): project_root = Path(str(self.project.file.path)).parent @@ -258,40 +257,21 @@ def create(self) -> Project: return Project.from_id(result.id) @property - def shape_models(self) -> Iterator[OptimizedShapeModel]: + def particles(self) -> Iterator[OptimizedParticles]: self.assert_remote() - return OptimizedShapeModel.list(project=self) + return OptimizedParticles.list(project=self) - def add_shape_model(self, parameters: Dict[str, Any]) -> OptimizedShapeModel: - return OptimizedShapeModel( - project=self, - parameters=parameters, - ).create() + def add_particles(self, parameters: Dict[str, Any]) -> OptimizedParticles: + return OptimizedParticles(project=self, **parameters).create() def download(self, path: Union[Path, str]): self.file.download(Path(path)) data, optimize = self.get_file_io().load_data(interpret=False) - - shape_model = next(self.shape_models) if any(True for _ in self.shape_models) else None - groomed_segmentations = { - PurePath(gs.file.name).stem: gs for gs in self.groomed_segmentations - } - local_files = ( - {PurePath(p.local.name).stem: p for p in shape_model.particles} if shape_model else {} - ) - - for _, groomed_file, _, local, world, constraints in self.get_file_io()._iter_data_sheet( - data, Path(path) - ): - if groomed_file and groomed_file.stem in groomed_segmentations: - gs = groomed_segmentations[groomed_file.stem] - gs.file.download(groomed_file.parent) - if local and world and local.stem in local_files: - particles = local_files[local.stem] - particles.local.download(local.parent) - particles.world.download(world.parent) - if particles.constraints and constraints: - particles.constraints.download(constraints.parent) + groomed_shapes: List[Union[GroomedMesh, GroomedSegmentation]] = list(self.groomed_meshes) + groomed_shapes += list(self.groomed_segmentations) + for data_row in data: + # TODO: Download related objects + print(data_row, groomed_shapes) ProjectFileIO.update_forward_refs() diff --git a/swcc/tests/factories.py b/swcc/tests/factories.py index e8c4d6d5..75e4e710 100644 --- a/swcc/tests/factories.py +++ b/swcc/tests/factories.py @@ -121,14 +121,6 @@ class Meta: project = SubFactory(ProjectFactory) -class OptimizedShapeModelFactory(Factory): - class Meta: - model = models.OptimizedShapeModel - - project = SubFactory(ProjectFactory) - parameters = Faker('pydict', value_types=[str, int, float]) - - class OptimizedParticlesFactory(Factory): class Meta: model = models.OptimizedParticles @@ -136,16 +128,6 @@ class Meta: world = Faker('file', extension='txt') local = Faker('file', extension='txt') transform = Faker('file', extension='txt') - shape_model = SubFactory(OptimizedShapeModelFactory) + project = SubFactory(ProjectFactory) groomed_segmentation = SubFactory(GroomedSegmentationFactory) groomed_mesh = SubFactory(GroomedMeshFactory) - - -class OptimizedPCAModelFactory(Factory): - class Meta: - model = models.OptimizedPCAModel - - mean_particles = Faker('file', extension='txt') - pca_modes = Faker('file', extension='txt') - eigen_spectrum = Faker('file', extension='txt') - shape_model = SubFactory(OptimizedShapeModelFactory) diff --git a/swcc/usage.ipynb b/swcc/usage.ipynb index f4ae8471..e4274315 100644 --- a/swcc/usage.ipynb +++ b/swcc/usage.ipynb @@ -45,7 +45,7 @@ "from swcc.api import swcc_session\n", "from swcc.models import (\n", " Dataset, GroomedSegmentation, OptimizedParticles,\n", - " OptimizedShapeModel, Project, Segmentation, Subject\n", + " Project, Segmentation, Subject\n", ")" ] }, @@ -13270,7 +13270,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.8.10 64-bit", "language": "python", "name": "python3" }, @@ -13285,6 +13285,11 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" + }, + "vscode": { + "interpreter": { + "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" + } } }, "nbformat": 4,