Merge pull request #145 from diging/develop

Develop

.gitignore

@@ -58,3 +58,5 @@ target/
 
 db.sqlite3
 jars/static/*
+dump.rdb
+celerybeat-schedule.db

.travis.yml

@@ -7,6 +7,7 @@ env:
     - BACKEND='postgres'
     - LOGLEVEL='ERROR'
     - DATABASE_URL='postgres://jars@localhost:5432/jars_tests'
+    - BOTO_CONFIG=/dev/null
 before_script:
   - createuser -d -U postgres jars;
   - psql -c 'alter role jars createdb;' -U postgres

cookies/accession/__init__.py

@@ -5,7 +5,7 @@
 from django.db import transaction
 from django.utils import timezone
 
-import importlib, mimetypes, copy, os, logging, requests
+import importlib, mimetypes, copy, os, logging, requests, urllib
 from cookies.models import *
 from uuid import uuid4
 from cookies import metadata
@@ -340,7 +340,7 @@ def create_resource(self, resource_data, relation_data, container=None):
         """
         data = copy.copy(self.resource_data)
         data.update(resource_data)
-        file_path = data.pop('link', None)
+        file_path = data.pop('location', None)
         location = data.pop('url', None)
         uri = data.get('uri')
 
@@ -455,12 +455,12 @@ def create_content_resource(self, content_data, resource):
         relation_data = {}
         for key, value in content_data.items():
             if key in self.model_fields + ['link', 'url']:
-                if key in ['url', 'location']:
+                if key in ['url', 'location', 'link']:
                     if value and type(value) is list:
                         value = value[0]
                     if value.startswith('http'):
                         resource_data['is_external'] = True
-                    resource_data['location'] = value
+                    resource_data['location'] = urllib.unquote(value)
                 elif key in ['external_source', 'content_type']:
                     resource_data[key] = value
                     if key == 'content_type':

cookies/forms.py

@@ -23,7 +23,7 @@ class CustomModelChoiceField(forms.ModelChoiceField):
     """
 
     def label_from_instance(self, obj):
-         return obj.name
+        return obj.name
 
 
 class TypeModelChoiceField(forms.ModelChoiceField):
@@ -33,7 +33,7 @@ class TypeModelChoiceField(forms.ModelChoiceField):
 
     def label_from_instance(self, obj):
         if obj.schema is not None:
-            return u'%s: %s' % (obj.schema.name, obj.name)
+            return u'%s (%s)' % (obj.name, obj.schema.name)
         else:
             return obj.name
 
@@ -195,7 +195,7 @@ class UserEditResourceForm(forms.Form):
     """
 
     name = forms.CharField(help_text='Give your resource a unique name')
-    resource_type = CustomModelChoiceField(**{
+    resource_type = TypeModelChoiceField(**{
         'queryset': Type.objects.all().order_by('name'),
         'help_text': 'Types help JARS determine what metadata fields are' \
                    + ' appropriate for your resource.',
@@ -328,7 +328,7 @@ class MetadatumTypeForm(forms.Form):
 
 
 class MetadatumForm(forms.Form):
-    predicate = CustomModelChoiceField(queryset=Field.objects.all().order_by('-name'))
+    predicate = TypeModelChoiceField(queryset=Field.objects.all().order_by('-name'))
     value_type = forms.ChoiceField(choices=(
         ('Int', 'Integer'),
         ('Float', 'Float'),
@@ -441,7 +441,7 @@ def clean(self):
 class DatasetForm(forms.ModelForm):
     filter_parameters = forms.CharField(widget=forms.HiddenInput())
 
-    description = forms.CharField(widget=forms.Textarea(attrs={'rows': 3}),
+    description = forms.CharField(required=False, widget=forms.Textarea(attrs={'rows': 3}),
                                   help_text="Please describe the purpose and"
                                             " content of the dataset"
                                             " (optional).")

cookies/giles.py

@@ -498,17 +498,22 @@ def _process_uploaded_file(self):
         resource_type = self.CONTENT_RESOURCE_TYPE_MAP.get(content_type,
                                                            self.__image__)
         self._save_content_resource(self._resource, resource_type, resource_uri,
-                                    giles_url, content_type=content_type)
+                                    giles_url, content_type=content_type,
+                                    name='%s (uploaded)' % (self._resource.name),
+                                    file_id=upload_data['id'])
 
     def _process_extracted_text(self):
         text_data = self._data.get('extractedText', None)
         if text_data is None:
             return
         text_content_type = text_data.get('content-type')
         text_uri = text_data.get('url')
-        content_resource = self._save_content_resource(self._resource, self.__text__,
-                                                       text_uri, text_uri,
-                                                       content_type=text_content_type)
+        content_resource = self._save_content_resource(
+            self._resource, self.__text__, text_uri, text_uri,
+            content_type=text_content_type,
+            name='%s (extracted)' % (self._resource.name),
+            file_id=text_data['id'],
+        )
         self._save_resource_creator(content_resource,
                                     self.__creator__,
                                     GILES_RESPONSE_CREATOR_MAP['extractedText'])
@@ -539,6 +544,7 @@ def _process_pages(self):
                     page_resource, self.__image__ if fmt == 'image' else self.__text__,
                     page_fmt_uri, fmt_data.get('url'),
                     public=False, content_type=fmt_data.get('content-type'),
+                    file_id=fmt_data['id'],
                     name='%s (%s)' % (page_resource.name, fmt)
                 )
 
@@ -607,6 +613,7 @@ def _get_resource_type(data):
                 resource_type,
                 uri, uri,
                 content_type=content_type,
+                file_id=additional_file['id'],
                 name=name_fn(additional_file),
             )
 
@@ -624,8 +631,12 @@ def process(self):
         # Content resource for extracted text, if available.
         self._process_extracted_text()
 
+        name_fn = lambda d: '%s - %s (%s)' % (self._resource.name, d.get('id'),
+                                              d.get('content-type'))
         # Content resource for each additional file, if available.
-        self._process_additional_files(self._data.get('additionalFiles', []), self._resource)
+        self._process_additional_files(self._data.get('additionalFiles', []),
+                                       self._resource,
+                                       name_fn=name_fn)
 
         self._process_pages()
 
@@ -681,9 +692,8 @@ def _save_content_resource(self, parent_resource, resource_type, uri, url,
                 content_rel.save()
 
         except KeyError:
-            content_resource = Resource.objects.create(**{
+            kwargs = {
                 'name': meta.get('name', url),
-                'location': url,
                 'public': meta.get('public', False),
                 'content_resource': True,
                 'created_by_id': self._user.id,
@@ -694,8 +704,13 @@ def _save_content_resource(self, parent_resource, resource_type, uri, url,
                 'external_source': Resource.GILES,
                 'uri': uri,
                 'container': parent_resource.container,
-            })
+            }
+            try:
+                kwargs['location_id'] = meta['file_id']
+            except KeyError:
+                kwargs['location'] = url
 
+            content_resource = Resource.objects.create(**kwargs)
             content_rel = ContentRelation.objects.create(**{
                 'for_resource': parent_resource,
                 'content_resource': content_resource,

cookies/migrations/0024_auto_20180803_2320.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.12 on 2018-08-03 23:20
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookies', '0023_auto_20180426_1837'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='gilesupload',
+            name='file_path',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

cookies/migrations/0024_resource_location_id.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.12 on 2018-04-27 23:06
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookies', '0023_auto_20180426_1837'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='resource',
+            name='location_id',
+            field=models.CharField(blank=True, help_text=b'Instead of storing the complete resource URL in `location`, use this field to store the unique part of URL and generate the complete URL at runtime.', max_length=255, null=True),
+        ),
+    ]

cookies/migrations/0025_merge_20181105_1812.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.12 on 2018-11-05 18:12
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookies', '0024_auto_20180803_2320'),
+        ('cookies', '0024_resource_location_id'),
+    ]
+
+    operations = [
+    ]

cookies/models.py

@@ -197,6 +197,22 @@ class Resource(ResourceBase):
     description = models.TextField(blank=True, null=True)
 
     name_index = TSVectorField(('name',), dictionary='simple')
+    location_id = models.CharField(
+        max_length=255, blank=True, null=True,
+        help_text="Instead of storing the complete resource URL in `location`, "
+                  "use this field to store the unique part of URL and generate "
+                  "the complete URL at runtime.""",
+    )
+
+    def __getattribute__(self, attr):
+        if attr != 'location':
+            return super(Resource, self).__getattribute__(attr)
+        location = super(Resource, self).__getattribute__(attr)
+        if location:
+            return location
+        if self.location_id and self.external_source == self.GILES:
+            return settings.GILES_CONTENT_FORMAT_STRING.format(giles_file_id=self.location_id)
+        return None
 
     @property
     def active_content(self):
@@ -639,7 +655,7 @@ class GilesUpload(models.Model):
     on_complete = models.TextField()
     """Serialized callback instructions."""
 
-    file_path = models.CharField(max_length=1000, blank=True, null=True)
+    file_path = models.TextField(blank=True, null=True)
     """Relative to MEDIA_ROOT."""