geodjango filtering by location and osdi de/serialization work

event_exim/migrations/0011_auto_20170625_2222.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.1 on 2017-06-25 22:22
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('event_exim', '0010_merge_20170614_2305'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='eventsource',
+            name='crm_type',
+            field=models.CharField(choices=[('actionkit_api', 'actionkit_api'), ('facebook', 'facebook')], max_length=16),
+        ),
+        migrations.AlterField(
+            model_name='eventsource',
+            name='update_style',
+            field=models.IntegerField(choices=[(0, 'manual or ping only'), (3, 'daily pull'), (4, 'hourly pull')]),
+        ),
+    ]

event_exim/models.py

@@ -4,6 +4,7 @@
 
 from django.conf import settings
 from django.contrib.auth.models import User, Group, Permission
+from django.contrib.gis.geos import Point
 from django.db import models
 from django.utils.functional import cached_property
 from django.db.models import Count
@@ -122,6 +123,9 @@ def update_events_from_dicts(self, event_dicts):
         existing_ids = set([e.organization_source_pk for e in existing])
         new_events = [Event(**e) for e in all_events.values()
                       if e['organization_source_pk'] not in existing_ids]
+        for e in new_events:
+            if e.longitude and e.latitude:
+                e.point = Point(e.longitude, e.latitude)
         Event.objects.bulk_create(new_events)
 
         # 4. save any changes to existing events
@@ -139,6 +143,8 @@ def update_event_from_dict(self, event, new_event_dict):
                 if getattr(event,k) != v:
                     setattr(event,k,v)
                     changed = True
+                    if k in ('longitude', 'latitude'):
+                        event.point = Point(event.longitude, event.latitude)
         if changed:
             event.save()
         return changed

event_exim/serializers.py

@@ -6,7 +6,30 @@
 
 from event_store.models import Event
 
-class OsdiEventSerializer(HalModelSerializer):
+
+class OsdiLocationGeoSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Event
+        fields = ['longitude','latitude']
+
+    def get_attribute(self, obj):
+        return obj
+
+class OsdiLocationSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Event
+        fields = ['venue', 'country', 'postal_code', 'locality', 'region', 'location']
+
+    postal_code = serializers.CharField(source="zip", required=False)
+    locality = serializers.CharField(source="city", required=False)
+    region = serializers.CharField(source="state", required=False)
+    location = OsdiLocationGeoSerializer(required=False)
+
+    def get_attribute(self, obj):
+        return obj
+
+# HalModelSerializer is failing with location (non)'nested' field
+class OsdiEventSerializer(serializers.ModelSerializer):
     class Meta:
         model = Event
         fields = [
@@ -28,16 +51,16 @@ class Meta:
             'human_date', #friendly display
         ]
 
-    origin_system = serializers.CharField(source="osdi_origin_system", read_only=True)
-    name = serializers.CharField(source="slug", read_only=True)
+    origin_system = serializers.CharField(source="osdi_origin_system", required=False)
+    name = serializers.CharField(source="slug", required=False)
     #title
-    description = serializers.CharField(source="public_description", read_only=True)
-    browser_url = serializers.CharField(source="url", read_only=True)
-    type = serializers.SerializerMethodField()
+    description = serializers.CharField(source="public_description", required=False)
+    browser_url = serializers.CharField(source="url", required=False)
+    type = serializers.SerializerMethodField(required=False)
     def get_type(self, obj):
         return obj.get_ticket_type_display()
-    total_accepted = serializers.IntegerField(source="attendee_count", read_only=True)
-    status = serializers.SerializerMethodField()
+    total_accepted = serializers.IntegerField(source="attendee_count", required=False)
+    status = serializers.SerializerMethodField(required=False)
     def get_status(self, obj):
         if obj.host_is_confirmed and obj.status == 'active':
             return 'confirmed'
@@ -47,32 +70,46 @@ def get_status(self, obj):
             return 'tentative'
 
     # e.g. 2017-07-04T19:00:00
-    start_date = serializers.DateTimeField(source='starts_at', format='iso-8601')
-    end_date = serializers.DateTimeField(source='ends_at', allow_null=True, format='iso-8601')
+    start_date = serializers.DateTimeField(source='starts_at', format='iso-8601', required=False)
+    end_date = serializers.DateTimeField(source='ends_at', allow_null=True, format='iso-8601',
+                                         required=False)
 
-    capacity = serializers.IntegerField(source="max_attendees", read_only=True)
-    visibility = serializers.SerializerMethodField()
+    capacity = serializers.IntegerField(source="max_attendees", required=False)
+    visibility = serializers.SerializerMethodField(required=False)
     def get_visibility(self, obj):
         """we are using this strictly as private/public, but is_searchable is different"""
         return obj.get_is_private_display()
-    location = serializers.SerializerMethodField()
-    def get_location(self, obj):
-        return {
-            'venue': obj.venue,
-            'location': {
-                'longitude': obj.longitude,
-                'latitude': obj.latitude,
-            },
-            'postal_code': obj.zip,
-            'locality': obj.city,
-            'region': obj.state
-        }
+    location = OsdiLocationSerializer(required=False)
 
-    human_date = serializers.SerializerMethodField()
+    human_date = serializers.SerializerMethodField(read_only=True)
     def get_human_date(self, obj):
         """e.g. 'Saturday, November 5' """
         return dateformat.format(obj.starts_at, 'l, F j')
-    human_time = serializers.SerializerMethodField()
+    human_time = serializers.SerializerMethodField(read_only=True)
     def get_human_time(self, obj):
         """e.g. '6:30pm' or '5am' """
         return dateformat.format(obj.starts_at, 'fa').replace('.', '')
+
+    def to_internal_value(self, data):
+        internal = super(OsdiEventSerializer,self).to_internal_value(data)
+        # Flatten the location fields into the same internal value
+        if 'location' in internal:
+            loc = internal.pop('location')
+            if loc:
+                if 'location' in loc:
+                    locloc = loc.pop('location')
+                    loc.update(locloc)
+                internal.update(loc)
+        return internal
+
+    @classmethod
+    def odata_field_mapper(cls, fieldtuple):
+        """
+        Maps a field tuple like ("location", "region")
+        to the django filter value (like 'region' for the above)
+        """
+        target = cls()
+        for f in fieldtuple:
+            target = target[f]
+        if target:
+            return target.source

event_exim/tests.py

@@ -35,4 +35,5 @@ def test_api_route(self):
         self.assertEqual(response.status_code, 200)
 
     def test_api_content_type(self):
+        response = self.c.get('/api/v1/events/')
         self.assertEqual(response['content-type'], 'application/json')

event_exim/urls.py

@@ -5,7 +5,7 @@
 
 # public API endpoint for developers to fetch data from
 public_patterns = [
-    url(r'^events/$', views.PublicEventViewSet.as_view({'get': 'list'}),
+    url(r'^events/?$', views.PublicEventViewSet.as_view({'get': 'list'}),
         name='osdi_public_events'),
 ]
 

event_exim/views.py

@@ -1,8 +1,14 @@
+import importlib
 import json
+import re
 
 from django.conf import settings
+from django.contrib.gis.geos import Point
+from django.contrib.gis.measure import Distance
 from django.http import HttpResponse
 from django.shortcuts import render, get_object_or_404
+
+import odata
 from osdi.pagination import OsdiPagination
 from rest_framework.viewsets import ModelViewSet
 
@@ -40,6 +46,36 @@ class PublicEventViewSet(ModelViewSet):
     def get_queryset(self):
         #self.request.query_params
         # TODO: exclude based on review
-        return Event.objects.filter(
+        queryset = Event.objects.filter(
             is_searchable=True, is_private=False).exclude(
                 organization_status_review__in=('questionable', 'limbo'))
+        rGET = self.request.GET
+        # supporting geo query described here:
+        # https://github.com/ResistanceCalendar/resistance-calendar-api#location
+        distance_max = rGET.get('distance_max') # unit is in meters
+        if distance_max:
+            coords = None
+            # e.g. distance_coords=[-98.435508,29.516496]&distance_max=10000
+            if rGET.get('distance_coords'):
+                coord_re = re.match(r'\[([-\d.]+),([-\d.]+)\]', rGET['distance_coords'])
+                if coord_re:
+                    coords = Point(coord_re.group(1), coord_re.group(2))
+            # e.g. distance_postal_code=94110&distance_max=10000
+            elif rGET.get('distance_postal_code') and getattr(settings, 'POSTAL_CODE_GEO_LIBRARY', None):
+                # This is mostly assuming POSTAL_CODE_GEO_LIBRARY = 'pyzipcode.ZipCodeDatabase'
+                # Anything else implementing it must have the same API
+                if not hasattr(self, 'postaldb'):
+                    module, obj = settings.POSTAL_CODE_GEO_LIBRARY.rsplit('.', 1)
+                    postal_module = importlib.import_module(module)
+                    self.postaldb = getattr(postal_module, obj)()
+                zipcode = self.postaldb[int(rGET.get('distance_postal_code'))]
+                if zipcode:
+                    coords = Point(zipcode.longitude, zipcode.latitude)
+            if coords:
+                queryset = queryset.filter(point__distance_lt=(coords, Distance(m=int(distance_max))))
+        odata_filters = odata.django_params(rGET, OsdiEventSerializer.odata_field_mapper)
+        if 'filter' in odata_filters:
+            queryset = queryset.filter(odata_filters['filter'])
+        if 'orderby' in odata_filters:
+            queryset = queryset.order_by(*odata_filters['orderby'])
+        return queryset

event_store/migrations/0009_auto_20170625_2222.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import django.contrib.gis.db.models.fields
+from django.db import migrations, models
+from django.contrib.gis.geos import Point
+
+def migrate_latlng_points(apps, schema_editor):
+    Event = apps.get_model('event_store', 'Event')
+    for event in Event.objects.filter(longitude__isnull=False, latitude__isnull=False):
+        event.point = Point(event.longitude, event.latitude, srid=4326)
+        event.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('event_store', '0008_auto_20170616_1506'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='event',
+            name='point',
+            field=django.contrib.gis.db.models.fields.PointField(null=True, blank=True, srid=4326),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='organization_status_review',
+            field=models.CharField(blank=True, choices=[(None, 'New'), ('reviewed', 'Reviewed'), ('vetted', 'Vetted'), ('questionable', 'Questionable'), ('limbo', 'Limbo')], db_index=True, max_length=32, null=True),
+        ),
+        migrations.RunPython(migrate_latlng_points, lambda apps, schema_editor: None),
+    ]

event_store/models.py

@@ -4,6 +4,7 @@
 
 from django.db import models
 from django.contrib.auth.models import User, Group
+from django.contrib.gis.db.models import PointField
 
 class Organization(models.Model):
     title = models.CharField(max_length=765)
@@ -111,6 +112,9 @@ class Event(models.Model):
     country = models.CharField(max_length=765, null=True, blank=True)
     longitude = models.FloatField(null=True, blank=True)
     latitude = models.FloatField(null=True, blank=True)
+
+    point = PointField(null=True, blank=True) # geo(lng, lat)
+
     title = models.CharField(max_length=765)
 
     starts_at = models.DateTimeField(null=True, blank=True, db_index=True)

eventroller/settings.py

@@ -94,7 +94,8 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
+        #'ENGINE': 'django.db.backends.sqlite3',
+        'ENGINE': 'django.contrib.gis.db.backends.spatialite',
         'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
     }
 }

requirements.txt

@@ -6,6 +6,7 @@ Django==1.10
 git+https://github.com/tzangms/django-bootstrap-form.git@5fff56f715bd9f2f29793f6a5a87baa1be25e409#egg=django-bootstrap-form
 -e git+https://github.com/MoveOnOrg/huerta.git@master#egg=huerta
 -e git+https://github.com/MoveOnOrg/python-actionkit.git@public-master#egg=python-actionkit
+-e git+https://github.com/MoveOnOrg/django-odata.git@master#egg=odata
 
 mysqlclient==1.3.7
 psycopg2==2.7.1

reviewer/migrations/0004_auto_20170625_2222.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.1 on 2017-06-25 22:22
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('event_store', '0009_auto_20170625_2222'),
+        ('auth', '0008_alter_user_username_max_length'),
+        ('reviewer', '0003_reviewlog_subject'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='reviewgroup',
+            unique_together=set([('organization', 'group')]),
+        ),
+    ]