files API

.envs/.local/.django

@@ -1,5 +1,8 @@
 # General
 # ------------------------------------------------------------------------------
+DATALAKE_STRICT_WHATS=False
+DATALAKE_STRICT_WORK_ID=False
 USE_DOCKER=yes
 IPYTHONDIR=/app/.ipython
 DJANGO_JWT_SECRET=ILIKEASCREThowlongdoesitNeedtTOBeHey
+DJANGO_FILE_STORAGE_PATH=/tmp/django-file-storage

Pipfile

@@ -23,6 +23,10 @@ python-jose = "*"
 tabulate = "*"
 django-environ = "*"
 python-dateutil = "*"
+boto3 = "*"
+datalake-common-dtkav = ">=0.29"
+py-multihash = "*"
+py-multibase = "*"
 
 [dev-packages]
 pylint = "*"

cli/psql

@@ -2,5 +2,5 @@
 COMPOSE_ENV=${COMPOSE_ENV:-local.yml}
 CMD='PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -U $POSTGRES_USER -p $POSTGRES_PORT $POSTGRES_DB'
 
-docker-compose -f $COMPOSE_ENV run --rm postgres \
+docker-compose -f $COMPOSE_ENV exec postgres \
     bash -c "$CMD"

config/settings/base.py

@@ -74,6 +74,7 @@
 LOCAL_APPS = [
     # Your stuff: custom apps go here
     'home',
+    'datalake',
 ]
 # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
@@ -260,6 +261,11 @@
 }
 
 
+# Datalake
+# ------------------------------------------------------------------------------
+DATALAKE_STRICT_WHATS = True
+DATALAKE_STRICT_WORK_ID = True
+
 # Your stuff...
 # ------------------------------------------------------------------------------
 EPHEM_DIR = os.path.abspath(os.path.join(APPS_DIR, 'ephemeris'))
@@ -279,3 +285,5 @@ def immutable_file_test(path, url):
 
 
 WHITENOISE_IMMUTABLE_FILE_TEST = immutable_file_test
+
+FILE_STORAGE_PATH = env.str('DJANGO_FILE_STORAGE_PATH')

config/settings/test.py

@@ -53,3 +53,4 @@
 
 # Your stuff...
 # ------------------------------------------------------------------------------
+FILE_STORAGE_PATH = 's3://bucketname/django-file-storage/'

local.yml

@@ -20,6 +20,7 @@ services:
     ports:
       - "8000:8000"
     command: /start
+    restart: always
 
   frontend:
     build:

missioncontrol/api.py

@@ -19,7 +19,7 @@ def object_does_not_exist(exception):
 
 
 def validation_error(exception):
-    problem = connexion.problem(400, "Validation Error", str(exception))
+    problem = connexion.problem(400, "Validation Error", exception.messages)
     return connexion.FlaskApi.get_response(problem)
 
 

missioncontrol/datalake/admin.py

@@ -0,0 +1,6 @@
+from django.contrib import admin
+from django import forms
+from . import models
+
+admin.site.register(models.What)
+admin.site.register(models.RelatedFile)

missioncontrol/datalake/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class DatalakeConfig(AppConfig):
+    name = 'datalake'

missioncontrol/datalake/migrations/0001_initial.py

@@ -0,0 +1,61 @@
+# Generated by Django 2.1.7 on 2019-03-20 06:35
+
+import datalake.models
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DatalakeFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('cid', models.CharField(max_length=33)),
+                ('what', models.TextField()),
+                ('where', models.TextField()),
+                ('path', models.TextField(blank=True, null=True)),
+                ('start', datalake.models.ISODateTimeField(default=django.utils.timezone.now, help_text='The time of the first event in the file. If instantaneous, set this and leave end as null')),
+                ('end', datalake.models.ISODateTimeField(blank=True, help_text='The time of the last event in the file. Can be blank if instantaneous file.', null=True)),
+                ('created', datalake.models.ISODateTimeField(auto_now_add=True)),
+                ('work_id', models.TextField(blank=True, null=True)),
+                ('version', models.IntegerField(choices=[(1, 1)])),
+            ],
+            options={
+                'ordering': ('-start',),
+            },
+            bases=(models.Model, datalake.models.Serializable),
+        ),
+        migrations.CreateModel(
+            name='RelatedFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('work_id', models.CharField(max_length=256)),
+                ('object_id', models.PositiveIntegerField()),
+                ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='What',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('what', models.CharField(max_length=128, unique=True)),
+            ],
+            bases=(models.Model, datalake.models.Serializable),
+        ),
+        migrations.AddField(
+            model_name='datalakefile',
+            name='_related_to',
+            field=models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, to='datalake.RelatedFile'),
+        ),
+    ]