Add models for running tasks.

Adds a TaskRun and a File model for storing results of task which have
been run.
Allows uploading of stdout / stderr to S3.

.envs/.local/.django

@@ -3,3 +3,4 @@
 USE_DOCKER=yes
 IPYTHONDIR=/app/.ipython
 DJANGO_JWT_SECRET=ILIKEASCREThowlongdoesitNeedtTOBeHey
+DJANGO_FILE_STORAGE_PATH=/tmp/django-file-storage

Pipfile

@@ -22,6 +22,7 @@ scipy = "*"
 python-jose = "*"
 tabulate = "*"
 django-environ = "*"
+boto3 = "*"
 
 [dev-packages]
 pylint = "*"

config/settings/base.py

@@ -279,3 +279,5 @@ def immutable_file_test(path, url):
 
 
 WHITENOISE_IMMUTABLE_FILE_TEST = immutable_file_test
+
+FILE_STORAGE_PATH = env.str('DJANGO_FILE_STORAGE_PATH')

config/settings/test.py

@@ -53,3 +53,4 @@
 
 # Your stuff...
 # ------------------------------------------------------------------------------
+FILE_STORAGE_PATH = 's3://bucketname/django-file-storage/'

local.yml

@@ -20,6 +20,7 @@ services:
     ports:
       - "8000:8000"
     command: /start
+    restart: always
 
   frontend:
     build:

missioncontrol/home/admin.py

@@ -64,3 +64,12 @@ class TaskStackAdmin(admin.ModelAdmin):
         'tasks'
     )
     list_filter = ('environment',)
+
+@admin.register(models.TaskRun)
+class TaskRunAdmin(admin.ModelAdmin):
+    list_display = (
+        'uuid',
+        'task',
+        'task_stack',
+        'task_pass',
+    )

missioncontrol/home/migrations/0021_auto_20190304_0300.py

@@ -0,0 +1,50 @@
+# Generated by Django 2.1.4 on 2019-03-04 03:00
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import home.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('home', '0020_merge_20190124_0028'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='S3File',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('cid', models.CharField(max_length=128, unique=True)),
+                ('what', models.TextField()),
+                ('where', models.TextField()),
+                ('path', models.TextField(blank=True, null=True)),
+                ('start', home.models.ISODateTimeField(default=django.utils.timezone.now, help_text='The time of the first event in the file. If instantaneous, set this and leave end as null')),
+                ('end', home.models.ISODateTimeField(blank=True, help_text='The time of the last event in the file. Can be blank if instantaneous file.', null=True)),
+                ('created', home.models.ISODateTimeField(auto_now_add=True)),
+            ],
+            bases=(models.Model, home.models.Serializable),
+        ),
+        migrations.CreateModel(
+            name='TaskRun',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('task', models.TextField()),
+                ('start_time', home.models.ISODateTimeField()),
+                ('end_time', home.models.ISODateTimeField()),
+                ('exit_code', models.IntegerField()),
+                ('task_pass', models.ForeignKey(db_column='pass', on_delete=django.db.models.deletion.PROTECT, to='home.Pass', to_field='uuid')),
+                ('task_stack', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='home.TaskStack', to_field='uuid')),
+            ],
+            bases=(models.Model, home.models.Serializable),
+        ),
+        migrations.AddField(
+            model_name='s3file',
+            name='task_run',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='files', to='home.TaskRun', to_field='uuid'),
+        ),
+    ]

missioncontrol/home/models.py

@@ -19,6 +19,7 @@
 from django.utils import timezone, dateformat
 from pytz import UTC
 from skyfield.api import Topos, EarthSatellite
+import boto3
 
 from v0.time import iso, utc
 
@@ -425,6 +426,111 @@ def __str__(self):
         return f"Pass: {self.uuid} - {self.satellite} - {self.groundstation} - {self.start_time}"
 
 
+
+
+class TaskRun(models.Model, Serializable):
+    uuid = models.UUIDField(default=uuid4, unique=True)
+    task = models.TextField()
+    task_stack = models.ForeignKey(TaskStack, on_delete=models.PROTECT, to_field='uuid')
+    task_pass = models.ForeignKey(Pass, on_delete=models.PROTECT, db_column='pass', to_field='uuid')
+    start_time = ISODateTimeField()
+    end_time = ISODateTimeField()
+    exit_code = models.IntegerField()
+
+    def to_dict(self):
+        # Because connexion expects the 'pass' key, but it's a keyword
+        retval = super().to_dict()
+        retval['pass'] = retval.pop('task_pass')
+        # TODO possibly generalize this higher using a property on `Meta`
+        retval['stdout'] = self.stdout
+        retval['stderr'] = self.stderr
+
+        return retval
+
+    def _get_file_cid_if_exists(self, what):
+        try:
+            f = self.files.get(what=what)
+        except S3File.DoesNotExist:
+            return None
+        return f.cid
+
+    @property
+    def stdout(self):
+        # return [x.to_dict() for x in self.files.all()]
+        return self._get_file_cid_if_exists('stdout')
+
+    @property
+    def stderr(self):
+        return self._get_file_cid_if_exists('stderr')
+
+    def __repr__(self):
+        print(self.task_pass)
+        return "<TaskRun: {uuid} - {task}>".format(**self.__dict__)
+
+    def __str__(self):
+        return self.__repr__()
+
+
+
+class S3File(models.Model, Serializable):
+    cid = models.CharField(max_length=128, unique=True)
+    what = models.TextField()
+    where = models.TextField()
+    path = models.TextField(null=True, blank=True)
+    start = ISODateTimeField(
+        help_text='The time of the first event in the file. '
+                  'If instantaneous, set this and leave end as null',
+        default=timezone.now)
+    end = ISODateTimeField(
+        help_text='The time of the last event in the file. '
+                  'Can be blank if instantaneous file.',
+        null=True, blank=True)
+    created = ISODateTimeField(auto_now_add=True)
+    task_run = models.ForeignKey(TaskRun, to_field='uuid', related_name='files',
+                                 on_delete=models.PROTECT, null=True, blank=True)
+
+    # s3://bucket/some_path
+    @property
+    def prefix(self):
+        path = settings.FILE_STORAGE_PATH
+        if path.startswith('s3://'):
+            return '/'.join(path.split('/')[3:])
+        raise NotImplementedError("Not yet implemented non s3 paths")
+
+    @property
+    def bucket(self):
+        path = settings.FILE_STORAGE_PATH
+        if path.startswith('s3://'):
+            return path.split('/')[2]
+        raise NotImplementedError("Not yet implemented non s3 paths")
+
+    @property
+    def key(self):
+        return f'{self.prefix}{self.cid}'
+
+    def get_download_url(self):
+        s3 = boto3.client('s3')
+        url = s3.generate_presigned_url(
+            ClientMethod='get_object',
+            Params={
+                'Bucket': self.bucket,
+                'Key': self.key,
+            }
+        )
+
+        return url
+
+    def get_upload_url(self):
+        s3 = boto3.client('s3')
+        post = s3.generate_presigned_post(
+            Bucket=self.bucket,
+            Key=self.key,
+        )
+
+        return post
+
+
+
 class CachedAccess(models.Model):
     # we store computed accesses by bucket_hash, where the hash is
     # hash(tle1, tle2, lat, lng, el, horizon_mask) + bucket_start + bucket_end