From 654ef9b502dceea37b4627746b96ebc3e4113ad6 Mon Sep 17 00:00:00 2001 From: Tim-Oliver Husser Date: Fri, 13 Aug 2021 15:36:13 +0200 Subject: [PATCH 1/7] added cors headers --- pyobs_archive/settings.py | 5 +++++ requirements.txt | 1 + 2 files changed, 6 insertions(+) diff --git a/pyobs_archive/settings.py b/pyobs_archive/settings.py index dfb0687..c294d8e 100644 --- a/pyobs_archive/settings.py +++ b/pyobs_archive/settings.py @@ -40,6 +40,7 @@ 'crispy_forms', 'rest_framework', 'rest_framework.authtoken', + 'corsheaders', 'pyobs_archive.api', 'pyobs_archive.authentication', 'pyobs_archive.frontend' @@ -60,6 +61,7 @@ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', + 'corsheaders.middleware.CorsMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', @@ -138,6 +140,9 @@ STATIC_URL = '/static/' STATIC_ROOT = '/static/' +# allow access from other pages, e.g. portal +CORS_ALLOW_ALL_ORIGINS = True + # logging LOGGING = { 'version': 1, diff --git a/requirements.txt b/requirements.txt index 1652dd0..83bcc43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ numpy psycopg2-binary zipstream django-crispy-forms +django-cors-headers From 5763360607a0412b85918643b35696f7060585d5 Mon Sep 17 00:00:00 2001 From: Tim-Oliver Husser Date: Thu, 9 Sep 2021 10:49:32 +0200 Subject: [PATCH 2/7] always sort frames by id as second criterium --- pyobs_archive/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyobs_archive/api/views.py b/pyobs_archive/api/views.py index f1dca37..5d2d7d3 100644 --- a/pyobs_archive/api/views.py +++ b/pyobs_archive/api/views.py @@ -166,7 +166,7 @@ def frames_view(request): sort_string = ('' if order == 'asc' else '-') + sort # get response - data = Frame.objects.order_by(sort_string) + data = Frame.objects.order_by(sort_string, 'id') # filter data = filter_frames(data, request) From ce96360f30dfac8876e1cb30307f1c2ef10db6a4 Mon Sep 17 00:00:00 2001 From: Tim-Oliver Husser Date: Thu, 28 Apr 2022 13:42:16 +0200 Subject: [PATCH 3/7] command for deleting files --- .../api/management/commands/delete.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 pyobs_archive/api/management/commands/delete.py diff --git a/pyobs_archive/api/management/commands/delete.py b/pyobs_archive/api/management/commands/delete.py new file mode 100644 index 0000000..8a89cc3 --- /dev/null +++ b/pyobs_archive/api/management/commands/delete.py @@ -0,0 +1,45 @@ +import os + +from django.core.management.base import BaseCommand + +from pyobs_archive.api.models import Frame +from pyobs_archive import settings + + +class Command(BaseCommand): + help = 'Ingest images' + + def add_arguments(self, parser): + parser.add_argument('files', type=str, nargs='+', help='Names of files to delete') + + def handle(self, *args, files: list = None, **options): + to_delete = [] + for filename in files: + frames = Frame.objects.filter(basename=filename) + if len(frames) > 0: + to_delete.append(frames) + + if not to_delete: + return + + print("Images to delete:") + for d in to_delete: + print(" - " + d.basename) + reply = None + while reply not in 'yYnN': + reply = input("Delete files? [yn]") + + if reply not in 'yY': + return + + for d in to_delete: + # get filename + root = settings.ARCHIVE_ROOT + filename = os.path.join(root, frame.path, frame.basename + '.fits.fz') + + # delete file + os.remove(filename) + + # delete db entry + d.delete() + From a256ce7733b4af919e03fce2cefaebb450b7bc90 Mon Sep 17 00:00:00 2001 From: Tim-Oliver Husser Date: Thu, 28 Apr 2022 13:42:36 +0200 Subject: [PATCH 4/7] command for deleting files --- pyobs_archive/api/management/commands/delete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyobs_archive/api/management/commands/delete.py b/pyobs_archive/api/management/commands/delete.py index 8a89cc3..cd09b02 100644 --- a/pyobs_archive/api/management/commands/delete.py +++ b/pyobs_archive/api/management/commands/delete.py @@ -35,7 +35,7 @@ def handle(self, *args, files: list = None, **options): for d in to_delete: # get filename root = settings.ARCHIVE_ROOT - filename = os.path.join(root, frame.path, frame.basename + '.fits.fz') + filename = os.path.join(root, d.path, d.basename + '.fits.fz') # delete file os.remove(filename) From fd59021210c79891ac13d136a9d33c05b9400f31 Mon Sep 17 00:00:00 2001 From: Tim-Oliver Husser Date: Thu, 28 Apr 2022 13:44:04 +0200 Subject: [PATCH 5/7] added file --- .dockerignore | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6fc3f26 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,29 @@ +*.pyc +*.pyo +*.mo +*.db +*.css.map +*.egg-info +*.sql.gz +.cache +.project +.idea +.pydevproject +.idea/workspace.xml +.DS_Store +.git/ +.sass-cache +.vagrant/ +__pycache__ +dist +docs +env +venv +logs +src/{{ project_name }}/settings/local.py +src/node_modules +web/media +web/static/CACHE +stats +Dockerfile +LabCourse/local_settings.py \ No newline at end of file From 5e7e01b49f391b8c8e7d6aa57467c2cfd4bbf21f Mon Sep 17 00:00:00 2001 From: Tim-Oliver Husser Date: Thu, 28 Apr 2022 13:59:51 +0200 Subject: [PATCH 6/7] fixed bug --- pyobs_archive/api/management/commands/delete.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyobs_archive/api/management/commands/delete.py b/pyobs_archive/api/management/commands/delete.py index cd09b02..f17f1bf 100644 --- a/pyobs_archive/api/management/commands/delete.py +++ b/pyobs_archive/api/management/commands/delete.py @@ -7,7 +7,7 @@ class Command(BaseCommand): - help = 'Ingest images' + help = 'Delete images' def add_arguments(self, parser): parser.add_argument('files', type=str, nargs='+', help='Names of files to delete') @@ -17,7 +17,7 @@ def handle(self, *args, files: list = None, **options): for filename in files: frames = Frame.objects.filter(basename=filename) if len(frames) > 0: - to_delete.append(frames) + to_delete.extend(frames) if not to_delete: return @@ -25,7 +25,7 @@ def handle(self, *args, files: list = None, **options): print("Images to delete:") for d in to_delete: print(" - " + d.basename) - reply = None + reply = "X" while reply not in 'yYnN': reply = input("Delete files? [yn]") From cb37125516dd5a574e0d686e326c249efcc91f43 Mon Sep 17 00:00:00 2001 From: Tim-Oliver Husser Date: Wed, 15 Feb 2023 13:13:05 +0100 Subject: [PATCH 7/7] added new button for download all --- pyobs_archive/api/views.py | 57 +++++++++++++++++-- pyobs_archive/frontend/static/js/app.js | 19 +++++++ .../frontend/templates/archive/index.html | 8 ++- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/pyobs_archive/api/views.py b/pyobs_archive/api/views.py index 5d2d7d3..857fb45 100644 --- a/pyobs_archive/api/views.py +++ b/pyobs_archive/api/views.py @@ -296,9 +296,56 @@ def preview_view(request, frame_id): return HttpResponse(bio.getvalue(), content_type="image/png") -@api_view(['POST']) @permission_classes([IsAuthenticated]) def zip_view(request): + if request.method == 'POST': + return zip_view_post(request) + elif request.method == 'GET': + return zip_view_get(request) + else: + raise Http404 + + +def zip_view_post(request): + # get frames + frames = [] + for frame_id in request.POST.getlist('frame_ids[]'): + # get frame + frames.append(_frame(frame_id)) + + # download + return _download_zip(request, frames) + + +def zip_view_get(request): + # get offset and limit + try: + offset = int(request.GET.get('offset', default=0)) + limit = int(request.GET.get('limit', default=1000)) + except ValueError: + raise ParseError('Invalid values for offset/limit.') + + # limit to 1000 + limit = max(0, min(limit, 1000)) + offset = max(0, offset) + + # sort + sort = request.GET.get('sort', default='DATE_OBS') + order = request.GET.get('order', default='asc') + sort_string = ('' if order == 'asc' else '-') + sort + + # filter + data = filter_frames(Frame.objects, request) + + # and frames + root = settings.ARCHIVE_ROOT + frames = [(frame, os.path.join(root, frame.path, frame.basename + '.fits.fz')) for frame in data] + + # download + return _download_zip(request, frames) + + +def _download_zip(request, frames): # get archive root root = settings.ARCHIVE_ROOT @@ -309,12 +356,10 @@ def zip_view(request): zip_file = zipstream.ZipFile() # add files - for frame_id in request.POST.getlist('frame_ids[]'): - # get frame - frame, filename = _frame(frame_id) - + for frame, filename in frames: # add file to zip - zip_file.write(filename, arcname=os.path.join(archive_name, os.path.basename(filename))) + if os.path.exists(filename): + zip_file.write(filename, arcname=os.path.join(archive_name, os.path.basename(filename))) # create and return response response = StreamingHttpResponse(zip_file, content_type='application/zip') diff --git a/pyobs_archive/frontend/static/js/app.js b/pyobs_archive/frontend/static/js/app.js index 75085df..d7017da 100644 --- a/pyobs_archive/frontend/static/js/app.js +++ b/pyobs_archive/frontend/static/js/app.js @@ -59,6 +59,17 @@ $(function () { setRequestHeader(xhr); } }, + onLoadSuccess: function() { + // update download search button + let downloadSearchBtn = $('#downloadSearchBtn'); + let rows = this.totalRows; + downloadSearchBtn.html('Download all (' + rows + ')'); + downloadSearchBtn.attr('href', 'frames/zip?q=a' + buildQueryParms()); + }, + onCheck: on_check, + onUncheck: on_check, + onCheckAll: on_check, + onUncheckAll: on_check, totalField: 'count', dataField: 'results', pagination: true, @@ -117,6 +128,14 @@ $(function () { }] }); + function on_check() { + + // update download button + let downloadBtn = $('#downloadBtn'); + var rows = $('#table').bootstrapTable('getSelections').length; + downloadBtn.html('Download selected (' + rows + ')'); + } + $('#daterange').daterangepicker({ 'locale': { 'format': 'YYYY-MM-DD HH:mm' diff --git a/pyobs_archive/frontend/templates/archive/index.html b/pyobs_archive/frontend/templates/archive/index.html index 67e8303..22aca86 100644 --- a/pyobs_archive/frontend/templates/archive/index.html +++ b/pyobs_archive/frontend/templates/archive/index.html @@ -84,7 +84,13 @@
{% csrf_token %} - + +
+
+
+
+ {% csrf_token %} + Download all