From c6e8ba9caaeb686fd52d325a6985bcc9aead4919 Mon Sep 17 00:00:00 2001 From: Zach Wolfenbarger Date: Wed, 22 May 2024 16:02:05 -0500 Subject: [PATCH] Updates to BatchAggregation & tests --- panoptes_aggregation/batch_aggregation.py | 74 +++++++++++--- .../tests/batch_aggregation/cls_export.csv | 8 ++ .../test_batch_aggregation.py | 99 +++++++++++++++---- .../tests/batch_aggregation/wf_export.csv | 7 ++ pyproject.toml | 2 + 5 files changed, 159 insertions(+), 31 deletions(-) create mode 100644 panoptes_aggregation/tests/batch_aggregation/cls_export.csv create mode 100644 panoptes_aggregation/tests/batch_aggregation/wf_export.csv diff --git a/panoptes_aggregation/batch_aggregation.py b/panoptes_aggregation/batch_aggregation.py index 1eff9584..8db4e574 100644 --- a/panoptes_aggregation/batch_aggregation.py +++ b/panoptes_aggregation/batch_aggregation.py @@ -3,12 +3,14 @@ import pandas as pd import os import urllib3 -from os import getenv +from shutil import make_archive +import uuid + +from azure.storage.blob import BlobServiceClient from panoptes_client import Panoptes, Project, Workflow -from panoptes_aggregation.workflow_config import workflow_extractor_config, workflow_reducer_config +from panoptes_aggregation.workflow_config import workflow_extractor_config from panoptes_aggregation.scripts import batch_utils -from panoptes_client.panoptes import PanoptesAPIException import logging panoptes_client_logger = logging.getLogger('panoptes_client').setLevel(logging.ERROR) @@ -20,15 +22,34 @@ @celery.task(name="run_aggregation") def run_aggregation(project_id, workflow_id, user_id): ba = BatchAggregator(project_id, workflow_id, user_id) - exports = ba.save_exports() + ba.save_exports() + wf_df = ba.process_wf_export(ba.wf_csv) cls_df = ba.process_cls_export(ba.cls_csv) extractor_config = workflow_extractor_config(ba.tasks) extracted_data = batch_utils.batch_extract(cls_df, extractor_config) - reducer_config = workflow_reducer_config(extractor_config) - reduced_data = batch_utils.batch_reduce(extracted_data, reducer_config) + batch_standard_reducers = { + 'question_extractor': ['question_reducer', 'question_consensus_reducer'], + 'survey_extractor': ['survey_reducer'] + } + + for task_type, extract_df in extracted_data.items(): + extract_df.to_csv(f'{ba.output_path}/{ba.workflow_id}_{task_type}.csv') + reducer_list = batch_standard_reducers[task_type] + reduced_data = {} + + for reducer in reducer_list: + # This is an override. The workflow_reducer_config method returns a config object + # that is incompatible with the batch_utils batch_reduce method + reducer_config = {'reducer_config': {reducer: {}}} + reduced_data[reducer] = batch_utils.batch_reduce(extract_df, reducer_config) + filename = f'{ba.output_path}/{ba.workflow_id}_reductions.csv' + reduced_data[reducer].to_csv(filename, mode='a') + ba.upload_files() + + # hit up panoptes, let em know you're done class BatchAggregator: """ @@ -39,20 +60,26 @@ def __init__(self, project_id, workflow_id, user_id): self.project_id = project_id self.workflow_id = workflow_id self.user_id = user_id + self._generate_uuid() self._connect_api_client() def save_exports(self): + self.output_path = f'tmp/{self.workflow_id}' + os.mkdir(self.output_path) + cls_export = Workflow(self.workflow_id).describe_export('classifications') full_cls_url = cls_export['media'][0]['src'] + cls_file = f'{self.output_path}/{self.workflow_id}_cls_export.csv' + self._download_export(full_cls_url, cls_file) + wf_export = Project(self.project_id).describe_export('workflows') full_wf_url = wf_export['media'][0]['src'] - cls_file = f'tmp/{self.workflow_id}_cls_export.csv' - self._download_export(full_cls_url, cls_file) - wf_file = f'tmp/{self.project_id}_workflow_export.csv' + wf_file = f'{self.output_path}/{self.workflow_id}_workflow_export.csv' self._download_export(full_wf_url, wf_file) + self.cls_csv = cls_file self.wf_csv = wf_file - return {'cls_csv': cls_file, 'wf_csv': wf_file} + return {'classifications': cls_file, 'workflows': wf_file} def process_wf_export(self, wf_csv): self.wf_df = pd.read_csv(wf_csv) @@ -68,6 +95,27 @@ def process_cls_export(self, cls_csv): self.cls_df = cls_df.query(f'workflow_version == {self.workflow_version}') return self.cls_df + def connect_blob_storage(self): + connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING') + self.blob_service_client = BlobServiceClient.from_connection_string(connect_str) + self.blob_service_client.create_container(name=self.id) + + def upload_file_to_storage(self, container_name, filepath): + blob = filepath.split('/')[-1] + blob_client = self.blob_service_client.get_blob_client(container=container_name, blob=blob) + with open(file=filepath, mode="rb") as data: + blob_client.upload_blob(data, overwrite=True) + + def upload_files(self): + self.connect_blob_storage() + reductions_file = f'{self.output_path}/{self.workflow_id}_reductions.csv' + self.upload_file_to_storage(self.id, reductions_file) + zipfile = make_archive(f'tmp/{self.id}', 'zip', self.output_path) + self.upload_file_to_storage(self.id, zipfile) + + def _generate_uuid(self): + self.id = uuid.uuid4().hex + def _download_export(self, url, filepath): http = urllib3.PoolManager() r = http.request('GET', url, preload_content=False) @@ -82,7 +130,7 @@ def _download_export(self, url, filepath): def _connect_api_client(self): # connect to the API only once for this function request Panoptes.connect( - endpoint=getenv('PANOPTES_URL', 'https://panoptes.zooniverse.org/'), - client_id=getenv('PANOPTES_CLIENT_ID'), - client_secret=getenv('PANOPTES_CLIENT_SECRET') + endpoint=os.getenv('PANOPTES_URL', 'https://panoptes.zooniverse.org/'), + client_id=os.getenv('PANOPTES_CLIENT_ID'), + client_secret=os.getenv('PANOPTES_CLIENT_SECRET') ) diff --git a/panoptes_aggregation/tests/batch_aggregation/cls_export.csv b/panoptes_aggregation/tests/batch_aggregation/cls_export.csv new file mode 100644 index 00000000..dd2ed4e4 --- /dev/null +++ b/panoptes_aggregation/tests/batch_aggregation/cls_export.csv @@ -0,0 +1,8 @@ +classification_id,user_name,user_id,user_ip,workflow_id,workflow_name,workflow_version,created_at,gold_standard,expert,metadata,annotations,subject_data,subject_ids +543695319,not-logged-in-b644753d0e3948f81dc2,,b644753d0e3948f81dc2,10,Superluminous Supernovae,16.55,2024-02-23 16:17:42 UTC,,,"{""source"":""api"",""session"":""7a1f4a17d190291faa1824be3b3febf1d8b77a4f2d25dd6f191f76ef335684bf"",""viewport"":{""width"":1710,""height"":948},""started_at"":""2024-02-23T16:16:35.085Z"",""user_agent"":""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"",""utc_offset"":""18000"",""finished_at"":""2024-02-23T16:17:42.334Z"",""live_project"":true,""interventions"":{""opt_in"":false,""messageShown"":false},""user_language"":""en"",""subject_dimensions"":[{""clientWidth"":558,""clientHeight"":419,""naturalWidth"":1200,""naturalHeight"":900},{""clientWidth"":150,""clientHeight"":150,""naturalWidth"":300,""naturalHeight"":300}],""subject_selection_state"":{""retired"":false,""selected_at"":""2024-02-23T16:16:35.003Z"",""already_seen"":false,""selection_state"":""normal"",""finished_workflow"":false,""user_has_finished_workflow"":false},""workflow_translation_id"":""28176""}","[{""task"":""T0"",""task_label"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""value"":""Yes""},{""task"":""T1"",""task_label"":""Is the cross-hair in the image close to a faint, fuzzy galaxy?"",""value"":""Yes""}]","{""96588114"":{""retired"":{""id"":125510348,""workflow_id"":10,""classifications_count"":10,""created_at"":""2024-02-21T09:44:28.745Z"",""updated_at"":""2024-02-23T16:17:42.488Z"",""retired_at"":""2024-02-23T16:17:42.479Z"",""subject_id"":96588114,""retirement_reason"":""classification_count""},""ramean"":123.7681641625,""ZTF_URL"":""https://lasair-ztf.lsst.ac.uk/objects/ZTF23abjrdem"",""decmean"":57.182124325000004,""objectId"":""ZTF23abjrdem""}}",96588114 +543695340,not-logged-in-b644753d0e3948f81dc2,,b644753d0e3948f81dc2,10,Superluminous Supernovae,16.55,2024-02-23 16:17:48 UTC,,,"{""source"":""api"",""session"":""7a1f4a17d190291faa1824be3b3febf1d8b77a4f2d25dd6f191f76ef335684bf"",""viewport"":{""width"":1710,""height"":948},""started_at"":""2024-02-23T16:17:42.351Z"",""user_agent"":""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"",""utc_offset"":""18000"",""finished_at"":""2024-02-23T16:17:48.539Z"",""live_project"":true,""interventions"":{""opt_in"":false,""messageShown"":false},""user_language"":""en"",""subject_dimensions"":[{""clientWidth"":558,""clientHeight"":419,""naturalWidth"":1200,""naturalHeight"":900},{""clientWidth"":300,""clientHeight"":300,""naturalWidth"":300,""naturalHeight"":300}],""subject_selection_state"":{""retired"":false,""selected_at"":""2024-02-23T16:16:35.003Z"",""already_seen"":false,""selection_state"":""normal"",""finished_workflow"":false,""user_has_finished_workflow"":false},""workflow_translation_id"":""28176""}","[{""task"":""T0"",""task_label"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""value"":""No""}]","{""96588105"":{""retired"":{""id"":125510263,""workflow_id"":10,""classifications_count"":10,""created_at"":""2024-02-21T09:43:24.074Z"",""updated_at"":""2024-02-23T16:17:48.696Z"",""retired_at"":""2024-02-23T16:17:48.686Z"",""subject_id"":96588105,""retirement_reason"":""classification_count""},""ramean"":157.50762885625,""ZTF_URL"":""https://lasair-ztf.lsst.ac.uk/objects/ZTF22abycniv"",""decmean"":8.21724599375,""objectId"":""ZTF22abycniv""}}",96588105 +543695374,not-logged-in-b644753d0e3948f81dc2,,b644753d0e3948f81dc2,10,Superluminous Supernovae,16.55,2024-02-23 16:18:02 UTC,,,"{""source"":""api"",""session"":""7a1f4a17d190291faa1824be3b3febf1d8b77a4f2d25dd6f191f76ef335684bf"",""viewport"":{""width"":1710,""height"":948},""started_at"":""2024-02-23T16:17:48.559Z"",""user_agent"":""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"",""utc_offset"":""18000"",""finished_at"":""2024-02-23T16:18:02.264Z"",""live_project"":true,""interventions"":{""opt_in"":false,""messageShown"":false},""user_language"":""en"",""subject_dimensions"":[{""clientWidth"":558,""clientHeight"":419,""naturalWidth"":1200,""naturalHeight"":900},{""clientWidth"":300,""clientHeight"":300,""naturalWidth"":300,""naturalHeight"":300}],""subject_selection_state"":{""retired"":false,""selected_at"":""2024-02-23T16:16:35.003Z"",""already_seen"":false,""selection_state"":""normal"",""finished_workflow"":false,""user_has_finished_workflow"":false},""workflow_translation_id"":""28176""}","[{""task"":""T0"",""task_label"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""value"":""Yes""},{""task"":""T1"",""task_label"":""Is the cross-hair in the image close to a faint, fuzzy galaxy?"",""value"":""No""}]","{""96588126"":{""retired"":{""id"":125510270,""workflow_id"":10,""classifications_count"":10,""created_at"":""2024-02-21T09:43:25.834Z"",""updated_at"":""2024-02-23T16:18:02.396Z"",""retired_at"":""2024-02-23T16:18:02.389Z"",""subject_id"":96588126,""retirement_reason"":""classification_count""},""ramean"":98.49884808888889,""ZTF_URL"":""https://lasair-ztf.lsst.ac.uk/objects/ZTF22abfnkve"",""decmean"":58.67660070000001,""objectId"":""ZTF22abfnkve""}}",96588126 +543695390,not-logged-in-b644753d0e3948f81dc2,,b644753d0e3948f81dc2,10,Superluminous Supernovae,16.55,2024-02-23 16:18:09 UTC,,,"{""source"":""api"",""session"":""7a1f4a17d190291faa1824be3b3febf1d8b77a4f2d25dd6f191f76ef335684bf"",""viewport"":{""width"":1710,""height"":948},""started_at"":""2024-02-23T16:18:02.283Z"",""user_agent"":""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"",""utc_offset"":""18000"",""finished_at"":""2024-02-23T16:18:09.532Z"",""live_project"":true,""interventions"":{""opt_in"":false,""messageShown"":false},""user_language"":""en"",""subject_dimensions"":[{""clientWidth"":558,""clientHeight"":419,""naturalWidth"":1200,""naturalHeight"":900},{""clientWidth"":300,""clientHeight"":300,""naturalWidth"":300,""naturalHeight"":300}],""subject_selection_state"":{""retired"":false,""selected_at"":""2024-02-23T16:16:35.003Z"",""already_seen"":false,""selection_state"":""normal"",""finished_workflow"":false,""user_has_finished_workflow"":false},""workflow_translation_id"":""28176""}","[{""task"":""T0"",""task_label"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""value"":""No""}]","{""96588128"":{""retired"":{""id"":125510290,""workflow_id"":10,""classifications_count"":10,""created_at"":""2024-02-21T09:43:36.461Z"",""updated_at"":""2024-02-23T16:18:09.674Z"",""retired_at"":""2024-02-23T16:18:09.667Z"",""subject_id"":96588128,""retirement_reason"":""classification_count""},""ramean"":41.573462775,""ZTF_URL"":""https://lasair-ztf.lsst.ac.uk/objects/ZTF23aavvcjd"",""decmean"":-5.001660237499999,""objectId"":""ZTF23aavvcjd""}}",96588128 +543695425,not-logged-in-b644753d0e3948f81dc2,,b644753d0e3948f81dc2,10,Superluminous Supernovae,16.55,2024-02-23 16:18:24 UTC,,,"{""source"":""api"",""session"":""7a1f4a17d190291faa1824be3b3febf1d8b77a4f2d25dd6f191f76ef335684bf"",""viewport"":{""width"":1710,""height"":948},""started_at"":""2024-02-23T16:18:09.551Z"",""user_agent"":""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"",""utc_offset"":""18000"",""finished_at"":""2024-02-23T16:18:24.225Z"",""live_project"":true,""interventions"":{""opt_in"":false,""messageShown"":false},""user_language"":""en"",""subject_dimensions"":[{""clientWidth"":558,""clientHeight"":419,""naturalWidth"":1200,""naturalHeight"":900},{""clientWidth"":300,""clientHeight"":300,""naturalWidth"":300,""naturalHeight"":300}],""subject_selection_state"":{""retired"":false,""selected_at"":""2024-02-23T16:16:35.003Z"",""already_seen"":false,""selection_state"":""normal"",""finished_workflow"":false,""user_has_finished_workflow"":false},""workflow_translation_id"":""28176""}","[{""task"":""T0"",""task_label"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""value"":""Yes""},{""task"":""T1"",""task_label"":""Is the cross-hair in the image close to a faint, fuzzy galaxy?"",""value"":""No""}]","{""96588109"":{""retired"":{""id"":125510335,""workflow_id"":10,""classifications_count"":10,""created_at"":""2024-02-21T09:44:14.501Z"",""updated_at"":""2024-02-23T16:18:24.390Z"",""retired_at"":""2024-02-23T16:18:24.378Z"",""subject_id"":96588109,""retirement_reason"":""classification_count""},""ramean"":11.719328585714285,""ZTF_URL"":""https://lasair-ztf.lsst.ac.uk/objects/ZTF23aatzhso"",""decmean"":42.02810038571429,""objectId"":""ZTF23aatzhso""}}",96588109 +543695436,not-logged-in-b644753d0e3948f81dc2,,b644753d0e3948f81dc2,10,Superluminous Supernovae,16.55,2024-02-23 16:18:27 UTC,,,"{""source"":""api"",""session"":""7a1f4a17d190291faa1824be3b3febf1d8b77a4f2d25dd6f191f76ef335684bf"",""viewport"":{""width"":1710,""height"":948},""started_at"":""2024-02-23T16:18:24.243Z"",""user_agent"":""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"",""utc_offset"":""18000"",""finished_at"":""2024-02-23T16:18:27.892Z"",""live_project"":true,""interventions"":{""opt_in"":false,""messageShown"":false},""user_language"":""en"",""subject_dimensions"":[{""clientWidth"":558,""clientHeight"":419,""naturalWidth"":1200,""naturalHeight"":900},{""clientWidth"":300,""clientHeight"":300,""naturalWidth"":300,""naturalHeight"":300}],""subject_selection_state"":{""retired"":false,""selected_at"":""2024-02-23T16:16:35.003Z"",""already_seen"":false,""selection_state"":""normal"",""finished_workflow"":false,""user_has_finished_workflow"":false},""workflow_translation_id"":""28176""}","[{""task"":""T0"",""task_label"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""value"":""No""}]","{""96588106"":{""retired"":{""id"":125510296,""workflow_id"":10,""classifications_count"":10,""created_at"":""2024-02-21T09:43:44.966Z"",""updated_at"":""2024-02-23T17:54:11.466Z"",""retired_at"":""2024-02-23T17:54:11.458Z"",""subject_id"":96588106,""retirement_reason"":""classification_count""},""ramean"":47.78652812,""ZTF_URL"":""https://lasair-ztf.lsst.ac.uk/objects/ZTF23aauyuay"",""decmean"":73.76492526000001,""objectId"":""ZTF23aauyuay""}}",96588106 +543695453,not-logged-in-b644753d0e3948f81dc2,,b644753d0e3948f81dc2,10,Superluminous Supernovae,16.55,2024-02-23 16:18:35 UTC,,,"{""source"":""api"",""session"":""7a1f4a17d190291faa1824be3b3febf1d8b77a4f2d25dd6f191f76ef335684bf"",""viewport"":{""width"":1710,""height"":948},""started_at"":""2024-02-23T16:18:27.902Z"",""user_agent"":""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"",""utc_offset"":""18000"",""finished_at"":""2024-02-23T16:18:35.478Z"",""live_project"":true,""interventions"":{""opt_in"":false,""messageShown"":false},""user_language"":""en"",""subject_dimensions"":[{""clientWidth"":558,""clientHeight"":419,""naturalWidth"":1200,""naturalHeight"":900},{""clientWidth"":300,""clientHeight"":300,""naturalWidth"":300,""naturalHeight"":300}],""subject_selection_state"":{""retired"":false,""selected_at"":""2024-02-23T16:16:35.003Z"",""already_seen"":false,""selection_state"":""normal"",""finished_workflow"":false,""user_has_finished_workflow"":false},""workflow_translation_id"":""28176""}","[{""task"":""T0"",""task_label"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""value"":""No""}]","{""96588131"":{""retired"":{""id"":125510305,""workflow_id"":10,""classifications_count"":10,""created_at"":""2024-02-21T09:43:48.500Z"",""updated_at"":""2024-02-23T18:31:31.686Z"",""retired_at"":""2024-02-23T18:31:31.677Z"",""subject_id"":96588131,""retirement_reason"":""classification_count""},""ramean"":158.6458489125,""ZTF_URL"":""https://lasair-ztf.lsst.ac.uk/objects/ZTF23absjgik"",""decmean"":-27.650916137499998,""objectId"":""ZTF23absjgik""}}",96588131 diff --git a/panoptes_aggregation/tests/batch_aggregation/test_batch_aggregation.py b/panoptes_aggregation/tests/batch_aggregation/test_batch_aggregation.py index 8f18a549..3e79914b 100644 --- a/panoptes_aggregation/tests/batch_aggregation/test_batch_aggregation.py +++ b/panoptes_aggregation/tests/batch_aggregation/test_batch_aggregation.py @@ -1,51 +1,114 @@ import unittest -from unittest.mock import patch, Mock, MagicMock +from unittest.mock import patch, Mock, MagicMock, call from panoptes_aggregation.scripts import batch_utils from panoptes_aggregation.batch_aggregation import run_aggregation from panoptes_aggregation import batch_aggregation as batch_agg +wf_export = f'panoptes_aggregation/tests/batch_aggregation/wf_export.csv' +cls_export = f'panoptes_aggregation/tests/batch_aggregation/cls_export.csv' + +@patch("panoptes_aggregation.batch_aggregation.BatchAggregator._connect_api_client", new=MagicMock()) class TestBatchAggregation(unittest.TestCase): @patch("panoptes_aggregation.batch_aggregation.workflow_extractor_config") - @patch("panoptes_aggregation.batch_aggregation.workflow_reducer_config") @patch("panoptes_aggregation.batch_aggregation.BatchAggregator") - def test_run_aggregation(self, mock_aggregator, mock_wf_red_conf, mock_wf_ext_conf): + def test_run_aggregation(self, mock_aggregator, mock_wf_ext_conf): mock_aggregator.process_wf_export.return_value = MagicMock() mock_aggregator.process_cls_export.return_value = MagicMock() - batch_utils.batch_extract = MagicMock() - batch_utils.batch_reduce = MagicMock() + + mock_df = MagicMock() + test_extracts = { 'question_extractor': mock_df } + batch_utils.batch_extract = MagicMock(return_value=test_extracts) + mock_reducer = MagicMock() + batch_utils.batch_reduce = mock_reducer + run_aggregation(1, 10, 100) mock_aggregator.assert_called_once_with(1, 10, 100) mock_wf_ext_conf.assert_called_once() - mock_wf_red_conf.assert_called_once() batch_utils.batch_extract.assert_called_once() - batch_utils.batch_reduce.assert_called_once() + mock_df.to_csv.assert_called() + batch_utils.batch_reduce.assert_called() + self.assertEqual(mock_reducer.call_count, 2) + + # The reducer's call list includes subsequent calls to to_csv, but the args are methods called on the mock + # rather than use the set values i.e. "" + # mock_aggregator.workflow_id = '10' + # mock_aggregator.output_path = 'tmp/10' + # mock_reducer.assert_has_calls([ + # call(mock_df, {'reducer_config': {'question_reducer': {}}}), + # call().to_csv('tmp/10/10_reducers.csv', mode='a'), + # call(mock_df, {'reducer_config': {'question_consensus_reducer': {}}}), + # call().to_csv('tmp/10/10_reducers.csv', mode='a'), + # ]) + # How do I test the specific instance of BatchAggregator rather than the mocked class? + # mock_aggregator.upload_files.assert_called_once() + + @patch("panoptes_aggregation.batch_aggregation.os.mkdir") @patch("panoptes_aggregation.batch_aggregation.Workflow") @patch("panoptes_aggregation.batch_aggregation.Project") @patch("panoptes_aggregation.batch_aggregation.Panoptes.connect") - def test_save_exports(self, mock_client, mock_project, mock_workflow): + def test_save_exports(self, mock_client, mock_project, mock_workflow, mock_mkdir): # Test that Panoptes calls are made and files are saved csv_dict = {'media': [ {'src': 'http://zooniverse.org/123.csv'} ] } mock_project.return_value.describe_export.return_value = csv_dict mock_workflow.return_value.describe_export.return_value = csv_dict ba = batch_agg.BatchAggregator(1, 10, 100) - batch_agg.BatchAggregator._download_export = MagicMock(side_effect=[f'./cls_export.csv', f'./wf_export.csv']) - expected_response = {'cls_csv': 'tmp/10_cls_export.csv', 'wf_csv': 'tmp/1_workflow_export.csv'} + batch_agg.BatchAggregator._download_export = MagicMock(side_effect=['./cls_export.csv', './wf_export.csv']) + mock_uuidgen = MagicMock(side_effect=ba._generate_uuid()) + ba._generate_uuid = mock_uuidgen + expected_response = {'classifications': 'tmp/10/10_cls_export.csv', 'workflows': 'tmp/10/10_workflow_export.csv'} + response = ba.save_exports() + + # Why do these mocked methods called in __init__ not get counted as called? + # They are def getting called as the attributes are set + # mock_uuidgen.assert_called_once() + # mock_client.assert_called_once() + self.assertEqual(response, expected_response) - mock_client.assert_called_once() + mock_mkdir.assert_called_once() mock_project.assert_called_once_with(1) mock_workflow.assert_called_once_with(10) mock_project.return_value.describe_export.assert_called_once_with('workflows') mock_workflow.return_value.describe_export.assert_called_once_with('classifications') def test_process_wf_export(self): - # Test that: - # the wf export is parsed - # the version instance vars are set - # dataframe is retuned - assert 1 == 1 + ba = batch_agg.BatchAggregator(1, 10, 100) + result = ba.process_wf_export(wf_export) + self.assertEqual(ba.wf_maj_version, 16) + self.assertEqual(ba.wf_min_version, 55) + self.assertEqual(ba.workflow_version, '16.55') + self.assertEqual(result.__class__.__name__, 'DataFrame') def test_process_cls_export(self): - # Test that the cls csv is parsed and a dataframe is returned - assert 1 == 1 + ba = batch_agg.BatchAggregator(1, 10, 100) + ba.workflow_version = '16.55' + result = ba.process_cls_export(cls_export) + self.assertEqual(result.__class__.__name__, 'DataFrame') + + @patch("panoptes_aggregation.batch_aggregation.BatchAggregator.connect_blob_storage") + @patch("panoptes_aggregation.batch_aggregation.make_archive") + def test_upload_files(self, archive_mock, client_mock): + zipped_mock = MagicMock() + archive_mock.return_value = zipped_mock + ba = batch_agg.BatchAggregator(1, 10, 100) + ba.upload_file_to_storage = MagicMock() + ba.output_path = 'tmp/10' + reductions_file = 'tmp/10/10_reductions.csv' + ba.upload_files() + client_mock.assert_called_once() + archive_mock.assert_called_once() + ba.upload_file_to_storage.assert_has_calls([call(ba.id, reductions_file), call(ba.id, zipped_mock)]) + + def test_upload_file_to_storage(self): + ba = batch_agg.BatchAggregator(1, 10, 100) + mock_client = MagicMock() + ba.blob_service_client = MagicMock(return_value=mock_client) + ba.upload_file_to_storage('container', cls_export) + mock_client.upload_blob.assert_called_once + + @patch("panoptes_aggregation.batch_aggregation.BlobServiceClient") + def test_connect_blob_storage(self, mock_client): + ba = batch_agg.BatchAggregator(1, 10, 100) + ba.connect_blob_storage() + ba.blob_service_client.create_container.assert_called_once_with(name=ba.id) diff --git a/panoptes_aggregation/tests/batch_aggregation/wf_export.csv b/panoptes_aggregation/tests/batch_aggregation/wf_export.csv new file mode 100644 index 00000000..14e7fe45 --- /dev/null +++ b/panoptes_aggregation/tests/batch_aggregation/wf_export.csv @@ -0,0 +1,7 @@ +workflow_id,display_name,version,active,classifications_count,pairwise,grouped,prioritized,primary_language,first_task,tutorial_subject_id,retired_set_member_subjects_count,tasks,retirement,aggregation,strings,minor_version +10,Superluminous Supernovae,14,true,105171,false,false,false,en,T0,,9056,"{""T0"":{""help"":""T0.help"",""type"":""single"",""answers"":[{""next"":""T1"",""label"":""T0.answers.0.label""},{""label"":""T0.answers.1.label""}],""question"":""T0.question""},""T1"":{""help"":""T1.help"",""type"":""single"",""answers"":[{""label"":""T1.answers.0.label""},{""label"":""T1.answers.1.label""}],""question"":""T1.question""}}","{""options"":{""count"":10},""criteria"":""classification_count""}",{},"{""T0.help"":""The lightcurve is the plot showing brightness over time. The numbers on the x-axis are in days. Look to see if the brightness has been increasing over a period of more than 20 days.\n\nThe blue diamonds represent how bright the supernova is in blue light (at around 450 nanometres)\nand red circles show how bright it is in red light (about 610 nanometres). Often supernovae start blue and\nbecome red as they cool.\n\nHere are some examples of lightcurves that have been rising for over 20 days:\n\n![ZTF20abobpcb_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/d04087a4-ea8d-48f2-b297-401a58a091dd.jpeg)\n\n![ZTF20aadcbvz_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/a8dde630-fcc6-4577-afbc-6c1b7f451857.jpeg)\n\nEven though we observed this object well into its decline, the start of it still has a rise more than 20 days long, so it is a good candidate.\n\nRemember that the \n"",""T1.help"":""Here are some examples of what a faint, fuzzy galaxy hosting a superluminous supernova looks like (Please note that the supernova is not visible in these images. The cross-hair marks where the candidate supernova was detected):\n\n![ZTF19abxekxi-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/928be57f-844b-4947-b30f-28e70dfeb365.png)\n\n![ZTF19abpbopt-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/1f125022-f2f8-411b-a1c4-b1d825d06d9f.png)"",""T0.question"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""T1.question"":""Is the cross-hair in the image close to a faint, fuzzy galaxy?"",""T0.answers.0.label"":""Yes"",""T0.answers.1.label"":""No"",""T1.answers.0.label"":""Yes"",""T1.answers.1.label"":""No""}",52 +10,Superluminous Supernovae,14,true,105171,false,false,false,en,T0,,9056,"{""T0"":{""help"":""T0.help"",""type"":""single"",""answers"":[{""next"":""T1"",""label"":""T0.answers.0.label""},{""label"":""T0.answers.1.label""}],""question"":""T0.question""},""T1"":{""help"":""T1.help"",""type"":""single"",""answers"":[{""label"":""T1.answers.0.label""},{""label"":""T1.answers.1.label""}],""question"":""T1.question""}}","{""options"":{""count"":10},""criteria"":""classification_count""}",{},"{""T0.help"":""The lightcurve is the plot showing brightness over time. The numbers on the x-axis are in days. Look to see if the brightness has been increasing over a period of more than 20 days.\n\nThe blue diamonds represent how bright the supernova is in blue light (at around 450 nanometres)\nand red circles show how bright it is in red light (about 610 nanometres). Often supernovae start blue and\nbecome red as they cool.\n\nHere are some examples of lightcurves that have been rising for over 20 days:\n\n![ZTF20abobpcb_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/d04087a4-ea8d-48f2-b297-401a58a091dd.jpeg)\n\n![ZTF20aadcbvz_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/a8dde630-fcc6-4577-afbc-6c1b7f451857.jpeg)\n\nEven though we observed this object well into its decline, the start of it still has a rise more than 20 days long, so it is a good candidate.\n\nRemember that the size of a 20-day period will change depending on the number of observations, so you need to look at the axis. \n"",""T1.help"":""Here are some examples of what a faint, fuzzy galaxy hosting a superluminous supernova looks like (Please note that the supernova is not visible in these images. The cross-hair marks where the candidate supernova was detected):\n\n![ZTF19abxekxi-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/928be57f-844b-4947-b30f-28e70dfeb365.png)\n\n![ZTF19abpbopt-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/1f125022-f2f8-411b-a1c4-b1d825d06d9f.png)"",""T0.question"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""T1.question"":""Is the cross-hair in the image close to a faint, fuzzy galaxy?"",""T0.answers.0.label"":""Yes"",""T0.answers.1.label"":""No"",""T1.answers.0.label"":""Yes"",""T1.answers.1.label"":""No""}",53 +10,Superluminous Supernovae,14,true,105171,false,false,false,en,T0,,9056,"{""T0"":{""help"":""T0.help"",""type"":""single"",""answers"":[{""next"":""T1"",""label"":""T0.answers.0.label""},{""label"":""T0.answers.1.label""}],""question"":""T0.question""},""T1"":{""help"":""T1.help"",""type"":""single"",""answers"":[{""label"":""T1.answers.0.label""},{""label"":""T1.answers.1.label""}],""question"":""T1.question""}}","{""options"":{""count"":10},""criteria"":""classification_count""}",{},"{""T0.help"":""The lightcurve is the plot showing brightness over time. The numbers on the x-axis are in days. Look to see if the brightness has been increasing over a period of more than 20 days.\n\nThe blue diamonds represent how bright the supernova is in blue light (at around 450 nanometres)and red circles show how bright it is in red light (about 610 nanometres). Often supernovae start blue and\nbecome red as they cool.\n\nHere are some examples of lightcurves that have been rising for over 20 days:\n\n![ZTF20abobpcb_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/d04087a4-ea8d-48f2-b297-401a58a091dd.jpeg)\n\n![ZTF20aadcbvz_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/a8dde630-fcc6-4577-afbc-6c1b7f451857.jpeg)\n\nEven though we observed this object well into its decline, the start of it still has a rise more than 20 days long, so it is a good candidate.\n\nRemember that the size of a 20-day period will change depending on the number of observations, so you need to look at the axis. \n"",""T1.help"":""Here are some examples of what a faint, fuzzy galaxy hosting a superluminous supernova looks like (Please note that the supernova is not visible in these images. The cross-hair marks where the candidate supernova was detected):\n\n![ZTF19abxekxi-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/928be57f-844b-4947-b30f-28e70dfeb365.png)\n\n![ZTF19abpbopt-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/1f125022-f2f8-411b-a1c4-b1d825d06d9f.png)"",""T0.question"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""T1.question"":""Is the cross-hair in the image close to a faint, fuzzy galaxy?"",""T0.answers.0.label"":""Yes"",""T0.answers.1.label"":""No"",""T1.answers.0.label"":""Yes"",""T1.answers.1.label"":""No""}",54 +10,Superluminous Supernovae,14,true,105171,false,false,false,en,T0,,9056,"{""T0"":{""help"":""T0.help"",""type"":""single"",""answers"":[{""next"":""T1"",""label"":""T0.answers.0.label""},{""label"":""T0.answers.1.label""}],""question"":""T0.question""},""T1"":{""help"":""T1.help"",""type"":""single"",""answers"":[{""label"":""T1.answers.0.label""},{""label"":""T1.answers.1.label""}],""question"":""T1.question""}}","{""options"":{""count"":10},""criteria"":""classification_count""}",{},"{""T0.help"":""The lightcurve is the plot showing brightness over time. The numbers on the x-axis are in days. Look to see if the brightness has been increasing over a period of more than 20 days.\n\nThe blue diamonds represent how bright the supernova is in blue light (at around 450 nanometres)and red circles show how bright it is in red light (about 610 nanometres). Often supernovae start blue and become red as they cool.\n\nHere are some examples of lightcurves that have been rising for over 20 days:\n\n![ZTF20abobpcb_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/d04087a4-ea8d-48f2-b297-401a58a091dd.jpeg)\n\n![ZTF20aadcbvz_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/a8dde630-fcc6-4577-afbc-6c1b7f451857.jpeg)\n\nEven though we observed this object well into its decline, the start of it still has a rise more than 20 days long, so it is a good candidate.\n\nRemember that the size of a 20-day period will change depending on the number of observations, so you need to look at the axis. \n"",""T1.help"":""Here are some examples of what a faint, fuzzy galaxy hosting a superluminous supernova looks like (Please note that the supernova is not visible in these images. The cross-hair marks where the candidate supernova was detected):\n\n![ZTF19abxekxi-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/928be57f-844b-4947-b30f-28e70dfeb365.png)\n\n![ZTF19abpbopt-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/1f125022-f2f8-411b-a1c4-b1d825d06d9f.png)"",""T0.question"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""T1.question"":""Is the cross-hair in the image close to a faint, fuzzy galaxy?"",""T0.answers.0.label"":""Yes"",""T0.answers.1.label"":""No"",""T1.answers.0.label"":""Yes"",""T1.answers.1.label"":""No""}",55 +10,Superluminous Supernovae,15,true,105171,false,false,false,en,T0,,9056,"{""T0"":{""help"":""T0.help"",""type"":""single"",""answers"":[{""next"":""T1"",""label"":""T0.answers.0.label""},{""label"":""T0.answers.1.label""}],""question"":""T0.question"",""required"":true},""T1"":{""help"":""T1.help"",""type"":""single"",""answers"":[{""label"":""T1.answers.0.label""},{""label"":""T1.answers.1.label""}],""question"":""T1.question""}}","{""options"":{""count"":10},""criteria"":""classification_count""}",{},"{""T0.help"":""The lightcurve is the plot showing brightness over time. The numbers on the x-axis are in days. Look to see if the brightness has been increasing over a period of more than 20 days.\n\nThe blue diamonds represent how bright the supernova is in blue light (at around 450 nanometres)and red circles show how bright it is in red light (about 610 nanometres). Often supernovae start blue and become red as they cool.\n\nHere are some examples of lightcurves that have been rising for over 20 days:\n\n![ZTF20abobpcb_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/d04087a4-ea8d-48f2-b297-401a58a091dd.jpeg)\n\n![ZTF20aadcbvz_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/a8dde630-fcc6-4577-afbc-6c1b7f451857.jpeg)\n\nEven though we observed this object well into its decline, the start of it still has a rise more than 20 days long, so it is a good candidate.\n\nRemember that the size of a 20-day period will change depending on the number of observations, so you need to look at the axis. \n"",""T1.help"":""Here are some examples of what a faint, fuzzy galaxy hosting a superluminous supernova looks like (Please note that the supernova is not visible in these images. The cross-hair marks where the candidate supernova was detected):\n\n![ZTF19abxekxi-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/928be57f-844b-4947-b30f-28e70dfeb365.png)\n\n![ZTF19abpbopt-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/1f125022-f2f8-411b-a1c4-b1d825d06d9f.png)"",""T0.question"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""T1.question"":""Is the cross-hair in the image close to a faint, fuzzy galaxy?"",""T0.answers.0.label"":""Yes"",""T0.answers.1.label"":""No"",""T1.answers.0.label"":""Yes"",""T1.answers.1.label"":""No""}",55 +10,Superluminous Supernovae,16,true,105171,false,false,false,en,T0,,9056,"{""T0"":{""help"":""T0.help"",""type"":""single"",""answers"":[{""next"":""T1"",""label"":""T0.answers.0.label""},{""label"":""T0.answers.1.label""}],""question"":""T0.question"",""required"":true},""T1"":{""help"":""T1.help"",""type"":""single"",""answers"":[{""label"":""T1.answers.0.label""},{""label"":""T1.answers.1.label""}],""question"":""T1.question"",""required"":true}}","{""options"":{""count"":10},""criteria"":""classification_count""}",{},"{""T0.help"":""The lightcurve is the plot showing brightness over time. The numbers on the x-axis are in days. Look to see if the brightness has been increasing over a period of more than 20 days.\n\nThe blue diamonds represent how bright the supernova is in blue light (at around 450 nanometres)and red circles show how bright it is in red light (about 610 nanometres). Often supernovae start blue and become red as they cool.\n\nHere are some examples of lightcurves that have been rising for over 20 days:\n\n![ZTF20abobpcb_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/d04087a4-ea8d-48f2-b297-401a58a091dd.jpeg)\n\n![ZTF20aadcbvz_light_curve.jpeg](https://panoptes-uploads.zooniverse.org/production/project_attached_image/a8dde630-fcc6-4577-afbc-6c1b7f451857.jpeg)\n\nEven though we observed this object well into its decline, the start of it still has a rise more than 20 days long, so it is a good candidate.\n\nRemember that the size of a 20-day period will change depending on the number of observations, so you need to look at the axis. \n"",""T1.help"":""Here are some examples of what a faint, fuzzy galaxy hosting a superluminous supernova looks like (Please note that the supernova is not visible in these images. The cross-hair marks where the candidate supernova was detected):\n\n![ZTF19abxekxi-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/928be57f-844b-4947-b30f-28e70dfeb365.png)\n\n![ZTF19abpbopt-image.png](https://panoptes-uploads.zooniverse.org/production/project_attached_image/1f125022-f2f8-411b-a1c4-b1d825d06d9f.png)"",""T0.question"":""**Has the lightcurve been rising for more than 20 days?**\n\nNOTE: The y-axis shows the magnitude (brightness) of the supernova. In astronomy smaller magnitudes are brighter!"",""T1.question"":""Is the cross-hair in the image close to a faint, fuzzy galaxy?"",""T0.answers.0.label"":""Yes"",""T0.answers.1.label"":""No"",""T1.answers.0.label"":""Yes"",""T1.answers.1.label"":""No""}",55 diff --git a/pyproject.toml b/pyproject.toml index a5be1a7c..28567368 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,8 @@ dependencies = [ [project.optional-dependencies] online = [ + "azure-identity>=1,<2", + "azure-storage-blob>=12,<13", "celery>=5.3,<5.4", "redis>=5,<6", "flower>2,<3",