diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..07f79e3 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,36 @@ +name: PEP8 + +on: + push: + branches: + - master + pull_request: + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.11] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff + - name: Lint + run: | + ruff check --statistics *.py + ruff check --statistics apps/ + ruff check --ignore D205 tests/ + - name: Format + run: | + ruff format --check *.py + ruff format --check apps/ + ruff format --check tests/ diff --git a/.github/workflows/run_test.yml b/.github/workflows/run_test.yml new file mode 100644 index 0000000..20ed55f --- /dev/null +++ b/.github/workflows/run_test.yml @@ -0,0 +1,28 @@ +name: Sentinel + +on: + push: + branches: + - main + pull_request: + +jobs: + test-suite: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.11] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python dependencies + run: | + pip install --upgrade pip setuptools wheel + pip install -r requirements.txt + cd .. + - name: Run test suites + run: | + ./run_tests.sh --url ${{ secrets.APIURLDEV }} diff --git a/apps/routes/objects/api.py b/apps/routes/objects/api.py index 8f2535d..025f4c2 100644 --- a/apps/routes/objects/api.py +++ b/apps/routes/objects/api.py @@ -20,6 +20,7 @@ bp = Blueprint("objects", __name__) + # Enable CORS for this blueprint @bp.after_request def after_request(response): @@ -53,7 +54,7 @@ def after_request(response): { "name": "columns", "required": False, - "description": f"Comma-separated data columns to transfer. Default is all columns.", + "description": "Comma-separated data columns to transfer. Default is all columns.", }, { "name": "output-format", @@ -62,6 +63,7 @@ def after_request(response): }, ] + @bp.route("/api/v1/objects", methods=["GET"]) def return_object_arguments(): """Obtain information about retrieving object data""" diff --git a/apps/routes/objects/utils.py b/apps/routes/objects/utils.py index 71c474a..41a8fbe 100644 --- a/apps/routes/objects/utils.py +++ b/apps/routes/objects/utils.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pandas as pd +from numpy import array as nparray from apps.utils.utils import download_cutout from apps.utils.client import connect_to_hbase_table -from apps.utils.decoding import format_hbase_output +from apps.utils.decoding import format_hbase_output, hbase_to_dict def extract_object_data(payload: dict) -> pd.DataFrame: @@ -101,9 +102,9 @@ def extract_object_data(payload: dict) -> pd.DataFrame: else: colname = "b:cutout{}_stampData".format(cutout_kind) pdf[colname] = pdf[["i:objectId", "i:candid"]].apply( - lambda x: pd.Series( - [download_cutout(x.iloc[0], x.iloc[1], cutout_kind)] - ), + lambda x: pd.Series([ + download_cutout(x.iloc[0], x.iloc[1], cutout_kind) + ]), axis=1, ) @@ -145,12 +146,10 @@ def extract_object_data(payload: dict) -> pd.DataFrame: if "i:jd" in pdfUP.columns: # workaround -- see https://github.com/astrolabsoftware/fink-science-portal/issues/216 - mask = np.array( - [ - False if float(i) in pdf["i:jd"].to_numpy() else True - for i in pdfUP["i:jd"].to_numpy() - ] - ) + mask = nparray([ + False if float(i) in pdf["i:jd"].to_numpy() else True + for i in pdfUP["i:jd"].to_numpy() + ]) pdfUP = pdfUP[mask] # Hacky way to avoid converting concatenated column to float @@ -171,4 +170,4 @@ def extract_object_data(payload: dict) -> pd.DataFrame: client.close() - return pdf \ No newline at end of file + return pdf diff --git a/apps/routes/template/api.py b/apps/routes/template/api.py index 9166f2e..320e10c 100644 --- a/apps/routes/template/api.py +++ b/apps/routes/template/api.py @@ -1,12 +1,12 @@ # Copyright 2024 AstroLab Software # Author: Julien Peloton -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,6 +20,7 @@ bp = Blueprint("template", __name__) + # Enable CORS for this blueprint @bp.after_request def after_request(response): @@ -42,6 +43,7 @@ def after_request(response): }, ] + @bp.route("/api/v1/template", methods=["GET"]) def return_template_arguments(): """Obtain information about retrieving object data""" @@ -51,6 +53,7 @@ def return_template_arguments(): else: return jsonify({"args": ARGS}) + @bp.route("/api/v1/template", methods=["POST"]) def return_template(payload=None): """Retrieve object data""" @@ -70,5 +73,3 @@ def return_template(payload=None): output_format = payload.get("output-format", "json") return send_tabular_data(out, output_format) - - diff --git a/apps/routes/template/utils.py b/apps/routes/template/utils.py index d5d3705..89e5767 100644 --- a/apps/routes/template/utils.py +++ b/apps/routes/template/utils.py @@ -14,5 +14,6 @@ # limitations under the License. import pandas as pd + def my_function(payload): return pd.DataFrame({payload["arg1"]: [1, 2, 3]}) diff --git a/apps/utils/utils.py b/apps/utils/utils.py index 4ded888..c23f0d8 100644 --- a/apps/utils/utils.py +++ b/apps/utils/utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Various utilities""" + import io import json import yaml @@ -23,6 +24,7 @@ from astropy.table import Table from astropy.io import votable + def extract_configuration(filename): """Extract user defined configuration @@ -43,6 +45,7 @@ def extract_configuration(filename): config["APIURL"] = "http://" + config["HOST"] + ":" + str(config["PORT"]) return config + def download_cutout(objectId, candid, kind): """ """ config = extract_configuration("config.yml") diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000..526cf0c --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright 2024 AstroLab Software +# Author: Julien Peloton +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +message_help=""" +Run the test suite of the modules\n\n +Usage:\n + \t./run_tests.sh [--url]\n\n + +--url is the Science Portal URL you would like to test against. +""" +# Grab the command line arguments +while [ "$#" -gt 0 ]; do + case "$1" in + --url) + URL="$2" + shift 2 + ;; + -h) + echo -e $message_help + exit + ;; + esac +done + +if [[ -f $URL ]]; then + echo "You need to specify an URL" $URL + exit +fi + +# Run the test suite on the utilities +cd tests +for filename in ./*.py +do + echo $filename + # Run test suite + python $filename $URL +done diff --git a/tests/api_single_object_test.py b/tests/api_single_object_test.py new file mode 100644 index 0000000..480481f --- /dev/null +++ b/tests/api_single_object_test.py @@ -0,0 +1,224 @@ +# Copyright 2022-2024 AstroLab Software +# Author: Julien Peloton +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import requests +import pandas as pd +import numpy as np + +import io +import sys + +APIURL = sys.argv[1] + +# Implement random name generator +OID = "ZTF21abfmbix" + + +def get_an_object( + oid="ZTF21abfmbix", + output_format="json", + columns="*", + withupperlim=False, + withcutouts=False, + cutout_kind=None, +): + """Query an object from the Science Portal using the Fink REST API""" + payload = { + "objectId": oid, + "columns": columns, + "output-format": output_format, + "withupperlim": withupperlim, + "withcutouts": withcutouts, + } + + if cutout_kind is not None: + payload.update({"cutout-kind": cutout_kind}) + + r = requests.post("{}/api/v1/objects".format(APIURL), json=payload) + + assert r.status_code == 200, r.content + + if output_format == "json": + # Format output in a DataFrame + pdf = pd.read_json(io.BytesIO(r.content)) + elif output_format == "csv": + pdf = pd.read_csv(io.BytesIO(r.content)) + elif output_format == "parquet": + pdf = pd.read_parquet(io.BytesIO(r.content)) + + return pdf + + +def test_single_object() -> None: + """ + Examples + -------- + >>> test_single_object() + """ + pdf = get_an_object(oid=OID) + + assert not pdf.empty + + +def test_single_object_csv() -> None: + """ + Examples + -------- + >>> test_single_object_csv() + """ + pdf = get_an_object(oid=OID, output_format="csv") + + assert not pdf.empty + + +def test_single_object_parquet() -> None: + """ + Examples + -------- + >>> test_single_object_parquet() + """ + pdf = get_an_object(oid=OID, output_format="parquet") + + assert not pdf.empty + + +def test_column_selection() -> None: + """ + Examples + -------- + >>> test_column_selection() + """ + pdf = get_an_object(oid=OID, columns="i:jd,i:magpsf") + + assert len(pdf.columns) == 2, "I count {} columns".format(len(pdf.columns)) + + +def test_column_length() -> None: + """ + Examples + -------- + >>> test_column_length() + """ + pdf = get_an_object(oid=OID) + + assert len(pdf.columns) == 129, "I count {} columns".format(len(pdf.columns)) + + +def test_withupperlim() -> None: + """ + Examples + -------- + >>> test_withupperlim() + """ + pdf = get_an_object(oid=OID, withupperlim=True) + assert "d:tag" in pdf.columns + + +def test_withcutouts() -> None: + """ + Examples + -------- + >>> test_withcutouts() + """ + pdf = get_an_object(oid=OID, withcutouts=True) + + assert isinstance(pdf["b:cutoutScience_stampData"].to_numpy()[0], list) + assert isinstance(pdf["b:cutoutTemplate_stampData"].to_numpy()[0], list) + assert isinstance(pdf["b:cutoutDifference_stampData"].to_numpy()[0], list) + + +def test_withcutouts_single_field() -> None: + """ + Examples + -------- + >>> test_withcutouts_single_field() + """ + pdf = get_an_object(oid=OID, withcutouts=True, cutout_kind="Science") + + assert isinstance(pdf["b:cutoutScience_stampData"].to_numpy()[0], list) + assert "b:cutoutTemplate_stampData" not in pdf.columns + + +def test_formatting() -> None: + """ + Examples + -------- + >>> test_formatting() + """ + pdf = get_an_object(oid=OID) + + # stupid python cast... + assert isinstance(pdf["i:fid"].to_numpy()[0], np.int64), type( + pdf["i:fid"].to_numpy()[0] + ) + assert isinstance(pdf["i:magpsf"].to_numpy()[0], np.double), type( + pdf["i:magpsf"].to_numpy()[0] + ) + + +def test_misc() -> None: + """ + Examples + -------- + >>> test_misc() + """ + pdf = get_an_object(oid=OID) + assert np.all(pdf["i:fid"].to_numpy() > 0) + assert np.all(pdf["i:magpsf"].to_numpy() > 6) + + +def test_bad_request() -> None: + """ + Examples + -------- + >>> test_bad_request() + """ + pdf = get_an_object(oid="ldfksjflkdsjf") + + assert pdf.empty + + +def test_multiple_objects() -> None: + """ + Examples + -------- + >>> test_multiple_objects() + """ + OIDS_ = ["ZTF21abfmbix", "ZTF21aaxtctv", "ZTF21abfaohe"] + OIDS = ",".join(OIDS_) + pdf = get_an_object(oid=OIDS) + + n_oids = len(np.unique(pdf.groupby("i:objectId").count()["i:ra"])) + assert n_oids == 3 + + n_oids_single = 0 + len_object = 0 + for oid in OIDS_: + pdf_ = get_an_object(oid=oid) + n_oid = len(np.unique(pdf_.groupby("i:objectId").count()["i:ra"])) + n_oids_single += n_oid + len_object += len(pdf_) + + assert n_oids == n_oids_single, "{} is not equal to {}".format( + n_oids, n_oids_single + ) + assert len_object == len(pdf), "{} is not equal to {}".format(len_object, len(pdf)) + + +if __name__ == "__main__": + """ Execute the test suite """ + import sys + import doctest + + sys.exit(doctest.testmod()[0])