Add the cutouts toute

apps/routes/cutouts/api.py

@@ -0,0 +1,104 @@
+# Copyright 2024 AstroLab Software
+# Author: Julien Peloton
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from flask import Blueprint, Response, jsonify, request
+from apps.utils import check_args
+
+from apps.routes.cutouts.utils import format_and_send_cutout
+
+bp = Blueprint("cutouts", __name__)
+
+
+# Enable CORS for this blueprint
+@bp.after_request
+def after_request(response):
+    response.headers.add("Access-Control-Allow-Origin", "*")
+    response.headers.add("Access-Control-Allow-Headers", "Content-Type,Authorization")
+    response.headers.add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS")
+    return response
+
+
+ARGS = [
+    {
+        "name": "objectId",
+        "required": True,
+        "description": "ZTF Object ID",
+    },
+    {
+        "name": "kind",
+        "required": True,
+        "description": "Science, Template, or Difference. For output-format=array, you can also specify `kind: All` to get the 3 cutouts.",
+    },
+    {
+        "name": "output-format",
+        "required": False,
+        "description": "PNG[default], FITS, array",
+    },
+    {
+        "name": "candid",
+        "required": False,
+        "description": "Candidate ID of the alert belonging to the object with `objectId`. If not filled, the cutouts of the latest alert is returned",
+    },
+    {
+        "name": "stretch",
+        "required": False,
+        "description": "Stretch function to be applied. Available: sigmoid[default], linear, sqrt, power, log.",
+    },
+    {
+        "name": "colormap",
+        "required": False,
+        "description": "Valid matplotlib colormap name (see matplotlib.cm). Default is grayscale.",
+    },
+    {
+        "name": "pmin",
+        "required": False,
+        "description": "The percentile value used to determine the pixel value of minimum cut level. Default is 0.5. No effect for sigmoid.",
+    },
+    {
+        "name": "pmax",
+        "required": False,
+        "description": "The percentile value used to determine the pixel value of maximum cut level. Default is 99.5. No effect for sigmoid.",
+    },
+    {
+        "name": "convolution_kernel",
+        "required": False,
+        "description": "Convolve the image with a kernel (gauss or box). Default is None (not specified).",
+    },
+]
+
+
+@bp.route("/api/v1/cutouts", methods=["GET"])
+def return_cutouts_arguments():
+    """Obtain information about cutouts"""
+    if len(request.args) > 0:
+        # POST from query URL
+        return return_cutouts(payload=request.args)
+    else:
+        return jsonify({"args": ARGS})
+
+
+@bp.route("/api/v1/cutouts", methods=["POST"])
+def return_cutouts(payload=None):
+    """Retrieve object data"""
+    # get payload from the JSON
+    if payload is None:
+        payload = request.json
+
+    rep = check_args(ARGS, payload)
+    if rep["status"] != "ok":
+        return Response(str(rep), 400)
+
+    assert payload["kind"] in ["Science", "Template", "Difference", "All"]
+
+    return format_and_send_cutout(payload)

apps/routes/cutouts/utils.py

@@ -0,0 +1,180 @@
+# Copyright 2024 AstroLab Software
+# Author: Julien Peloton
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from flask import send_file, jsonify
+
+import io
+import json
+import requests
+
+import numpy as np
+from matplotlib import cm
+from PIL import Image
+
+from apps.utils.client import connect_to_hbase_table
+from apps.utils.plotting import sigmoid_normalizer, legacy_normalizer, convolve
+from apps.utils.decoding import format_hbase_output
+from apps.utils.utils import extract_configuration
+
+
+def format_and_send_cutout(payload: dict):
+    """Extract data returned by HBase and jsonify it
+
+    Data is from /api/v1/cutouts
+
+    Parameters
+    ----------
+    payload: dict
+        See https://fink-portal.org/api/v1/cutouts
+
+    Return
+    ----------
+    out: pandas dataframe
+    """
+    output_format = payload.get("output-format", "PNG")
+
+    # default stretch is sigmoid
+    if "stretch" in payload:
+        stretch = payload["stretch"]
+    else:
+        stretch = "sigmoid"
+
+    if payload["kind"] == "All" and payload["output-format"] != "array":
+        # TODO: error 400
+        pass
+
+    # default name based on parameters
+    filename = "{}_{}".format(
+        payload["objectId"],
+        payload["kind"],
+    )
+
+    if output_format == "PNG":
+        filename = filename + ".png"
+    elif output_format == "JPEG":
+        filename = filename + ".jpg"
+    elif output_format == "FITS":
+        filename = filename + ".fits"
+
+    # Query the Database (object query)
+    client = connect_to_hbase_table("ztf.cutouts")
+    results = client.scan(
+        "",
+        "key:key:{}".format(payload["objectId"]),
+        "d:hdfs_path,i:jd,i:candid,i:objectId",
+        0,
+        False,
+        False,
+    )
+
+    # Format the results
+    schema_client = client.schema()
+    client.close()
+
+    pdf = format_hbase_output(
+        results,
+        schema_client,
+        group_alerts=False,
+        truncated=True,
+        extract_color=False,
+    )
+
+    json_payload = {}
+    # Extract only the alert of interest
+    if "candid" in payload:
+        mask = pdf["i:candid"].astype(str) == str(payload["candid"])
+        json_payload.update({"candid": str(payload["candid"])})
+        pos_target = np.where(mask)[0][0]
+    else:
+        # pdf has been sorted in `format_hbase_output`
+        pdf = pdf.iloc[0:1]
+        pos_target = 0
+
+    json_payload.update(
+        {
+            "hdfsPath": pdf["d:hdfs_path"].to_numpy()[pos_target].split("8020")[1],
+            "kind": payload["kind"],
+            "objectId": pdf["i:objectId"].to_numpy()[pos_target],
+        }
+    )
+
+    if pdf.empty:
+        return send_file(
+            io.BytesIO(),
+            mimetype="image/png",
+            as_attachment=True,
+            download_name=filename,
+        )
+    # Extract cutouts
+    user_config = extract_configuration("config.yml")
+    if output_format == "FITS":
+        json_payload.update({"return_type": "FITS"})
+        r0 = requests.post("{}/api/v1/cutouts".format(user_config["CUTOUTAPIURL"]), json=json_payload)
+        cutout = io.BytesIO(r0.content)
+    elif output_format in ["PNG", "array"]:
+        json_payload.update({"return_type": "array"})
+        r0 = requests.post("{}/api/v1/cutouts".format(user_config["CUTOUTAPIURL"]), json=json_payload)
+        cutout = json.loads(r0.content)
+
+    # send the FITS file
+    if output_format == "FITS":
+        return send_file(
+            cutout,
+            mimetype="application/octet-stream",
+            as_attachment=True,
+            download_name=filename,
+        )
+    # send the array
+    elif output_format == "array":
+        if payload["kind"] != "All":
+            return jsonify({"b:cutout{}_stampData".format(payload["kind"]): cutout[0]})
+        else:
+            out = {
+                "b:cutoutScience_stampData": cutout[0],
+                "b:cutoutTemplate_stampData": cutout[1],
+                "b:cutoutDifference_stampData": cutout[2],
+            }
+            return jsonify(out)
+
+    array = np.nan_to_num(np.array(cutout[0], dtype=float))
+    if stretch == "sigmoid":
+        array = sigmoid_normalizer(array, 0, 1)
+    elif stretch is not None:
+        pmin = 0.5
+        if "pmin" in payload:
+            pmin = float(payload["pmin"])
+        pmax = 99.5
+        if "pmax" in payload:
+            pmax = float(payload["pmax"])
+        array = legacy_normalizer(array, stretch=stretch, pmin=pmin, pmax=pmax)
+
+    if "convolution_kernel" in payload:
+        assert payload["convolution_kernel"] in ["gauss", "box"]
+        array = convolve(array, smooth=1, kernel=payload["convolution_kernel"])
+
+    # colormap
+    if "colormap" in payload:
+        colormap = getattr(cm, payload["colormap"])
+    else:
+        colormap = lambda x: x  # noqa: E731
+    array = np.uint8(colormap(array) * 255)
+
+    # Convert to PNG
+    data = Image.fromarray(array)
+    datab = io.BytesIO()
+    data.save(datab, format="PNG")
+    datab.seek(0)
+    return send_file(
+        datab, mimetype="image/png", as_attachment=True, download_name=filename
+    )