Start date and end date parameters are now optional with default valu…

…e based on now - deltatime.

Dockerfile.efdtransform

@@ -37,9 +37,13 @@ ENV EFD_USERNAME="efdreader"
 # Consdb Transform DATABASE Variables
 ENV DB_URI="sqlite:////opt/lsst/software/stack/data/test.db"
 
+# Processing time interval in minutes
+ENV TIMEDELTA="5"
+
 ENV LOG_FILE="/opt/lsst/software/stack/data/transform.log"
 
-CMD ["bash", "-c", "source loadLSST.bash; setup lsst_distrib; python ./consdb/efd_transform/transform_efd.py -c \"$CONFIG_FILE\" -i \"$INSTRUMENT\" -s \"$START_TIME\" -e \"$END_TIME\" -r \"$BUTLER_REPO\" -d \"$DB_URI\"  -E \"$EFD\" -l \"$LOG_FILE\""]
+CMD ["bash", "-c", "source loadLSST.bash; setup lsst_distrib; python ./consdb/efd_transform/transform_efd.py -c \"$CONFIG_FILE\" -i \"$INSTRUMENT\" -r \"$BUTLER_REPO\" -d \"$DB_URI\"  -E \"$EFD\" -t \"$TIMEDELTA\" -l \"$LOG_FILE\""]
+# CMD ["bash", "-c", "source loadLSST.bash; setup lsst_distrib; python ./consdb/efd_transform/transform_efd.py -c \"$CONFIG_FILE\" -i \"$INSTRUMENT\" -s \"$START_TIME\" -e \"$END_TIME\" -r \"$BUTLER_REPO\" -d \"$DB_URI\"  -E \"$EFD\" -l \"$LOG_FILE\""]
 
 # Exemple of command used to execute transform_efd with docker
 # docker run --rm -it --volume $PWD/data:/opt/lsst/software/stack/data  -e CONFIG_FILE=config_LATISS.yaml -e ...  consdb/efd_transform:latest

python/lsst/consdb/efd_transform/transform_efd.py

@@ -2,18 +2,19 @@
 import asyncio
 import logging
 import sys
+from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Any, Dict
 
 import astropy.time
 
 # import lsst_efd_client
 import yaml
-from dao.influxdb import InfluxDbDao
 from config_model import ConfigModel
-from transform import Transform
+from dao.influxdb import InfluxDbDao
 from lsst.daf.butler import Butler
 from pydantic import ValidationError
+from transform import Transform
 
 # from sqlalchemy import create_engine
 
@@ -73,21 +74,21 @@ def build_argparser() -> argparse.ArgumentParser:
         "-s",
         "--start",
         dest="start_time",
-        required=True,
+        required=False,
         help="start time (ISO, YYYY-MM-DDTHH:MM:SS)",
     )
     parser.add_argument(
         "-e",
         "--end",
         dest="end_time",
-        required=True,
+        required=False,
         help="end time (ISO, YYYY-MM-DDTHH:MM:SS)",
     )
     parser.add_argument(
         "-r",
         "--repo",
         dest="repo",
-        # default="/repo/embargo",
+        default="s3://rubin-summit-users/butler.yaml",
         required=True,
         help="Butler repo",
     )
@@ -107,7 +108,14 @@ def build_argparser() -> argparse.ArgumentParser:
         required=True,
         help="EFD connection string",
     )
-
+    parser.add_argument(
+        "-t",
+        "--timedelta",
+        dest="timedelta",
+        default=5,
+        required=False,
+        help="Processing time interval in minutes",
+    )
     parser.add_argument(
         "-l",
         "--logfile",
@@ -158,25 +166,40 @@ async def main() -> None:
 
     log = get_logger(args.logfile)
 
+    # defining the start and end time
+    # TODO: Should I use UTC???
+    # now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
+    now = datetime.now().replace(second=0, microsecond=0)
+
+    if args.start_time is None:
+        start_time = now - timedelta(minutes=int(args.timedelta))
+    else:
+        start_time = datetime.fromisoformat(args.start_time)
+
+    if args.end_time is None:
+        end_time = now
+    else:
+        end_time = datetime.fromisoformat(args.end_time)
+
+    start_time = astropy.time.Time(start_time.isoformat(), format="isot")
+    end_time = astropy.time.Time(end_time.isoformat(), format="isot")
+
+    # Instantiate the butler
     butler = Butler(args.repo)
 
-    # db = create_engine(args.db_conn_str)
+    # Instantiate the EFD
     db_uri = args.db_conn_str
-
     efd = InfluxDbDao(args.efd_conn_str)
 
     config = read_config(args.config_name)
 
-    # TODO: Commit every can be a setting
+    # TODO: Commit every can be a enviroment variable
     commit_every = 100
 
+    # Instantiate the main class transform
     tm = Transform(
         butler=butler, db_uri=db_uri, efd=efd, config=config, logger=log, commit_every=commit_every
     )
-
-    start_time = astropy.time.Time(args.start_time, format="isot")
-    end_time = astropy.time.Time(args.end_time, format="isot")
-
     tm.process_interval(
         args.instrument,
         start_time,

tests/efd_transform/run_transform_efd.sh

@@ -1,11 +1,21 @@
 # Example of command used to execute transform_efd
 # Considering the command is being executed at the root of the consdb repository
+
 python python/lsst/consdb/efd_transform/transform_efd.py \
     -i LATISS \
-    -s 2024-04-24T23:00:00  \
-    -e 2024-04-25T00:00:00 \
-    -r /repo/embargo \
+    -r s3://rubin-summit-users/butler.yaml \
     -d sqlite:///$PWD/tests/efd_transform/test.db \
     -E usdf_efd \
     -c python/lsst/consdb/efd_transform/config_LATISS.yaml \
+    -t 60 \
     -l $PWD/tmp/transform.log
+
+# python python/lsst/consdb/efd_transform/transform_efd.py \
+#     -i LATISS \
+#     -s 2024-04-24T23:00:00  \
+#     -e 2024-04-25T00:00:00 \
+#     -r s3://rubin-summit-users/butler.yaml \
+#     -d sqlite:///$PWD/tests/efd_transform/test.db \
+#     -E usdf_efd \
+#     -c python/lsst/consdb/efd_transform/config_LATISS.yaml \
+#     -l $PWD/tmp/transform.log