fix orbit cli types (#145)

* Issue/123/kbo neo issue (#134) * add seeding with dbscan and the corresponding notebook * pep8 requirements * add dbscan test * add possibilities to choose between dbscan or the old intra night assoc in the config file, add test dbscan run in the ci * fix dbscan argument in offline mode * work in progress * wip notebook paper * finish kbo analysis and add neo analysis * add script to make the data used by the kbo and neo issue notebook * pep8 * update path * add complexity plots and the associated data (#139) * add complexity plots and the associated data * update notebook * update notebook * Layout for figures * fix layout for all figures * remove title for one figure * Layout for the complexity plots --------- Co-authored-by: FusRoman <[email protected]> Co-authored-by: JulienPeloton <[email protected]> * fix orbit cli types * add kalman assoc command line * working on the cli * fix kalman filter, improve performance and fix bug in the kalman ci, still some improvement on the kalman filter need to be done * pep8 * fix predict bug * add functionality for roid fitting using polynomial function * small refactoring * small fix on the 1.0 * rename notebook into fink_fat_notebook * update notebooks for the amateur * Issue/147/inverse tags (#148) * tags inversion * inverse tags and make tests works * pep8 * Issue/146/rename notebook dir (#150) * rename notebook dir into fink_fat_notebook * remove big files * bump to 0.15.4 * [HOTFIX] update gitignore * Issue/152/fix orbfit params (#153) * fix orb params, pass params from config file to orbfit * forgot verbose param * bump to 0.15.5 * fix cli test with new orbital params * Issue/155/orbfit timeout (#156) * addtimeout option and orbfit logs * incorporate verbose options from command line to orbfit * pep8 * pep8 * fix isinstance call * Minimal extension for external files (#154) * Minimal setup for EUclid * Update file processor * Add Euclid sample data * Configuration file for Euclid-like data (ssopipe) * Update verbose placement * Revert the conf file * Bring back required column definition * PEP8 * Expand unit tests * Add internal check for the file processor * bump to 0.16.0 --------- Co-authored-by: Roman <[email protected]> * remove bin dir * bump to 1.0.0 * add associations between the new alerts from the stream and the trajectories * pep8 * fix kalman test * small fix * fix assoc stream test --------- Co-authored-by: FusRoman <[email protected]> Co-authored-by: JulienPeloton <[email protected]>
FusRoman · Dec 1, 2023 · e0afe86 · e0afe86
1 parent 20215ca
commit e0afe86
Show file tree

Hide file tree

Showing 826 changed files with 11,304 additions and 240 deletions.
diff --git a/.flake8 b/.flake8
@@ -21,9 +21,9 @@ per-file-ignores =
     ../fink-fat/fink_fat/orbit_fitting/orbfit_merger.py:W503
     ../fink-fat/fink_fat/command_line/fink_fat_cli.py:W503
     ../fink-fat/fink_fat/test/test_sample.py:F601
-    ../fink-fat/notebook/results_analysis/utils.py:W503
-    ../fink-fat/notebook/parameters_selection/exploring_script.py:E721
-    ../fink-fat/notebook/follow_up/ephem_association.py:W503
+    ../fink-fat/fink_fat_notebook/results_analysis/utils.py:W503
+    ../fink-fat/fink_fat_notebook/parameters_selection/exploring_script.py:E721
+    ../fink-fat/fink_fat_notebook/follow_up/ephem_association.py:W503
     ../fink-fat/fink_fat/seeding/dbscan_seeding.py:W503
     ../fink-fat/fink_fat/command_line/cli_main/stats.py:W503
     ../fink-fat/fink_fat/kalman/asteroid_kalman.py:W503

diff --git a/.gitignore b/.gitignore
@@ -154,7 +154,7 @@ mpcobs/
 OrbitFit
 asteroid_study/*.png
 asteroid_study/residuals/
-trajectory_df.parquet
+# trajectory_df.parquet
 
 fink_fat/others/perf_test
 
@@ -182,4 +182,9 @@ merge_traj.parquet
 test_merger/
 track_bug/
 transient_data.json
-fink_fat_test/
+fink_fat_test/
+
+fink_fat_notebook/fink_fat_experiments/confirmed_all_fink_fat/
+fink_fat_notebook/fink_fat_experiments/confirmed_mops_fink_fat/
+fink_fat_notebook/fink_fat_experiments/data/
+fink_fat_notebook/parameters_selection/res_orbit_nb_point/
diff --git a/docker/centos7/Dockerfile b/docker/centos7/Dockerfile
@@ -39,37 +39,37 @@ COPY . fink-fat
 
 # Install system build dependencies
 RUN yum -y update \
-&& yum -y install which git wget java-11-openjdk-devel \
-&& echo "export JAVA_HOME=$(dirname $(dirname $(readlink -f $(type -P java))))" > /etc/profile.d/javahome.sh \
-&& yum -y groupinstall "Development Tools" \
-&& yum -y clean all \
-&& rm -rf /var/cache
+    && yum -y install which git wget java-11-openjdk-devel \
+    && echo "export JAVA_HOME=$(dirname $(dirname $(readlink -f $(type -P java))))" > /etc/profile.d/javahome.sh \
+    && yum -y groupinstall "Development Tools" \
+    && yum -y clean all \
+    && rm -rf /var/cache
 # && echo "export JAVA_HOME=$(dirname $(dirname $(readlink -f $(type -P java))))" > /etc/profile.d/javahome.sh
 
 # install python and the dependencies
 RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${PYTHON_VERSION}-Linux-x86_64.sh -O ~/miniconda.sh \
-&& bash ~/miniconda.sh -b -p ${USRLIBS}/miniconda
+    && bash ~/miniconda.sh -b -p ${USRLIBS}/miniconda
 
 RUN pip install --no-cache-dir --upgrade pip setuptools wheel \
-&& cd ${USRLIBS}/fink-fat/script \
-&& source ./install_python_deps.sh \
-&& cd ${USRLIBS}
+    && cd ${USRLIBS}/fink-fat/script \
+    && source ./install_python_deps.sh \
+    && cd ${USRLIBS}
 
 
 # install spark
 RUN wget --quiet https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-${HADOOP_VERSION}.tgz \
-&& tar -xf spark-${SPARK_VERSION}-bin-${HADOOP_VERSION}.tgz \
-&& rm spark-${SPARK_VERSION}-bin-${HADOOP_VERSION}.tgz \
-&& echo "export SPARK_HOME=$PWD/spark-${SPARK_VERSION}-bin-${HADOOP_VERSION}" > /etc/profile.d/sparkhome.sh
+    && tar -xf spark-${SPARK_VERSION}-bin-${HADOOP_VERSION}.tgz \
+    && rm spark-${SPARK_VERSION}-bin-${HADOOP_VERSION}.tgz \
+    && echo "export SPARK_HOME=$PWD/spark-${SPARK_VERSION}-bin-${HADOOP_VERSION}" > /etc/profile.d/sparkhome.sh
 
 # install OrbFit
 RUN yum -y install epel-release \
-&& yum -y install aria2 \
-&& mkdir OrbFit \
-&& cd ${USRLIBS}/fink-fat/script \
-&& source ./orbFit_installer.sh --install_path ${USRLIBS}/OrbFit \
-&& echo "export ORBFIT_HOME=${USRLIBS}/OrbFit" > /etc/profile.d/orbfithome.sh \
-&& cd ${USRLIBS}
+    && yum -y install aria2 \
+    && mkdir OrbFit \
+    && cd ${USRLIBS}/fink-fat/script \
+    && source ./orbFit_installer.sh --install_path ${USRLIBS}/OrbFit \
+    && echo "export ORBFIT_HOME=${USRLIBS}/OrbFit" > /etc/profile.d/orbfithome.sh \
+    && cd ${USRLIBS}
 
 ENV SPARK_HOME=$USRLIBS/spark-${SPARK_VERSION}-bin-${HADOOP_VERSION}
 ENV SPARKLIB=${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.9-src.zip

diff --git a/fink_fat/__init__.py b/fink_fat/__init__.py
@@ -12,4 +12,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-__version__ = "0.15.2"
+__version__ = "1.0.0"
diff --git a/fink_fat/associations/association_kalman.py b/fink_fat/associations/association_kalman.py
@@ -7,6 +7,7 @@
 from copy import deepcopy
 
 from fink_fat.others.utils import repeat_chunk
+from fink_fat.others.utils import LoggerNewLine
 
 
 def update_kalman(
@@ -19,7 +20,7 @@ def update_kalman(
     tr_id: int,
 ) -> pd.Series:
     """
-    Update the kalman filter contains in the
+    Update the kalman filter in input inplace.
 
     Parameters
     ----------
@@ -43,29 +44,13 @@ def update_kalman(
     pd.Series
         same row as input but updated with the new alert
     """
-    Y = np.array(
-        [
-            [
-                ra_alert,
-                dec_alert,
-            ]
-        ]
-    )
-
     dt = jd_alert - kalman_copy["jd_1"].values[0]
-    A = np.array(
-        [
-            [1, 0, dt, 0],
-            [0, 1, 0, dt],
-            [0, 0, 1, 0],
-            [0, 0, 0, 1],
-        ]
-    )
 
-    kalman_copy["kalman"].values[0].update(
-        Y,
-        A,
-    )
+    if dt == 0:
+        # if the dt is 0, the previous data point of the kalman are in the same exposure.
+        # it happens only for the assocations between the tracklets and the kalman
+        # and skip this point is not too bad for the rest of the kalman estimation.
+        return kalman_copy
     with pd.option_context("mode.chained_assignment", None):
         kalman_copy["ra_0"] = kalman_copy["ra_1"]
         kalman_copy["dec_0"] = kalman_copy["dec_1"]
@@ -87,6 +72,29 @@ def update_kalman(
         ) / kalman_copy["dt"]
         kalman_copy["trajectory_id"] = tr_id
 
+    A = np.array(
+        [
+            [1, 0, dt, 0],
+            [0, 1, 0, dt],
+            [0, 0, 1, 0],
+            [0, 0, 0, 1],
+        ]
+    )
+
+    Y = np.array(
+        [
+            [ra_alert],
+            [dec_alert],
+            [kalman_copy["vel_ra"].values[0]],
+            [kalman_copy["vel_dec"].values[0]],
+        ]
+    )
+
+    kalman_copy["kalman"].values[0].update(
+        Y,
+        A,
+    )
+
     return kalman_copy
 
 
@@ -182,7 +190,11 @@ def trajectory_extension(
 
 
 def tracklets_associations(
-    trajectory_df: pd.DataFrame, kalman_df: pd.DataFrame, new_alerts: pd.DataFrame
+    trajectory_df: pd.DataFrame,
+    kalman_df: pd.DataFrame,
+    new_alerts: pd.DataFrame,
+    logger: LoggerNewLine,
+    verbose: bool,
 ) -> Tuple[pd.DataFrame, pd.DataFrame]:
     """
     Associates the intra night trajectories with the kalman trajectories.
@@ -195,6 +207,10 @@ def tracklets_associations(
         dataframe containing the kalman filters informations
     new_alerts : pd.DataFrame
         new associateds alerts from the new observing night.
+    logger : LoggerNewLine
+        logger class used to print the logs
+    verbose : bool
+        if true, print the logs
 
     Returns
     -------
@@ -241,12 +257,21 @@ def tracklets_associations(
             new_tr_id += 1
 
     if len(res_updated_traj) == 0:
+        if verbose:
+            logger.info(
+                "no associations between the intra night tracklets and the trajectories"
+            )
         return (
             pd.DataFrame(columns=trajectory_df.columns),
             pd.DataFrame(columns=kalman_df.columns),
             new_tr_id,
         )
 
+    if verbose:
+        logger.info(
+            f"number of associations between the intra night tracklets and the trajectories: {len(res_updated_kalman)}"
+        )
+
     # merge the extended trajectories
     all_extended_traj = pd.concat(res_updated_traj)
     all_new_kalman = pd.concat(res_updated_kalman)
@@ -259,6 +284,8 @@ def single_alerts_associations(
     kalman_df: pd.DataFrame,
     new_alerts: pd.DataFrame,
     max_tr_id: int,
+    logger: LoggerNewLine,
+    verbose: bool,
 ) -> Tuple[pd.DataFrame, pd.DataFrame]:
     """
     Associates the single alerts with the kalman trajectories
@@ -273,6 +300,10 @@ def single_alerts_associations(
         new associateds alerts from the new observing night.
     max_tr_id : int
         maximum trajectory id to assign to the new kalman trajectories
+    logger : LoggerNewLine
+        logger class used to print the logs
+    verbose : bool
+        if true, print the logs
 
     Returns
     -------
@@ -290,6 +321,10 @@ def single_alerts_associations(
     cluster_df = cluster_df.sort_values("estimator_id")
 
     if len(cluster_df) == 0:
+        if verbose:
+            logger.info(
+                "no associations between the single alerts and the trajectories"
+            )
         return pd.DataFrame(columns=trajectory_df.columns), pd.DataFrame(
             columns=kalman_df.columns
         )
@@ -310,6 +345,10 @@ def single_alerts_associations(
             ].values
         ]
     )
+    if verbose:
+        logger.info(
+            f"number of kalman trajectories to updated with single alert: {len(new_kalman)}"
+        )
     traj_to_update = (
         trajectory_df[trajectory_df["trajectory_id"].isin(cluster_df["estimator_id"])]
         .sort_values(["trajectory_id", "jd"])
@@ -324,10 +363,6 @@ def single_alerts_associations(
     nb_repeat = np.repeat(traj_size.values, traj_counts_duplicates.values)
     tr_id_repeat = np.repeat(cluster_df["trajectory_id"].values, nb_repeat)
 
-    traj_duplicate["trajectory_id"] = tr_id_repeat
-    nb_repeat = np.repeat(traj_size.values, traj_counts_duplicates.values)
-    tr_id_repeat = np.repeat(cluster_df["trajectory_id"].values, nb_repeat)
-
     traj_duplicate["trajectory_id"] = tr_id_repeat
     new_traj = pd.concat([traj_duplicate, cluster_df.drop("estimator_id", axis=1)])
     return new_traj, new_kalman
@@ -337,6 +372,8 @@ def kalman_association(
     trajectory_df: pd.DataFrame,
     kalman_df: pd.DataFrame,
     new_alerts: pd.DataFrame,
+    logger: LoggerNewLine,
+    verbose: bool,
     confirmed_sso: bool = False,
 ) -> Tuple[pd.DataFrame, pd.DataFrame]:
     """
@@ -352,6 +389,10 @@ def kalman_association(
     new_alerts : pd.DataFrame
         dataframe containing the alerts from the current nigth to associates with the kalman filters,
         the dataframe must contains the trajectory_id corresponding to the seeds find by the seeding.
+    logger : LoggerNewLine
+        logger class used to print the logs
+    verbose : bool
+        if true, print the logs
     confirmed_sso : boolean
         if true, used the confirmed sso (for test purpose)
 
@@ -370,7 +411,12 @@ def kalman_association(
         roid_flag = [3, 4]
     else:
         roid_flag = [1, 2, 4]
-    new_alerts = new_alerts[new_alerts["roid"].isin(roid_flag)]
+
+    if verbose:
+        logger.info("- start the association with the kalman filters")
+    new_alerts = new_alerts[new_alerts["roid"].isin(roid_flag)].explode(
+        ["estimator_id", "ffdistnr"]
+    )
     new_alerts = new_alerts.explode(["ffdistnr", "estimator_id"])
     new_alerts["estimator_id"] = new_alerts["estimator_id"].fillna(-1).astype(int)
     traj_id_to_update = np.sort(new_alerts["estimator_id"].unique())
@@ -382,10 +428,10 @@ def kalman_association(
     non_kalman_update = kalman_df[~mask_kalman_update]
 
     res_tr, res_kalman, max_tr_id = tracklets_associations(
-        trajectory_df, kalman_df, new_alerts
+        trajectory_df, kalman_df, new_alerts, logger, verbose
     )
     new_traj, new_kalman = single_alerts_associations(
-        trajectory_df, kalman_df, new_alerts, max_tr_id
+        trajectory_df, kalman_df, new_alerts, max_tr_id, logger, verbose
     )
 
     new_traj = pd.concat([res_tr, new_traj])
@@ -398,7 +444,8 @@ def kalman_association(
 
     traj_results = pd.concat([non_tr_update_df, new_traj])
     kalman_results = pd.concat([non_kalman_update, new_kalman])
-
+    if verbose:
+        logger.newline()
     return traj_results, kalman_results
 
 

diff --git a/fink_fat/associations/association_orbit.py b/fink_fat/associations/association_orbit.py
@@ -63,13 +63,16 @@ def orbit_associations(
     Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]
         the new orbits, the updated trajectories and the old orbits
     """
+    if verbose:
+        logger.info("start the associations with the orbits")
     orbit_cols_to_keep = list(orbits.columns)
     traj_cols_to_keep = list(trajectory_df.columns)
     if "ffdistnr" not in traj_cols_to_keep:
         traj_cols_to_keep.append("ffdistnr")
 
     orbit_alert_assoc = (
         new_alerts[new_alerts["roid"] == 5]
+        .explode(["estimator_id", "ffdistnr"])
         .rename({"estimator_id": "ssoCandId"}, axis=1)
         .drop("roid", axis=1)
     )
@@ -87,10 +90,11 @@ def orbit_associations(
         logger.info(f"number of orbits to update: {len(updated_sso_id)}")
 
     duplicated_id = orbit_alert_assoc[
-        orbit_alert_assoc[["ssoCandId", "jd"]].duplicated()
+        orbit_alert_assoc[["ssoCandId", "candid"]].duplicated()
     ]["ssoCandId"]
     assert len(duplicated_id) == 0
 
+    # recompute the orbit using local or cluster mode
     new_orbit_pdf = switch_local_cluster(config, traj_to_new_orbit)
 
     # remove the failed orbits
@@ -99,6 +103,7 @@ def orbit_associations(
         logger.info(
             f"number of successfull updated orbits: {len(new_orbit_pdf)} ({(len(new_orbit_pdf) / len(updated_sso_id)) * 100} %)"
         )
+        logger.newline(2)
     new_orbit_pdf["ssoCandId"] = new_orbit_pdf["trajectory_id"].map(trid_to_ssoid)
     updated_id = new_orbit_pdf["ssoCandId"]