From 331ca453516a310cf9e4db4ad24bd252d4d5e5d4 Mon Sep 17 00:00:00 2001 From: data hound Date: Mon, 13 Dec 2021 20:25:10 +0530 Subject: [PATCH 1/9] Delete cluster scripts and remove unwanted imports from grid.py --- cluster/com.sh | 22 --------------- cluster/com_and_dannce.sh | 29 -------------------- cluster/com_and_dannce_multi_gpu.sh | 34 ------------------------ cluster/com_multi_gpu.sh | 31 --------------------- cluster/dannce.sh | 24 ----------------- cluster/dannce_multi_gpu.sh | 29 -------------------- cluster/grid.py | 7 ----- cluster/holy_com_predict.sh | 22 --------------- cluster/holy_com_predict_multi_gpu.sh | 22 --------------- cluster/holy_com_train.sh | 22 --------------- cluster/holy_dannce_predict.sh | 22 --------------- cluster/holy_dannce_predict_multi_gpu.sh | 24 ----------------- cluster/holy_dannce_train.sh | 22 --------------- cluster/holy_dannce_train_grid.sh | 28 ------------------- 14 files changed, 338 deletions(-) delete mode 100755 cluster/com.sh delete mode 100755 cluster/com_and_dannce.sh delete mode 100755 cluster/com_and_dannce_multi_gpu.sh delete mode 100755 cluster/com_multi_gpu.sh delete mode 100755 cluster/dannce.sh delete mode 100755 cluster/dannce_multi_gpu.sh delete mode 100755 cluster/holy_com_predict.sh delete mode 100755 cluster/holy_com_predict_multi_gpu.sh delete mode 100755 cluster/holy_com_train.sh delete mode 100755 cluster/holy_dannce_predict.sh delete mode 100755 cluster/holy_dannce_predict_multi_gpu.sh delete mode 100755 cluster/holy_dannce_train.sh delete mode 100755 cluster/holy_dannce_train_grid.sh diff --git a/cluster/com.sh b/cluster/com.sh deleted file mode 100755 index bc59219..0000000 --- a/cluster/com.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Script to run all steps of dannce in a single job. -# -# Inputs: com_config - path to com config. -# Example: sbatch com.sh /path/to/com_config.yaml -#SBATCH --job-name=com -#SBATCH --mem=5000 -#SBATCH -t 5-00:00 -#SBATCH -N 1 -#SBATCH -c 1 -#SBATCH -p common - -# module load Anaconda3/5.1.0 -. ~/.bashrc -source activate dannce_cuda11 -# Commented to make script DCC compatible -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 -set -e -sbatch --wait holy_com_train.sh $1 -wait -sbatch --wait holy_com_predict.sh $1 diff --git a/cluster/com_and_dannce.sh b/cluster/com_and_dannce.sh deleted file mode 100755 index a7beef6..0000000 --- a/cluster/com_and_dannce.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Script to run all steps of dannce in a single job. -# -# Inputs: com_config - path to com config. -# dannce_config - path to com config. -# Example: sbatch com_and_dannce.sh /path/to/com_config.yaml /path/to/dannce_config.yaml -#SBATCH --job-name=com_and_dannce -#SBATCH --mem=5000 -#SBATCH -t 5-00:00 -#SBATCH -N 1 -#SBATCH -c 1 -#SBATCH -p common - -# module load Anaconda3/5.1.0 -. ~/.bashrc -source activate dannce_cuda11 - -# Commented out to make script DCC compatible -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 - -set -e -sbatch --wait holy_com_train.sh $1 -wait -sbatch --wait holy_com_predict.sh $1 -wait -sbatch --wait holy_dannce_train.sh $2 -wait -sbatch --wait holy_dannce_predict.sh $2 diff --git a/cluster/com_and_dannce_multi_gpu.sh b/cluster/com_and_dannce_multi_gpu.sh deleted file mode 100755 index e29b263..0000000 --- a/cluster/com_and_dannce_multi_gpu.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Script to run all steps of dannce in a single job using multi-gpu prediction. -# -# Inputs: com_config - path to com config. -# dannce_config - path to com config. -# Example: sbatch com_and_dannce_multi_gpu.sh /path/to/com_config.yaml /path/to/dannce_config.yaml -#SBATCH --job-name=com_and_dannce -#SBATCH --mem=10000 -#SBATCH -t 5-00:00 -#SBATCH -N 1 -#SBATCH -c 1 -#SBATCH -p common -set -e - -# Setup the dannce environment -# module load Anaconda3/5.1.0 -. ~/.bashrc -source activate dannce_cuda11 - -# Commented out to make script DCC compatible -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 - -# Train com network -sbatch --wait holy_com_train.sh $1 -wait - -# Predict with com network in parallel and merge results -com-predict-multi-gpu $1 -com-merge $1 - -# Predict with dannce network in parallel and merge results -dannce-predict-multi-gpu $2 -dannce-merge $2 diff --git a/cluster/com_multi_gpu.sh b/cluster/com_multi_gpu.sh deleted file mode 100755 index 6925fb0..0000000 --- a/cluster/com_multi_gpu.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Script to run all steps of com in a single job using multi-gpu prediction. -# -# Inputs: com_config - path to com config. -# -# Example: sbatch com_multi_gpu.sh /path/to/com_config.yaml -#SBATCH --job-name=com_multi_gpu -#SBATCH --mem=10000 -#SBATCH -t 5-00:00 -#SBATCH -N 1 -#SBATCH -c 1 -#SBATCH -p common -set -e - -# Setup the dannce environment -# module load Anaconda3/5.1.0 -. ~/.bashrc -source activate dannce_cuda11 - -# Commented out to make script DCC compatible -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 - -# Train com network -sbatch --wait holy_com_train.sh $1 -wait - -# Predict with com network in parallel and merge results -com-predict-multi-gpu $1 -com-merge $1 - diff --git a/cluster/dannce.sh b/cluster/dannce.sh deleted file mode 100755 index 043da1c..0000000 --- a/cluster/dannce.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Script to run all steps of dannce in a single job. -# -# Inputs: dannce_config - path to com config. -# Example: sbatch dannce.sh /path/to/dannce_config.yaml -#SBATCH --job-name=dannce -#SBATCH --mem=5000 -#SBATCH -t 5-00:00 -#SBATCH -N 1 -#SBATCH -c 1 -#SBATCH -p common - -# module load Anaconda3/5.1.0 -. ~/.bashrc -conda activate dannce_tf26 - -# Commented to make script DCC Compatible -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 - -set -e -sbatch --wait holy_dannce_train.sh $1 -wait -sbatch --wait holy_dannce_predict.sh $1 diff --git a/cluster/dannce_multi_gpu.sh b/cluster/dannce_multi_gpu.sh deleted file mode 100755 index b8fe7f5..0000000 --- a/cluster/dannce_multi_gpu.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Script to run all steps of dannce in a single job using multi-gpu prediction. -# -# Inputs: dannce_config - path to com config. -# Example: sbatch dannce_multi_gpu.sh /path/to/dannce_config.yaml -#SBATCH --job-name=dannce_multi_gpu -#SBATCH --mem=15000 -#SBATCH -t 5-00:00 -#SBATCH -N 1 -#SBATCH -c 1 -#SBATCH -p common -set -e - -# Setup the dannce environment -# module load Anaconda3/5.1.0 -. ~/.bashrc -source activate dannce_cuda11 - -# Commented to make script DCC compatible -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 - -# Train dannce network -sbatch --wait holy_dannce_train.sh $1 -wait - -# Predict with dannce network in parallel and merge results -dannce-predict-multi-gpu $1 -dannce-merge $1 diff --git a/cluster/grid.py b/cluster/grid.py index a2e38b9..9dfebc6 100755 --- a/cluster/grid.py +++ b/cluster/grid.py @@ -5,13 +5,6 @@ import yaml import argparse import ast -from dannce.engine.io import load_sync, load_com -from dannce.engine.processing import prepare_save_metadata -from dannce import ( - _param_defaults_shared, - _param_defaults_dannce, - _param_defaults_com, -) from typing import Text, List, Tuple from multi_gpu import build_params_from_config_and_batch diff --git a/cluster/holy_com_predict.sh b/cluster/holy_com_predict.sh deleted file mode 100755 index 17c0c55..0000000 --- a/cluster/holy_com_predict.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# This script will not be called directly from the console. -# It is called by other scripts only - -#SBATCH --job-name=predCOM -# Job name -#SBATCH --mem=30000 -# Job memory request -#SBATCH -t 3-00:00 -# Time limit hrs:min:sec -#SBATCH -N 1 -#SBATCH -c 8 -#SBATCH -p common,tdunn -#SBATCH --gres=gpu:1 - -# module load Anaconda3/5.1.0 -. ~/.bashrc -source activate dannce_cuda11 -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 -com-predict "$@" diff --git a/cluster/holy_com_predict_multi_gpu.sh b/cluster/holy_com_predict_multi_gpu.sh deleted file mode 100755 index d24290d..0000000 --- a/cluster/holy_com_predict_multi_gpu.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# This script will not be called directly from the console. -# It is called by other scripts only - -#SBATCH --job-name=predictCom -# Job name -#SBATCH --mem=10000 -# Job memory request -#SBATCH -t 0-03:00 -# Time limit hrs:min:sec -#SBATCH -N 1 -#SBATCH -n 8 -#SBATCH -p common,tdunn -#SBATCH --gres=gpu:1 - -# module load Anaconda3/5.1.0 -. ~/.bashrc -source activate dannce_cuda11 -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 -com-predict-single-batch "$@" diff --git a/cluster/holy_com_train.sh b/cluster/holy_com_train.sh deleted file mode 100755 index e5fcfaa..0000000 --- a/cluster/holy_com_train.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# This script will not be called directly from the console. -# It is called by other scripts only - -#SBATCH --job-name=trainCOM -# Job name -#SBATCH --mem=30000 -# Job memory request -#SBATCH -t 3-00:00 -# Time limit hrs:min:sec -#SBATCH -N 1 -#SBATCH -c 8 -#SBATCH -p common,tdunn -#SBATCH --gres=gpu:1 - -# module load Anaconda3/5.1.0 -. ~/.bashrc -source activate dannce_cuda11 -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 -com-train "$@" diff --git a/cluster/holy_dannce_predict.sh b/cluster/holy_dannce_predict.sh deleted file mode 100755 index 5796d3c..0000000 --- a/cluster/holy_dannce_predict.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# This script will not be called directly from the console. -# It is called by other scripts only - -#SBATCH --job-name=predictDannce -# Job name -#SBATCH --mem=30000 -# Job memory request -#SBATCH -t 1-00:00 -# Time limit hrs:min:sec -#SBATCH -N 1 -#SBATCH -c 8 -#SBATCH -p common,tdunn -#SBATCH --gres=gpu:1 - -# module load Anaconda3/5.1.0 -. ~/.bashrc -source activate dannce_cuda11 -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 -dannce-predict "$@" diff --git a/cluster/holy_dannce_predict_multi_gpu.sh b/cluster/holy_dannce_predict_multi_gpu.sh deleted file mode 100755 index fb4d5a7..0000000 --- a/cluster/holy_dannce_predict_multi_gpu.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# This script will not be called directly from the console. -# It is called by other scripts only - -#SBATCH --job-name=predictDannce -# Job name -#SBATCH --mem=30000 -# Job memory request -#SBATCH -t 0-03:00 -# Time limit hrs:min:sec -#SBATCH -N 1 -#SBATCH -n 8 -#SBATCH -p scavenger-gpu,tdunn --account=tdunn -#SBATCH --gres=gpu:1 - -# module load Anaconda3/5.1.0 -. ~/.bashrc -# conda info - For debugging -conda activate dannce_cuda11 -# conda info - For debugging -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 -dannce-predict-single-batch "$@" diff --git a/cluster/holy_dannce_train.sh b/cluster/holy_dannce_train.sh deleted file mode 100755 index 2ae62c5..0000000 --- a/cluster/holy_dannce_train.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# This script will not be called directly from the console. -# It is called by other scripts only - -#SBATCH --job-name=trainDannce -# Job name -#SBATCH --mem=80000 -# Job memory request -#SBATCH -t 3-00:00 -# Time limit hrs:min:sec -#SBATCH -N 1 -#SBATCH -c 16 -#SBATCH -p gpu-common,dsplus-gpu --account=plusds -#SBATCH --gres=gpu:1 - -# module load Anaconda3/5.1.0 -. ~/.bashrc -conda activate dannce_tf26 -# module load cuda/11.0.3-fasrc01 -# module load cudnn/8.0.4.30_cuda11.0-fasrc01 -dannce-train "$@" diff --git a/cluster/holy_dannce_train_grid.sh b/cluster/holy_dannce_train_grid.sh deleted file mode 100755 index 8332689..0000000 --- a/cluster/holy_dannce_train_grid.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# This script will not be called directly from the console. -# It is called by other scripts only - -#SBATCH --job-name=trainDannce -# Job name -#SBATCH --mem=80000 -# Job memory request -#SBATCH -t 3-00:00 -# Time limit hrs:min:sec -#SBATCH -N 1 -#SBATCH -c 16 -#SBATCH -p gpu-common,dsplus-gpu --account=plusds -#SBATCH --gres=gpu:1 - -# module load Anaconda3/5.1.0 -. ~/.bashrc -# source activate dannce_cuda11 -conda activate dannce_tf26 - -echo "CUDA VISIBLE DEVICES = " -echo $CUDA_VISIBLE_DEVICES -echo "Slurm Node = " -echo $SLURMD_NODENAME -#module load cuda/11.0.3-fasrc01 -#module load cudnn/8.0.4.30_cuda11.0-fasrc01 -dannce-train-single-batch "$@" From a7f82e21a4ad99ffcec34a4659edd7fa869a07b0 Mon Sep 17 00:00:00 2001 From: data hound Date: Sun, 30 Jan 2022 00:26:52 +0530 Subject: [PATCH 2/9] Logging Messages - First Drop --- cluster/grid.py | 25 ++++- cluster/multi_gpu.py | 34 +++++-- dannce/__init__.py | 2 + dannce/cli.py | 12 +++ dannce/engine/generator.py | 20 ++-- dannce/engine/inference.py | 26 +++-- dannce/engine/nets.py | 40 ++++++-- dannce/engine/ops.py | 1 + dannce/engine/processing.py | 71 ++++++++++---- dannce/engine/serve_data_DANNCE.py | 19 +++- dannce/engine/video.py | 14 ++- dannce/interface.py | 146 +++++++++++++++++++---------- 12 files changed, 296 insertions(+), 114 deletions(-) diff --git a/cluster/grid.py b/cluster/grid.py index 9dfebc6..f44806b 100755 --- a/cluster/grid.py +++ b/cluster/grid.py @@ -11,6 +11,9 @@ import subprocess import time +import logging + +FILE_PATH = "dance.cluster.grid" class GridHandler: def __init__( @@ -84,10 +87,13 @@ def submit_jobs(self, batch_params: List, cmd: Text): batch_params (List): List of batch training parameters. cmd (Text): System command to be issued. """ + # Set logging prepend + prepend_log_msg = FILE_PATH + ".GridHandler.submit_jobs " + if self.verbose: for batch_param in batch_params: - print(batch_param) - print("Command issued: ", cmd) + logging.info(prepend_log_msg + batch_param) + logging.info(prepend_log_msg + "Command issued: ", cmd) if not self.test: if isinstance(cmd, list): for i in range(len(cmd)): @@ -129,6 +135,10 @@ def submit_dannce_train_grid(self) -> Tuple[List, Text]: """ batch_params = self.generate_batch_params_dannce() + # Setup Logging for dannce_train_single_batch + logging.basicConfig(filename=self.load_params(self.config)["log_dest"], level=self.load_params(self.config)["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + slurm_config = self.load_params(self.load_params(self.config)["slurm_config"]) cmd = ( 'sbatch --wait --array=0-%d %s --wrap="%s dannce-train-single-batch %s %s"' @@ -156,13 +166,20 @@ def dannce_train_single_batch(): handler = GridHandler(config, grid_config) batch_params = handler.load_batch_params() task_id = int(os.getenv("SLURM_ARRAY_TASK_ID")) - print("Task ID = ", task_id) batch_param = batch_params[task_id] - print(batch_param) + # Build final parameter dictionary params = build_params_from_config_and_batch(config, batch_param) + # Setup Logging for dannce_train_single_batch + logging.basicConfig(filename=params["log_dest"], level=params["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + prepend_log_msg = FILE_PATH + ".dannce_train_single_batch " + + logging.info(prepend_log_msg + "Task ID = ", task_id) + logging.info(prepend_log_msg + batch_param) + # Train dannce_train(params) diff --git a/cluster/multi_gpu.py b/cluster/multi_gpu.py index 63d0441..284ba4f 100755 --- a/cluster/multi_gpu.py +++ b/cluster/multi_gpu.py @@ -15,9 +15,11 @@ ) import scipy.io as spio from typing import Dict, List, Text, Union, Tuple +import logging DANNCE_PRED_FILE_BASE_NAME = "save_data_AVG" COM_PRED_FILE_BASE_NAME = "com3d" +FILE_PATH = "dannce.cluster.multi_gpu" def loadmat(filename: Text) -> Dict: @@ -100,6 +102,8 @@ def __init__( self.dannce_file = self.load_dannce_file() else: self.dannce_file = dannce_file + + self.setup_logging() def load_params(self, param_path: Text) -> Dict: """Load a params file @@ -113,6 +117,11 @@ def load_params(self, param_path: Text) -> Dict: with open(param_path, "rb") as file: params = yaml.safe_load(file) return params + + def setup_logging(self): + params = self.load_params(self.config) + logging.basicConfig(filename=params["log_dest"], level=params["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') def save_batch_params(self, batch_params: List): """Save the batch_param dictionary to the batch_param file. @@ -340,11 +349,12 @@ def submit_jobs(self, batch_params: List, cmd: str): batch_params (List): Batch parameters list cmd (str): System command """ + prepend_log_msg = FILE_PATH + ".MultiGpuHandler.submit_jobs " if self.verbose: for batch_param in batch_params: - print("Start sample:", batch_param["start_sample"]) - print("End sample:", batch_param["max_num_samples"]) - print("Command issued: ", cmd) + logging.debug("Start sample:", batch_param["start_sample"]) + logging.debug("End sample:", batch_param["max_num_samples"]) + logging.info(prepend_log_msg + "Command issued: ", cmd) if not self.test: sys.exit(os.WEXITSTATUS(os.system(cmd))) @@ -379,8 +389,10 @@ def submit_com_predict_multi_gpu(self): Divide project into equal chunks of n_samples_per_gpu samples. Submit an array job that predicts over each chunk in parallel. """ + prepend_log_msg = FILE_PATH + ".MultiGpuHandler.submit_com_predict_multi_gpu " + n_samples = self.get_n_samples(self.dannce_file, use_com=False) - print(n_samples) + logging.info(prepend_log_msg + n_samples) batch_params = self.generate_batch_params_com(n_samples) slurm_config = self.load_params(self.load_params(self.config)["slurm_config"]) cmd = ( @@ -571,13 +583,18 @@ def dannce_predict_single_batch(): """CLI entrypoint to predict a single batch.""" from dannce.interface import dannce_predict + prepend_log_msg = FILE_PATH + "dannce_predict_single_batch" + # Load in parameters to modify config = sys.argv[1] handler = MultiGpuHandler(config) batch_params = handler.load_batch_params() task_id = int(os.getenv("SLURM_ARRAY_TASK_ID")) batch_param = batch_params[task_id] - print(batch_param) + logging.basicConfig(filename=handler.load_params(handler.config)["log_dest"], + level=handler.load_params(handler.config)["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + logging.info(prepend_log_msg + batch_param) # Build final parameter dictionary params = build_params_from_config_and_batch(config, batch_param) @@ -590,6 +607,8 @@ def com_predict_single_batch(): """CLI entrypoint to predict a single batch.""" from dannce.interface import com_predict + prepend_log_msg = FILE_PATH + "com_predict_single_batch" + # Load in parameters to modify config = sys.argv[1] handler = MultiGpuHandler(config) @@ -597,7 +616,10 @@ def com_predict_single_batch(): task_id = int(os.getenv("SLURM_ARRAY_TASK_ID")) # task_id = 0 batch_param = batch_params[task_id] - print(batch_param) + logging.basicConfig(filename=handler.load_params(handler.config)["log_dest"], + level=handler.load_params(handler.config)["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + logging.info(prepend_log_msg + batch_param) # Build final parameter dictionary params = build_params_from_config_and_batch(config, batch_param, dannce_net=False) diff --git a/dannce/__init__.py b/dannce/__init__.py index a313a2c..c50f221 100755 --- a/dannce/__init__.py +++ b/dannce/__init__.py @@ -46,6 +46,8 @@ "valid_exp": None, "norm_method":"layer", "slurm_config": None, + "log_level": "INFO", + "log_dest": "../logs/dannce.log", } _param_defaults_dannce = { "metric": ["euclidean_distance_3D"], diff --git a/dannce/cli.py b/dannce/cli.py index 73e2cf1..357a13b 100755 --- a/dannce/cli.py +++ b/dannce/cli.py @@ -255,6 +255,18 @@ def add_shared_args( help="Normalization method to use, can be 'batch', 'instance', or 'layer'.", ) + parser.add_argument( + "--log-level", + dest="log_level", + help="Level of logging to use, can be 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'. Default is 'INFO'.", + ) + + parser.add_argument( + "--log-dest", + dest="log_dest", + help="Log File location to where logs are to be written to. By default, location is set to logs/dannce.log ", + ) + return parser diff --git a/dannce/engine/generator.py b/dannce/engine/generator.py index 308f627..46833dc 100755 --- a/dannce/engine/generator.py +++ b/dannce/engine/generator.py @@ -11,6 +11,7 @@ import time import scipy.ndimage.interpolation import tensorflow as tf +import logging # from tensorflow_graphics.geometry.transformation.axis_angle import rotate from multiprocessing.dummy import Pool as ThreadPool @@ -24,10 +25,11 @@ ) TF_GPU_MEMORY_FRACTION = 0.9 +FILE_PATH = "dannce.engine.generator" class DataGenerator(keras.utils.Sequence): - """Generate data for Keras. + """Generate data for Keras. The object creating instance of this class should have logging enabled. Attributes: batch_size (int): Batch size to generate @@ -186,7 +188,7 @@ def random_rotate(self, X: np.ndarray, y_3d: np.ndarray, log: bool = False): class DataGenerator_3Dconv(DataGenerator): - """Update generator class to handle multiple experiments. + """Update generator class to handle multiple experiments. The object creator should have logging enabled. Attributes: camera_params (Dict): Camera parameters dictionary. @@ -330,7 +332,7 @@ def __init__( self.interp = interp self.depth = depth self.channel_combo = channel_combo - print(self.channel_combo) + logging.info(self.channel_combo) self.mode = mode self.immode = immode self.tifdirs = tifdirs @@ -912,7 +914,7 @@ def __init__( self.interp = interp self.depth = depth self.channel_combo = channel_combo - print(self.channel_combo) + logging.info(FILE_PATH + ".DataGenerator_3Dconv_torch.__init__" + self.channel_combo) self.gpu_id = gpu_id self.mode = mode self.immode = immode @@ -941,7 +943,7 @@ def __init__( config.gpu_options.per_process_gpu_memory_fraction = TF_GPU_MEMORY_FRACTION config.gpu_options.allow_growth = True self.session = tf.compat.v1.Session(config=config, graph=tf.Graph()) - print("Executing eagerly: ", tf.executing_eagerly(), flush=True) + logging.info(FILE_PATH + ".DataGenerator_3Dconv_torch.__init__" + "Executing eagerly: ", tf.executing_eagerly(), flush=True) for i, ID in enumerate(list_IDs): experimentID = int(ID.split("_")[0]) for camname in self.camnames[experimentID]: @@ -954,7 +956,7 @@ def __init__( ) self.camera_params[experimentID][camname]["M"] = M - print("Init took {} sec.".format(time.time() - ts)) + logging.info(FILE_PATH + ".DataGenerator_3Dconv_torch.__init__" + "Init took {} sec.".format(time.time() - ts)) def __getitem__(self, index: int): """Generate one batch of data. @@ -1543,7 +1545,7 @@ def __init__( self.interp = interp self.depth = depth self.channel_combo = channel_combo - print(self.channel_combo) + logging.info(FILE_PATH + ".DataGenerator_3Dconv_tf.__init__ " + self.channel_combo) self.gpu_id = gpu_id self.mode = mode self.immode = immode @@ -1583,7 +1585,7 @@ def __init__( ops.camera_matrix(K, R, t), dtype="float32" ) - print("Init took {} sec.".format(time.time() - ts)) + logging.info(FILE_PATH + ".DataGenerator_3Dconv_tf.__init__ " + "Init took {} sec.".format(time.time() - ts)) def __getitem__(self, index): """Generate one batch of data. @@ -2689,7 +2691,7 @@ def on_epoch_end(self): """Update indexes after each epoch.""" self.indexes = np.arange(len(self.list_IDs)) if self.shuffle == True: - print("SHUFFLING DATA INDICES") + logging.info( FILE_PATH + ".DataGenerator_3Dconv_npy.on_epoch_end " + "SHUFFLING DATA INDICES") np.random.shuffle(self.indexes) def rot90(self, X): diff --git a/dannce/engine/inference.py b/dannce/engine/inference.py index 5e12c40..2bdd153 100755 --- a/dannce/engine/inference.py +++ b/dannce/engine/inference.py @@ -1,5 +1,6 @@ """Handle inference procedures for dannce and com networks. """ +from Dannce.repos.dannce.dannce.engine.generator import FILE_PATH import numpy as np import os import time @@ -12,10 +13,13 @@ import torch import matplotlib from dannce.engine.processing import savedata_tomat, savedata_expval +import logging matplotlib.use("Agg") import matplotlib.pyplot as plt +FILE_PATH = "dannce.engine.inference" + def print_checkpoint( n_frame: int, start_ind: int, end_time: float, sample_save: int = 100 @@ -31,10 +35,11 @@ def print_checkpoint( No Longer Returned: float: New timing reference. """ - print("Predicting on sample %d" % (n_frame), flush=True) + prepend_log_msg = FILE_PATH + ".print_checkpoint " + logging.info(prepend_log_msg + "Predicting on sample %d" % (n_frame), flush=True) if (n_frame - start_ind) % sample_save == 0 and n_frame != start_ind: - print(n_frame) - print("{} samples took {} seconds".format(sample_save, time.time() - end_time)) + logging.info(prepend_log_msg + n_frame) + logging.info(prepend_log_msg + "{} samples took {} seconds".format(sample_save, time.time() - end_time)) end_time = time.time() return end_time @@ -85,6 +90,7 @@ def debug_com( n_batch (int): Batch number n_cam (int): Camera number """ + prepend_log_msg = FILE_PATH + ".debug_com " com_predict_dir = params["com_predict_dir"] cmapdir = os.path.join(com_predict_dir, "cmap") overlaydir = os.path.join(com_predict_dir, "overlay") @@ -92,8 +98,8 @@ def debug_com( os.makedirs(cmapdir) if not os.path.exists(overlaydir): os.makedirs(overlaydir) - print("Writing " + params["com_debug"] + " confidence maps to " + cmapdir) - print("Writing " + params["com_debug"] + "COM-image overlays to " + overlaydir) + logging.info(prepend_log_msg + "Writing " + params["com_debug"] + " confidence maps to " + cmapdir) + logging.info(prepend_log_msg + "Writing " + params["com_debug"] + "COM-image overlays to " + overlaydir) batch_size = pred_batch.shape[0] # Write preds @@ -662,17 +668,17 @@ def infer_dannce( device (Text): Gpu device name n_chn (int): Number of output channels """ - + prepend_log_msg = FILE_PATH + ".infer_dannce " end_time = time.time() for idx, i in enumerate(range(start_ind, end_ind)): - print("Predicting on batch {}".format(i), flush=True) + logging.debug("Predicting on batch {}".format(i), flush=True) if (i - start_ind) % 10 == 0 and i != start_ind: - print(i) - print("10 batches took {} seconds".format(time.time() - end_time)) + logging.debug(i) + logging.debug("10 batches took {} seconds".format(time.time() - end_time)) end_time = time.time() if (i - start_ind) % 1000 == 0 and i != start_ind: - print("Saving checkpoint at {}th batch".format(i)) + logging.debug("Saving checkpoint at {}th batch".format(i)) if params["expval"]: p_n = savedata_expval( params["dannce_predict_dir"] + "save_data_AVG.mat", diff --git a/dannce/engine/nets.py b/dannce/engine/nets.py index b10ec70..ef30cce 100644 --- a/dannce/engine/nets.py +++ b/dannce/engine/nets.py @@ -16,6 +16,23 @@ import numpy as np import h5py import tensorflow as tf +import logging + +FILE_PATH = "dannce.engine.nets" + +def setup_logging(logfile_path, log_lvl, params=None): + if logfile_path != None and log_lvl != None: + logging.basicConfig(filename= logfile_path, + level= log_lvl, + format='%(asctime)s %(levelname)s:%(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p') + elif params != None: + logging.basicConfig(filename=params["log_dest"], + level=params["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p') + else: + print("log file path anf log level, or params dictionary must be passed") def get_metrics(params): """ @@ -38,24 +55,26 @@ def norm_fun( """ method: Normalization method can be "batch", "instance", or "layer" """ + prepend_log_msg = FILE_PATH + ".norm_fun " + method_parse = norm_method.lower() if method_parse.startswith("batch"): - print("using batch normalization") + logging.info(prepend_log_msg + "using batch normalization") def fun(inputs): - print("calling batch norm fun") + logging.info(prepend_log_msg + "calling batch norm fun") return BatchNormalization()(inputs) elif method_parse.startswith("layer"): - print("using layer normalization") + logging.info(prepend_log_msg + "using layer normalization") def fun(inputs): - print("calling layer norm fun") + logging.info(prepend_log_msg + "calling layer norm fun") return ops.InstanceNormalization(axis=None)(inputs) elif method_parse.startswith("instance"): - print("using instance normalization") + logging.info(prepend_log_msg + "using instance normalization") def fun(inputs): - print("calling instance norm fun") + logging.info(prepend_log_msg + "calling instance norm fun") return ops.InstanceNormalization(axis=-1)(inputs) else: def fun(inputs): @@ -501,6 +520,9 @@ def finetune_AVG( num_layers_locked (int) is the number of layers, starting from the input layer, that will be locked (non-trainable) during fine-tuning. """ + + prepend_log_msg = ".finetune_AVG " + # model = netobj() model = unet3d_big_expectedvalue( lossfunc, @@ -519,10 +541,10 @@ def finetune_AVG( post = model.get_weights() - print("evaluating weight deltas in the first conv layer") + logging.info(prepend_log_msg + "evaluating weight deltas in the first conv layer") - print("pre-weights") - print(pre[1][0]) + logging.info(prepend_log_msg + "pre-weights") + logging.info(prepend_log_msg + pre[1][0]) print("post-weights") print(post[1][0]) print("delta:") diff --git a/dannce/engine/ops.py b/dannce/engine/ops.py index 030b6d7..2753cf1 100755 --- a/dannce/engine/ops.py +++ b/dannce/engine/ops.py @@ -708,6 +708,7 @@ def proj_slice( # Compute Xc - points in camera frame Xc = tf.matrix_triangular_solve(K_, rs_grid, lower=False, name="KinvX") + # Leaving this print statement as is. There are no calls going into this print(K.int_shape(Xc)) # Define z values of samples along ray diff --git a/dannce/engine/processing.py b/dannce/engine/processing.py index 657e7b8..cfda81e 100755 --- a/dannce/engine/processing.py +++ b/dannce/engine/processing.py @@ -22,6 +22,9 @@ import time from typing import Dict from tensorflow.keras.models import Model +import logging + +FILE_PATH = "dannce.engine.preprocessing.py" def write_debug( @@ -43,6 +46,8 @@ def write_debug( trainData (bool, optional): If True use training data for debug. Defaults to True. """ + prepend_log_msg = FILE_PATH + ".write_debug " + def plot_out(imo, lo, imn): plot_markers_2d(norm_im(imo), lo, newfig=False) plt.gca().xaxis.set_major_locator(plt.NullLocator()) @@ -65,7 +70,7 @@ def plot_out(imo, lo, imn): # Plot all training images and save # create new directory for images if necessary debugdir = os.path.join(params["com_train_dir"], outdir) - print("Saving debug images to: " + debugdir) + logging.info(prepend_log_msg + "Saving debug images to: " + debugdir) if not os.path.exists(debugdir): os.makedirs(debugdir) @@ -85,7 +90,7 @@ def plot_out(imo, lo, imn): plot_out(ims_out[i], label_out[i], str(i) + ".png") elif params["debug"] and params["multi_mode"]: - print("Note: Cannot output debug information in COM multi-mode") + logging.info( prepend_log_msg + "Note: Cannot output debug information in COM multi-mode") def initialize_vids(params, datadict, e, vids, pathonly=True): @@ -146,6 +151,10 @@ def infer_params(params, dannce_net, prediction): Some parameters that were previously specified in configs can just be inferred from others, thus relieving config bloat """ + # Setting up logging + logging.basicConfig(filename=params["log_dest"], level=params["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + # Grab the camnames from *dannce.mat if not in config if params["camnames"] is None: f = grab_predict_label3d_file() @@ -339,8 +348,9 @@ def infer_params(params, dannce_net, prediction): def print_and_set(params, varname, value): # Should add new values to params in place, no need to return + prepend_log_msg = FILE_PATH + ".print_and_set " params[varname] = value - print("Setting {} to {}.".format(varname, params[varname])) + logging.info(prepend_log_msg + "Setting {} to {}.".format(varname, params[varname])) def check_config(params, dannce_net, prediction): @@ -426,6 +436,7 @@ def copy_config(results_dir, main_config, io_config): Copies config files into the results directory, and creates results directory if necessary """ + # Leaving this print statement as is since there does not seem to be any references going out or into it print("Saving results to: {}".format(results_dir)) if not os.path.exists(results_dir): @@ -445,6 +456,8 @@ def make_data_splits(samples, params, results_dir, num_experiments): Make train/validation splits from list of samples, or load in a specific list of sampleIDs if desired. """ + # Setup prepend for log messages + prepend_log_msg = FILE_PATH + ".make_data_split " # TODO: Switch to .mat from .pickle so that these lists are easier to read # and change. @@ -501,7 +514,7 @@ def make_data_splits(samples, params, results_dir, num_experiments): else: train_expts = np.arange(num_experiments) - print("TRAIN EXPTS: {}".format(train_expts)) + logging.info(prepend_log_msg + "TRAIN EXPTS: {}".format(train_expts)) if params["num_train_per_exp"] is not None: # Then sample randomly without replacement from training sampleIDs @@ -511,8 +524,8 @@ def make_data_splits(samples, params, results_dir, num_experiments): for i in range(len(train_samples)) if int(train_samples[i].split("_")[0]) == e ] - print(e) - print(len(tinds)) + logging.debug(e) + logging.debug(len(tinds)) train_inds = train_inds + list( np.random.choice( tinds, (params["num_train_per_exp"],), replace=False @@ -607,6 +620,7 @@ def remove_samples_npy(npydir, samples, params): Remove any samples from sample list if they do not have corresponding volumes in the image or grid directories """ + prepend_log_msg = FILE_PATH + ".remove_samples_npy " # image_volumes # grid_volumes samps = [] @@ -630,7 +644,7 @@ def remove_samples_npy(npydir, samples, params): sampdiff = len(npysamps) - len(goodsamps) # import pdb; pdb.set_trace() - print( + logging.info(prepend_log_msg + "Removed {} samples from {} because corresponding image or grid files could not be found".format( sampdiff, params["experiment"][e]["label3d_file"] ) @@ -745,13 +759,18 @@ def save_COM_dannce_mat(params, com3d, sampleID): Instead of saving 3D COM to com3d.mat, save it into the dannce.mat file, which streamlines subsequent dannce access. """ + # Setup Logging + logging.basicConfig(filename=params["log_dest"], level=params["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + prepend_log_msg = FILE_PATH + ".save_COM_dannce_mat " + com = {} com["com3d"] = com3d com["sampleID"] = sampleID com["metadata"] = prepare_save_metadata(params) # Open dannce.mat file, add com and re-save - print("Saving COM predictions to " + params["label3d_file"]) + logging.info(prepend_log_msg + "Saving COM predictions to " + params["label3d_file"]) rr = sio.loadmat(params["label3d_file"]) # For safety, save old file to temp and delete it at the end sio.savemat(params["label3d_file"] + ".temp", rr) @@ -768,6 +787,9 @@ def save_COM_checkpoint( Saves COM pickle and matfiles """ + # Set Prepend log message + prepend_log_msg = FILE_PATH + ".save_COM_dannce_mat " + # Save undistorted 2D COMs and their 3D triangulations f = open(os.path.join(results_dir, file_name + ".pickle"), "wb") cPickle.dump(save_data, f) @@ -804,7 +826,7 @@ def save_COM_checkpoint( ) cfilename = os.path.join(results_dir, file_name + ".mat") - print("Saving 3D COM to {}".format(cfilename)) + logging.info(prepend_log_msg + "Saving 3D COM to {}".format(cfilename)) samples_keys = list(com3d_dict.keys()) if params["n_instances"] > 1: @@ -836,6 +858,7 @@ def inherit_config(child, parent, keys): for key in keys: if key not in child.keys(): child[key] = parent[key] + # Leaving this print statement as is, since params are not built at this point print( "{} not found in io.yaml file, falling back to main config".format(key) ) @@ -847,6 +870,8 @@ def grab_predict_label3d_file(defaultdir=""): """ Finds the paths to the training experiment yaml files. """ + # Set Logging prepend message + prepend_log_msg = FILE_PATH + ".grab_predict_label3d_file " def_ep = os.path.join(".", defaultdir) label3d_files = os.listdir(def_ep) label3d_files = [ @@ -856,7 +881,7 @@ def grab_predict_label3d_file(defaultdir=""): if len(label3d_files) == 0: raise Exception("Did not find any *dannce.mat file in {}".format(def_ep)) - print("Using the following *dannce.mat files: {}".format(label3d_files[0])) + logging.debug(prepend_log_msg + "Using the following *dannce.mat files: {}".format(label3d_files[0])) return label3d_files[0] @@ -865,6 +890,9 @@ def load_expdict(params, e, expdict, _DEFAULT_VIDDIR): Load in camnames and video directories and label3d files for a single experiment during training. """ + # Set Logging prepend msg + prepend_log_msg = FILE_PATH + ".load_expdict " + _DEFAULT_NPY_DIR = "npy_volumes" exp = params.copy() exp = make_paths_safe(exp) @@ -877,14 +905,14 @@ def load_expdict(params, e, expdict, _DEFAULT_VIDDIR): exp["viddir"] = os.path.join(exp["base_exp_folder"], _DEFAULT_VIDDIR) else: exp["viddir"] = expdict["viddir"] - print("Experiment {} using videos in {}".format(e, exp["viddir"])) + logging.debug(prepend_log_msg + "Experiment {} using videos in {}".format(e, exp["viddir"])) l3d_camnames = io.load_camnames(expdict["label3d_file"]) if "camnames" in expdict: exp["camnames"] = expdict["camnames"] elif l3d_camnames is not None: exp["camnames"] = l3d_camnames - print("Experiment {} using camnames: {}".format(e, exp["camnames"])) + logging.debug(prepend_log_msg + "Experiment {} using camnames: {}".format(e, exp["camnames"])) # Use the camnames to find the chunks for each video chunks = {} @@ -902,7 +930,7 @@ def load_expdict(params, e, expdict, _DEFAULT_VIDDIR): [int(x.split(".")[0]) for x in video_files] ) exp["chunks"] = chunks - print(chunks) + logging.debug(prepend_log_msg + chunks) # For npy volume training if params["use_npy"]: @@ -1028,6 +1056,9 @@ def generate_readers( viddir, camname, minopt=0, maxopt=300000, pathonly=False, extension=".mp4" ): """Open all mp4 objects with imageio, and return them in a dictionary.""" + # Set prepend message + prepend_log_msg = FILE_PATH + ".generate_readers " + out = {} mp4files = [ os.path.join(camname, f) @@ -1054,7 +1085,7 @@ def generate_readers( if pathonly: out[mp4files_scrub[i]] = os.path.join(viddir, mp4files[i]) else: - print( + logging.info( "NOTE: Ignoring {} files numbered above {}".format(extensions, maxopt) ) out[mp4files_scrub[i]] = imageio.get_reader( @@ -1439,6 +1470,9 @@ def dupe_params(exp, dupes): 2-camera system), automatically duplicate necessary parameters to match the required n_views. """ + # Set log prepend msg + prepend_log_msg = FILE_PATH + ".dupe_params " + n_views = exp["n_views"] for d in dupes: val = exp[d] @@ -1457,7 +1491,7 @@ def dupe_params(exp, dupes): duped = val * num_reps for i in range(num_extra): duped.append(duped[i]) - print("Duping {}. Changed from {} to {}".format(d, val, duped)) + logging.info(prepend_log_msg + "Duping {}. Changed from {} to {}".format(d, val, duped)) exp[d] = duped else: raise Exception( @@ -1474,6 +1508,9 @@ def write_npy(uri, gen): Creates a new image folder and grid folder at the uri and uses the generator to generate samples and save them as npy files """ + # Set log prepend msg + prepend_log_msg = FILE_PATH + ".write_npy " + imdir = os.path.join(uri, "image_volumes") if not os.path.exists(imdir): os.makedirs(imdir) @@ -1494,7 +1531,7 @@ def write_npy(uri, gen): bs = gen.batch_size for i in range(len(gen)): if i % 1000 == 0: - print(i) + logging.debug(i) # Generate batch bch = gen.__getitem__(i) # loop over all examples in batch and save volume @@ -1503,6 +1540,6 @@ def write_npy(uri, gen): fname = gen.list_IDs[gen.indexes[i * bs + j]] # and save - print(fname) + logging.debug(fname) np.save(os.path.join(imdir, fname + ".npy"), bch[0][0][j].astype("uint8")) np.save(os.path.join(griddir, fname + ".npy"), bch[0][1][j]) diff --git a/dannce/engine/serve_data_DANNCE.py b/dannce/engine/serve_data_DANNCE.py index 4b3b59e..1443f2d 100755 --- a/dannce/engine/serve_data_DANNCE.py +++ b/dannce/engine/serve_data_DANNCE.py @@ -9,6 +9,9 @@ from scipy.ndimage import median_filter import warnings from copy import deepcopy +import logging + +FILE_PATH = "dannce.engine.serve_data_DANNCE" def prepare_data( @@ -27,6 +30,9 @@ def prepare_data( multimode: when this True, we output all 2D markers AND their 2D COM """ + # Set Log Prepend Msg + prepend_log_msg = FILE_PATH + ".prepare_data " + if prediction: labels = load_sync(params["label3d_file"]) nFrames = np.max(labels[0]["data_frame"].shape) @@ -40,7 +46,7 @@ def prepare_data( # import pdb # pdb.set_trace() else: - print(params["label3d_file"]) + logging.info(prepend_log_msg + params["label3d_file"]) labels = load_labels(params["label3d_file"]) camera_params = load_camera_params(params["label3d_file"]) @@ -87,7 +93,7 @@ def prepare_data( data[:, 1] = params["raw_im_h"] - data[:, 1] - 1 if params["multi_mode"]: - print("Entering multi-mode with {} + 1 targets".format(data.shape[-1])) + logging.debug(prepend_log_msg + "Entering multi-mode with {} + 1 targets".format(data.shape[-1])) if nanflag: dcom = np.mean(data, axis=2, keepdims=True) else: @@ -108,7 +114,7 @@ def prepare_data( # If specific markers are set to be excluded, set them to NaN here. if params["drop_landmark"] is not None and not prediction: - print( + logging.debug( prepend_log_msg + "Setting landmarks {} to NaN. These landmarks will not be included in loss or metric evaluations".format( params["drop_landmark"] ) @@ -243,16 +249,18 @@ def prepare_COM( detected by the generator to return nans such that bad camera frames do not get averaged in to image data """ + # Set log prepend msg + prepend_log_msg = FILE_PATH + ".prepare_COM " with open(comfile, "rb") as f: com = cPickle.load(f) com3d_dict = {} if method == "mean": - print("using mean to get 3D COM") + logging.debug(prepend_log_msg + "using mean to get 3D COM") elif method == "median": - print("using median to get 3D COM") + logging.debug(prepend_log_msg + "using median to get 3D COM") firstkey = list(com.keys())[0] @@ -395,6 +403,7 @@ def remove_samples(s, d3d, mode="clean", auxmode=None): sample_mask[i] = 0 if auxmode == "JDM52d2": + # Leaving this print statement as is, since there are no calls to this module print("removing bad JDM52d2 frames") for i in range(len(s)): if s[i] >= 20000 and s[i] <= 32000: diff --git a/dannce/engine/video.py b/dannce/engine/video.py index 36358d0..a52bce8 100755 --- a/dannce/engine/video.py +++ b/dannce/engine/video.py @@ -1,6 +1,7 @@ """ Video reading and writing interfaces for different formats. """ from copy import Error import os +from Dannce.Experimental.exp1file1 import FILE_PATH import cv2 import numpy as np import attr @@ -8,6 +9,9 @@ import imageio import time from typing import List, Dict, Tuple, Text +import logging + +FILE_PATH = "dannce.engine.video" @attr.s(auto_attribs=True, eq=False, order=False) @@ -201,6 +205,9 @@ def load_vid_frame( Returns: np.ndarray: Video frame as w x h x c numpy ndarray """ + # Set log prepend msg + prepend_log_msg = FILE_PATH + ".LoadVideoFrame.load_vid_frame " + chunks = self._N_VIDEO_FRAMES[camname] cur_video_id = np.nonzero([c <= ind for c in chunks])[0][-1] cur_first_frame = chunks[cur_video_id] @@ -229,7 +236,7 @@ def load_vid_frame( if self.predict_flag else MediaVideo(thisvid_name, grayscale=False) ) - print("Loading new video: {} for {}".format(abname, camname)) + logging.info(prepend_log_msg + "Loading new video: {} for {}".format(abname, camname)) self.currvideo_name[camname] = abname # close current vid # Without a sleep here, ffmpeg can hang on video close @@ -264,6 +271,9 @@ def _load_frame_multiple_attempts(self, frame_num, vid, n_attempts=10): return im def _load_frame(self, frame_num, vid): + # Set log prepend msg + prepend_log_msg = FILE_PATH + ".LoadVideoFrame._load_frame " + im = None try: im = ( @@ -273,7 +283,7 @@ def _load_frame(self, frame_num, vid): ) # This deals with a strange indexing error in the pup data. except IndexError: - print("Indexing error, using previous frame") + logging.error(prepend_log_msg + "Indexing error, using previous frame") im = ( vid.get_data(frame_num - 1).astype("uint8") if self.predict_flag diff --git a/dannce/interface.py b/dannce/interface.py index 5a444b6..4a72664 100755 --- a/dannce/interface.py +++ b/dannce/interface.py @@ -29,8 +29,10 @@ import dannce.engine.inference as inference from typing import List, Dict, Text import os, psutil +import logging process = psutil.Process(os.getpid()) +file_path = "dannce.interface" _DEFAULT_VIDDIR = "videos" _DEFAULT_COMSTRING = "COM" @@ -106,6 +108,11 @@ def com_predict(params: Dict): Args: params (Dict): Parameters dictionary. """ + # Enable Logging for com_predict + logging.basicConfig(filename=params["log_dest"], level=params["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + prepend_log_msg = file_path + ".com_predict " + params = setup_com_predict(params) # Get the model @@ -201,7 +208,7 @@ def com_predict(params: Dict): file_name="com3d%d" % (params["start_sample"]), ) - print("done!") + logging.info(prepend_log_msg+"done!") def setup_com_predict(params: Dict): @@ -210,6 +217,8 @@ def setup_com_predict(params: Dict): Args: params (Dict): Parameters dictionary """ + # Prepend part for logging + prepend_log_msg = file_path + ".setup_com_predict " # Make the prediction directory if it does not exist. make_folder("com_predict_dir", params) @@ -231,7 +240,7 @@ def setup_com_predict(params: Dict): # Grab the input file for prediction params["label3d_file"] = processing.grab_predict_label3d_file() - print("Using camnames: {}".format(params["camnames"])) + logging.info(prepend_log_msg+"Using camnames: {}".format(params["camnames"])) # Also add parent params under the 'experiment' key for compatibility # with DANNCE's video loading function @@ -252,11 +261,14 @@ def build_com_network(params: Dict) -> Model: Returns: Model: com network """ + # Prepend Logging message for build_com_network + prepend_log_msg = file_path + ".build_com_network " + # channels out is equal to the number of views when using a single video stream with mirrors eff_n_channels_out = ( int(params["n_views"]) if params["mirror"] else params["n_channels_out"] ) - print("Initializing Network...") + logging.info("Initializing Network...") # Build net model = params["net"]( params["loss"], @@ -274,10 +286,10 @@ def build_com_network(params: Dict) -> Model: weights = weights[-1] params["com_predict_weights"] = os.path.join(params["com_train_dir"], weights) - print("Loading weights from " + params["com_predict_weights"]) + logging.info(prepend_log_msg + "Loading weights from " + params["com_predict_weights"]) model.load_weights(params["com_predict_weights"]) - print("COMPLETE\n") + logging.info(prepend_log_msg+"COMPLETE\n") return model @@ -320,6 +332,12 @@ def com_train(params: Dict): Args: params (Dict): Parameters dictionary. """ + + # Setup Logging for com_train + logging.basicConfig(filename=params["log_dest"], level=params["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + prepend_log_msg = file_path + ".com_train " + params = setup_com_train(params) # Use the same label files and experiment settings as DANNCE unless @@ -387,7 +405,7 @@ def com_train(params: Dict): for e in range(num_experiments): vids = processing.initialize_vids(params, datadict, e, vids, pathonly=True) - print("Using {} downsampling".format(params["dsmode"])) + logging.info(prepend_log_msg + "Using {} downsampling".format(params["dsmode"])) train_params = { "dim_in": ( @@ -427,7 +445,7 @@ def com_train(params: Dict): ) # Build net - print("Initializing Network...") + logging.info(prepend_log_msg + "Initializing Network...") model = params["net"]( params["loss"], @@ -437,7 +455,7 @@ def com_train(params: Dict): params["norm_method"], ["mse"], ) - print("COMPLETE\n") + logging.info(prepend_log_msg + "COMPLETE\n") if params["com_finetune_weights"] is not None: weights = os.listdir(params["com_finetune_weights"]) @@ -447,7 +465,7 @@ def com_train(params: Dict): try: model.load_weights(os.path.join(params["com_finetune_weights"], weights)) except: - print( + logging.error( prepend_log_msg + "Note: model weights could not be loaded due to a mismatch in dimensions.\ Assuming that this is a fine-tune with a different number of outputs and removing \ the top of the net accordingly" @@ -545,15 +563,15 @@ def com_train(params: Dict): partition["valid_sampleIDs"], labels, vids, **valid_params ) - print("Loading data") + logging.info(prepend_log_msg + "Loading data") for i in range(len(partition["train_sampleIDs"])): - print(i, end="\r") + logging.debug(i, end="\r") ims = train_generator.__getitem__(i) ims_train[i * ncams : (i + 1) * ncams] = ims[0] y_train[i * ncams : (i + 1) * ncams] = ims[1] for i in range(len(partition["valid_sampleIDs"])): - print(i, end="\r") + logging.debug(i, end="\r") ims = valid_generator.__getitem__(i) ims_valid[i * ncams : (i + 1) * ncams] = ims[0] y_valid[i * ncams : (i + 1) * ncams] = ims[1] @@ -604,10 +622,10 @@ def com_train(params: Dict): params, ims_train, ims_valid, y_train, model, trainData=False ) - print("Renaming weights file with best epoch description") + logging.info(prepend_log_msg + "Renaming weights file with best epoch description") processing.rename_weights(params["com_train_dir"], kkey, mon) - print("Saving full model at end of training") + logging.info(prepend_log_msg + "Saving full model at end of training") sdir = os.path.join(params["com_train_dir"], "fullmodel_weights") if not os.path.exists(sdir): os.makedirs(sdir) @@ -639,6 +657,12 @@ def dannce_train(params: Dict): Raises: Exception: Error if training mode is invalid. """ + + # Setup Logging for com_train + logging.basicConfig(filename=params["log_dest"], level=params["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + prepend_log_msg = file_path + ".dannce_train " + # Depth disabled until next release. params["depth"] = False params["multi_mode"] = False @@ -674,7 +698,7 @@ def dannce_train(params: Dict): # find the weights given config path if params["dannce_finetune_weights"] is not None: params["dannce_finetune_weights"] = processing.get_ft_wt(params) - print("Fine-tuning from {}".format(params["dannce_finetune_weights"])) + logging.info(prepend_log_msg + "Fine-tuning from {}".format(params["dannce_finetune_weights"])) samples = [] datadict = {} @@ -700,7 +724,7 @@ def dannce_train(params: Dict): com3d_dict_, ) = do_COM_load(exp, expdict, e, params) - print("Using {} samples total.".format(len(samples_))) + logging.debug(prepend_log_msg + "Using {} samples total.".format(len(samples_))) ( samples, @@ -721,7 +745,7 @@ def dannce_train(params: Dict): cameras[e] = cameras_ camnames[e] = exp["camnames"] - print("Using the following cameras: {}".format(camnames[e])) + logging.debug(prepend_log_msg + "Using the following cameras: {}".format(camnames[e])) params["experiment"][e] = exp for name, chunk in exp["chunks"].items(): total_chunks[name] = chunk @@ -907,13 +931,13 @@ def dannce_train(params: Dict): dtype="float32", ) - print( - "Loading training data into memory. This can take a while to seek through", - "large sets of video. This process is much faster if the frame indices", + logging.info( prepend_log_msg + + "Loading training data into memory. This can take a while to seek through" + + "large sets of video. This process is much faster if the frame indices" + "are sorted in ascending order in your label data file.", ) for i in range(len(partition["train_sampleIDs"])): - print(i, end="\r") + logging.debug(i, end="\r") rr = train_generator.__getitem__(i) if params["expval"]: X_train[i] = rr[0][0] @@ -928,7 +952,7 @@ def dannce_train(params: Dict): # This can be used for debugging problems with calibration or # COM estimation tifdir = params["debug_volume_tifdir"] - print("Dump training volumes to {}".format(tifdir)) + logging.info(prepend_log_msg + "Dump training volumes to {}".format(tifdir)) for i in range(X_train.shape[0]): for j in range(len(camnames[0])): im = X_train[ @@ -947,9 +971,9 @@ def dannce_train(params: Dict): imageio.mimwrite(of, np.transpose(im, [2, 0, 1, 3])) return - print("Loading validation data into memory") + logging.info(prepend_log_msg + "Loading validation data into memory") for i in range(len(partition["valid_sampleIDs"])): - print(i, end="\r") + logging.debug(i, end="\r") rr = valid_generator.__getitem__(i) if params["expval"]: X_valid[i] = rr[0][0] @@ -977,12 +1001,12 @@ def dannce_train(params: Dict): randflag = True if params["n_rand_views"] == 0: - print( + logging.info( prepend_log_msg + "Using default n_rand_views augmentation with {} views and with replacement".format( params["n_views"] ) ) - print("To disable n_rand_views augmentation, set it to None in the config.") + logging.info( prepend_log_msg + "To disable n_rand_views augmentation, set it to None in the config.") params["n_rand_views"] = params["n_views"] params["rand_view_replace"] = True @@ -1079,7 +1103,7 @@ def dannce_train(params: Dict): valid_generator = genfunc(**args_valid) # Build net - print("Initializing Network...") + logging.info( prepend_log_msg + "Initializing Network...") # Currently, we expect four modes of use: # 1) Training a new network from scratch @@ -1088,10 +1112,10 @@ def dannce_train(params: Dict): # if params["multi_gpu_train"]: strategy = tf.distribute.MirroredStrategy() - print("Number of devices: {}".format(strategy.num_replicas_in_sync)) + logging.info(prepend_log_msg + "Number of devices: {}".format(strategy.num_replicas_in_sync)) scoping = strategy.scope() - print("NUM CAMERAS: {}".format(len(camnames[0]))) + logging.info(prepend_log_msg + "NUM CAMERAS: {}".format(len(camnames[0]))) with scoping: if params["train_mode"] == "new": @@ -1123,7 +1147,7 @@ def dannce_train(params: Dict): model = params["net"](*fargs) except: if params["expval"]: - print( + logging.warning(prepend_log_msg + "Could not load weights for finetune (likely because you are finetuning a previously finetuned network). Attempting to finetune from a full finetune model file." ) model = nets.finetune_fullmodel_AVG(*fargs) @@ -1180,13 +1204,13 @@ def dannce_train(params: Dict): ) if params["lr"] != model.optimizer.learning_rate: - print("Changing learning rate to {}".format(params["lr"])) + logging.debug(prepend_log_msg + "Changing learning rate to {}".format(params["lr"])) K.set_value(model.optimizer.learning_rate, params["lr"]) - print( + logging.debug( prepend_log_msg + "Confirming new learning rate: {}".format(model.optimizer.learning_rate) ) - print("COMPLETE\n") + logging.info(prepend_log_msg + "COMPLETE\n") # Create checkpoint and logging callbacks kkey = "weights.hdf5" @@ -1228,7 +1252,8 @@ def on_epoch_end(self, epoch, logs=None): or logs[lkey] < self.val_loss and epoch > 25 ): - print( + logging.info( + prepend_log_msg + "Saving predictions on train and validation data, after epoch {}".format( epoch ) @@ -1299,7 +1324,7 @@ def on_epoch_end(self, epoch, logs=None): # Calculate euclidean_distance_3d e3d = K.eval(losses.euclidean_distance_3D(self.vLabel, pred_out_world)) - print("epoch {} euclidean_distance_3d: {}".format(epoch, e3d)) + logging.info("epoch {} euclidean_distance_3d: {}".format(epoch, e3d)) with open(self.fn, "a") as fd: fd.write("{},{}\n".format(epoch, e3d)) @@ -1314,7 +1339,7 @@ def on_epoch_end(self, epoch, logs=None): if epoch in self.saveE: # Do a garbage collect to combat keras memory leak gc.collect() - print("Saving checkpoint weights at epoch {}".format(epoch)) + logging.info( prepend_log_msg + "Saving checkpoint weights at epoch {}".format(epoch)) savename = "weights.checkpoint.epoch{}.{}{:.5f}.hdf5".format( epoch, lkey, val_loss ) @@ -1369,10 +1394,10 @@ def on_epoch_end(self, epoch, logs=None): # workers=6, ) - print("Renaming weights file with best epoch description") + logging.info(prepend_log_msg + "Renaming weights file with best epoch description") processing.rename_weights(dannce_train_dir, kkey, mon) - print("Saving full model at end of training") + logging.info(prepend_log_msg + "Saving full model at end of training") sdir = os.path.join(params["dannce_train_dir"], "fullmodel_weights") if not os.path.exists(sdir): os.makedirs(sdir) @@ -1380,7 +1405,7 @@ def on_epoch_end(self, epoch, logs=None): model = nets.remove_heatmap_output(model, params) model.save(os.path.join(sdir, "fullmodel_end.hdf5")) - print("done!") + logging.info(prepend_log_msg + "done!") def dannce_predict(params: Dict): @@ -1389,6 +1414,12 @@ def dannce_predict(params: Dict): Args: params (Dict): Paremeters dictionary. """ + + # Setup Logging for com_train + logging.basicConfig(filename=params["log_dest"], level=params["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + prepend_log_msg = file_path + ".dannce_predict " + # Depth disabled until next release. params["depth"] = False # Make the prediction directory if it does not exist. @@ -1419,9 +1450,9 @@ def dannce_predict(params: Dict): params["predict_mode"] if params["predict_mode"] is not None else "numpy" ) params["multi_mode"] = False - print("Using {} predict mode".format(params["predict_mode"])) + logging.info(prepend_log_msg + "Using {} predict mode".format(params["predict_mode"])) - print("Using camnames: {}".format(params["camnames"])) + logging.info(prepend_log_msg + "Using camnames: {}".format(params["camnames"])) # Also add parent params under the 'experiment' key for compatibility # with DANNCE's video loading function params["experiment"] = {} @@ -1560,7 +1591,8 @@ def dannce_predict(params: Dict): max_eval_batch = params["maxbatch"] if max_eval_batch != "max" and max_eval_batch > len(valid_generator): - print( + logging.info( + prepend_log_msg + "Maxbatch was set to a larger number of matches than exist in the video. Truncating" ) max_eval_batch = len(valid_generator) @@ -1585,7 +1617,7 @@ def dannce_predict(params: Dict): # for working with large datasets (such as Rat 7M) because # .npy files can be loaded in quickly with random access # during training. - print("Writing samples to .npy files") + logging.info(prepend_log_msg + "Writing samples to .npy files") processing.write_npy(params["write_npy"], valid_generator) return @@ -1638,8 +1670,9 @@ def dannce_predict(params: Dict): def write_com_file(params, samples_, com3d_dict_): + prepend_log_msg = file_path + ".write_com_file " cfilename = os.path.join(params["dannce_predict_dir"], "com3d_used.mat") - print("Saving 3D COM to {}".format(cfilename)) + logging.info(prepend_log_msg + "Saving 3D COM to {}".format(cfilename)) c3d = np.zeros((len(samples_), 3)) for i in range(len(samples_)): c3d[i] = com3d_dict_[samples_[i]] @@ -1648,7 +1681,8 @@ def write_com_file(params, samples_, com3d_dict_): def build_model(params, camnames): # Build net - print("Initializing Network...") + prepend_log_msg = file_path + ".build_model " + logging.info(prepend_log_msg + "Initializing Network...") # This requires that the network be saved as a full model, not just weights. # As a precaution, we import all possible custom objects that could be used @@ -1668,7 +1702,7 @@ def build_model(params, camnames): # set this file to dannce_predict_model so that it will still get saved with metadata params["dannce_predict_model"] = mdl_file - print("Loading model from " + mdl_file) + logging.info(prepend_log_msg + "Loading model from " + mdl_file) if ( params["net_name"] == "unet3d_big_tiedfirstlayer_expectedvalue" @@ -1678,7 +1712,8 @@ def build_model(params, camnames): params["dannce_finetune_weights"] = processing.get_ft_wt(params) if params["train_mode"] == "finetune": - print( + logging.info( + prepend_log_msg + "Initializing a finetune network from {}, into which weights from {} will be loaded.".format( params["dannce_finetune_weights"], mdl_file ) @@ -1759,6 +1794,9 @@ def do_COM_load(exp: Dict, expdict: Dict, e, params: Dict, training=True): Raises: Exception: Exception when invalid com file format. """ + # Set Prepend logging message + prepend_log_msg = file_path + ".do_COM_load " + ( samples_, datadict_, @@ -1775,7 +1813,7 @@ def do_COM_load(exp: Dict, expdict: Dict, e, params: Dict, training=True): # If there is "clean" data (full marker set), can take the # 3D COM from the labels if exp["com_fromlabels"] and training: - print("For experiment {}, calculating 3D COM from labels".format(e)) + logging.info(prepend_log_msg + "For experiment {}, calculating 3D COM from labels".format(e)) com3d_dict_ = deepcopy(datadict_3d_) for key in com3d_dict_.keys(): com3d_dict_[key] = np.nanmean(datadict_3d_[key], axis=1, keepdims=True) @@ -1805,10 +1843,11 @@ def do_COM_load(exp: Dict, expdict: Dict, e, params: Dict, training=True): c3dfile = io.load_com(exp["com_file"]) com3d_dict_ = check_COM_load(c3dfile, "com3d", params["medfilt_window"]) - print("Experiment {} using com3d: {}".format(e, exp["com_file"])) + logging.info(prepend_log_msg + "Experiment {} using com3d: {}".format(e, exp["com_file"])) if params["medfilt_window"] is not None: - print( + logging.info( + prepend_log_msg + "Median filtering COM trace with window size {}".format( params["medfilt_window"] ) @@ -1825,7 +1864,7 @@ def do_COM_load(exp: Dict, expdict: Dict, e, params: Dict, training=True): cthresh=exp["cthresh"], ) msg = "Removed {} samples from the dataset because they either had COM positions over cthresh, or did not have matching sampleIDs in the COM file" - print(msg.format(pre - len(samples_))) + logging.info(prepend_log_msg + msg.format(pre - len(samples_))) return exp, samples_, datadict_, datadict_3d_, cameras_, com3d_dict_ @@ -1841,13 +1880,16 @@ def check_COM_load(c3dfile: Dict, kkey: Text, win_size: int): Returns: Dict: Dictionary containing com data. """ + # Set logging prepend message + prepend_log_msg = file_path + ".check_COM_load " + c3d = c3dfile[kkey] # do a median filter on the COM traces if indicated if win_size is not None: if win_size % 2 == 0: win_size += 1 - print("medfilt_window was not odd, changing to: {}".format(win_size)) + logging.info(prepend_log_msg + "medfilt_window was not odd, changing to: {}".format(win_size)) from scipy.signal import medfilt From 9128a5c459ceae83f4afd99953ff679af071f4ff Mon Sep 17 00:00:00 2001 From: data hound Date: Sun, 30 Jan 2022 01:16:58 +0530 Subject: [PATCH 3/9] Resolving conflicts in processing.py --- dannce/engine/processing.py | 45 ------------------------------------- 1 file changed, 45 deletions(-) diff --git a/dannce/engine/processing.py b/dannce/engine/processing.py index 19318c5..aa3ac2b 100755 --- a/dannce/engine/processing.py +++ b/dannce/engine/processing.py @@ -1461,51 +1461,6 @@ def spatial_entropy(map_): return -1 * np.sum(map_ * np.log(map_)) -<<<<<<< HEAD -======= -def dupe_params(exp, dupes): - """ - When The number of views (n_views) required - as input to the network is greater than the - number of actual cameras (e.g. when trying to - fine-tune a 6-camera network on data from a - 2-camera system), automatically duplicate necessary - parameters to match the required n_views. - """ - # Set log prepend msg - prepend_log_msg = FILE_PATH + ".dupe_params " - - n_views = exp["n_views"] - for d in dupes: - val = exp[d] - if n_views % len(val) == 0: - num_reps = n_views // len(val) - exp[d] = val * num_reps - - else: - prompt = "The length of the {} list must divide evenly into {}. Duplicate a subset of the views starting from the first camera (y/n)?".format( - d, n_views - ) - val_in = input(prompt) - if val_in == "y": - num_reps = n_views // len(val) - num_extra = n_views % len(val) - duped = val * num_reps - for i in range(num_extra): - duped.append(duped[i]) - logging.info(prepend_log_msg + "Duping {}. Changed from {} to {}".format(d, val, duped)) - exp[d] = duped - else: - raise Exception( - "The length of the {} list must divide evenly into {}. Exiting".format( - d, n_views - ) - ) - - return exp - - ->>>>>>> a7f82e21a4ad99ffcec34a4659edd7fa869a07b0 def write_npy(uri, gen): """ Creates a new image folder and grid folder at the uri and uses From 25f34b8be0a327fd2f8842b291fd4b8aa0635f83 Mon Sep 17 00:00:00 2001 From: Anshuman Sabath Date: Tue, 12 Apr 2022 12:10:01 -0400 Subject: [PATCH 4/9] Added a print statement to interface --- dannce/interface.py | 1 + demo/markerless_mouse_1/videos/link_to_videos.txt | 0 2 files changed, 1 insertion(+) mode change 100644 => 100755 demo/markerless_mouse_1/videos/link_to_videos.txt diff --git a/dannce/interface.py b/dannce/interface.py index e56b39f..e98fc44 100755 --- a/dannce/interface.py +++ b/dannce/interface.py @@ -60,6 +60,7 @@ def check_unrecognized_params(params: Dict): in_com = key in _param_defaults_com in_dannce = key in _param_defaults_dannce in_shared = key in _param_defaults_shared + print (in_com, in_dannce, in_shared) if not (in_com or in_dannce or in_shared): invalid_keys.append(key) diff --git a/demo/markerless_mouse_1/videos/link_to_videos.txt b/demo/markerless_mouse_1/videos/link_to_videos.txt old mode 100644 new mode 100755 From 7fab6df1fa0331ec6ddc0556e087729369cea483 Mon Sep 17 00:00:00 2001 From: Anshuman Sabath Date: Fri, 15 Apr 2022 16:29:50 -0400 Subject: [PATCH 5/9] Added check for confirming the log directory location --- cluster/duke.yaml | 2 +- configs/dannce_mouse_config.yaml | 5 ++++- dannce/cli.py | 2 +- dannce/engine/generator.py | 6 +++--- dannce/engine/inference.py | 6 +++--- dannce/engine/nets.py | 5 +++++ dannce/engine/processing.py | 9 +++++++++ dannce/engine/video.py | 1 - dannce/interface.py | 10 ++++++++++ setup.py | 15 ++++++++++++--- 10 files changed, 48 insertions(+), 13 deletions(-) diff --git a/cluster/duke.yaml b/cluster/duke.yaml index a0e9fcb..049f05f 100644 --- a/cluster/duke.yaml +++ b/cluster/duke.yaml @@ -10,4 +10,4 @@ com_predict: "--job-name=predictCom -p gpu-common,dsplus-gpu --mem=10000 -t 3-00 com_multi_predict: "--job-name=predictCom -p gpu-common,dsplus-gpu --mem=10000 -t 0-03:00 --gres=gpu:1 -N 1 -n 8 --account=plusds" # Setup functions (optional, set to "" if no setup is required. Trailing ; is required) -setup: ". ~/.bashrc; conda activate dannce_tf26;" \ No newline at end of file +setup: ". ~/.bashrc; conda activate dannce_cuda1;" \ No newline at end of file diff --git a/configs/dannce_mouse_config.yaml b/configs/dannce_mouse_config.yaml index 73473b6..4f36d85 100755 --- a/configs/dannce_mouse_config.yaml +++ b/configs/dannce_mouse_config.yaml @@ -33,4 +33,7 @@ nvox: 64 max_num_samples: 1000 # By default, will load in the first hdf5 file at this location for fine-tuning. If training from scratch, set to None -dannce_finetune_weights: ./DANNCE/weights/weights.rat.MAX/ \ No newline at end of file +dannce_finetune_weights: ./DANNCE/weights/weights.rat.MAX/ + +log_level: DEBUG +log_dest: ../logs/dannce-04-15-22.log \ No newline at end of file diff --git a/dannce/cli.py b/dannce/cli.py index 357a13b..2b55925 100755 --- a/dannce/cli.py +++ b/dannce/cli.py @@ -264,7 +264,7 @@ def add_shared_args( parser.add_argument( "--log-dest", dest="log_dest", - help="Log File location to where logs are to be written to. By default, location is set to logs/dannce.log ", + help="Log File location to where logs are to be written to. By default, location is set to ../logs/dannce.log ", ) return parser diff --git a/dannce/engine/generator.py b/dannce/engine/generator.py index 46833dc..f5cfb75 100755 --- a/dannce/engine/generator.py +++ b/dannce/engine/generator.py @@ -914,7 +914,7 @@ def __init__( self.interp = interp self.depth = depth self.channel_combo = channel_combo - logging.info(FILE_PATH + ".DataGenerator_3Dconv_torch.__init__" + self.channel_combo) + logging.info(FILE_PATH + ".DataGenerator_3Dconv_torch.__init__" + self.channel_combo if channel_combo is not None else "None") self.gpu_id = gpu_id self.mode = mode self.immode = immode @@ -943,7 +943,7 @@ def __init__( config.gpu_options.per_process_gpu_memory_fraction = TF_GPU_MEMORY_FRACTION config.gpu_options.allow_growth = True self.session = tf.compat.v1.Session(config=config, graph=tf.Graph()) - logging.info(FILE_PATH + ".DataGenerator_3Dconv_torch.__init__" + "Executing eagerly: ", tf.executing_eagerly(), flush=True) + logging.info(FILE_PATH + ".DataGenerator_3Dconv_torch.__init__" + "Executing eagerly: " + str(tf.executing_eagerly()))#, flush=True) for i, ID in enumerate(list_IDs): experimentID = int(ID.split("_")[0]) for camname in self.camnames[experimentID]: @@ -1545,7 +1545,7 @@ def __init__( self.interp = interp self.depth = depth self.channel_combo = channel_combo - logging.info(FILE_PATH + ".DataGenerator_3Dconv_tf.__init__ " + self.channel_combo) + logging.info(FILE_PATH + ".DataGenerator_3Dconv_tf.__init__ " + self.channel_combo if channel_combo is not None else "None") self.gpu_id = gpu_id self.mode = mode self.immode = immode diff --git a/dannce/engine/inference.py b/dannce/engine/inference.py index 5aaa971..ccd57a0 100755 --- a/dannce/engine/inference.py +++ b/dannce/engine/inference.py @@ -1,6 +1,6 @@ """Handle inference procedures for dannce and com networks. """ -from Dannce.repos.dannce.dannce.engine.generator import FILE_PATH + import numpy as np import os import time @@ -36,7 +36,7 @@ def print_checkpoint( float: New timing reference. """ prepend_log_msg = FILE_PATH + ".print_checkpoint " - logging.info(prepend_log_msg + "Predicting on sample %d" % (n_frame), flush=True) + logging.info(prepend_log_msg + "Predicting on sample %d" % (n_frame))# flush=True) if (n_frame - start_ind) % sample_save == 0 and n_frame != start_ind: logging.info(prepend_log_msg + n_frame) logging.info(prepend_log_msg + "{} samples took {} seconds".format(sample_save, time.time() - end_time)) @@ -670,7 +670,7 @@ def infer_dannce( start_ind = params["start_batch"] end_ind = params["maxbatch"] for idx, i in enumerate(range(start_ind, end_ind)): - logging.debug("Predicting on batch {}".format(i), flush=True) + logging.debug("Predicting on batch {}".format(i))#, flush=True) if (i - start_ind) % 10 == 0 and i != start_ind: logging.debug(i) logging.debug("10 batches took {} seconds".format(time.time() - end_time)) diff --git a/dannce/engine/nets.py b/dannce/engine/nets.py index ef30cce..7b29533 100644 --- a/dannce/engine/nets.py +++ b/dannce/engine/nets.py @@ -21,12 +21,17 @@ FILE_PATH = "dannce.engine.nets" def setup_logging(logfile_path, log_lvl, params=None): + import os if logfile_path != None and log_lvl != None: + if not os.path.exists(os.path.dirname(params["log_dest"])): + os.makedirs(os.path.dirname(params["log_dest"])) logging.basicConfig(filename= logfile_path, level= log_lvl, format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') elif params != None: + if not os.path.exists(os.path.dirname(params["log_dest"])): + os.makedirs(os.path.dirname(params["log_dest"])) logging.basicConfig(filename=params["log_dest"], level=params["log_level"], format='%(asctime)s %(levelname)s:%(message)s', diff --git a/dannce/engine/processing.py b/dannce/engine/processing.py index aa3ac2b..78b10cf 100755 --- a/dannce/engine/processing.py +++ b/dannce/engine/processing.py @@ -70,6 +70,7 @@ def plot_out(imo, lo, imn): # Plot all training images and save # create new directory for images if necessary debugdir = os.path.join(params["com_train_dir"], outdir) + import pdb; pdb.set_trace() logging.info(prepend_log_msg + "Saving debug images to: " + debugdir) if not os.path.exists(debugdir): os.makedirs(debugdir) @@ -151,6 +152,12 @@ def infer_params(params, dannce_net, prediction): Some parameters that were previously specified in configs can just be inferred from others, thus relieving config bloat """ + + curr_dir = os.path.dirname(__file__) + os.environ["DANNCE_HOME"] = os.path.dirname(curr_dir) + + if not os.path.exists(os.path.dirname(params["log_dest"])): + os.makedirs(os.path.dirname(params["log_dest"])) # Setting up logging logging.basicConfig(filename=params["log_dest"], level=params["log_level"], format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') @@ -760,6 +767,8 @@ def save_COM_dannce_mat(params, com3d, sampleID): streamlines subsequent dannce access. """ # Setup Logging + if not os.path.exists(os.path.dirname(params["log_dest"])): + os.makedirs(os.path.dirname(params["log_dest"])) logging.basicConfig(filename=params["log_dest"], level=params["log_level"], format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') prepend_log_msg = FILE_PATH + ".save_COM_dannce_mat " diff --git a/dannce/engine/video.py b/dannce/engine/video.py index a52bce8..2650a18 100755 --- a/dannce/engine/video.py +++ b/dannce/engine/video.py @@ -1,7 +1,6 @@ """ Video reading and writing interfaces for different formats. """ from copy import Error import os -from Dannce.Experimental.exp1file1 import FILE_PATH import cv2 import numpy as np import attr diff --git a/dannce/interface.py b/dannce/interface.py index dd88d95..fb6e1b4 100755 --- a/dannce/interface.py +++ b/dannce/interface.py @@ -111,6 +111,8 @@ def com_predict(params: Dict): params (Dict): Parameters dictionary. """ # Enable Logging for com_predict + if not os.path.exists(os.path.dirname(params["log_dest"])): + os.makedirs(os.path.dirname(params["log_dest"])) logging.basicConfig(filename=params["log_dest"], level=params["log_level"], format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') prepend_log_msg = file_path + ".com_predict " @@ -337,6 +339,8 @@ def com_train(params: Dict): """ # Setup Logging for com_train + if not os.path.exists(os.path.dirname(params["log_dest"])): + os.makedirs(os.path.dirname(params["log_dest"])) logging.basicConfig(filename=params["log_dest"], level=params["log_level"], format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') prepend_log_msg = file_path + ".com_train " @@ -662,6 +666,8 @@ def dannce_train(params: Dict): """ # Setup Logging for com_train + if not os.path.exists(os.path.dirname(params["log_dest"])): + os.makedirs(os.path.dirname(params["log_dest"])) logging.basicConfig(filename=params["log_dest"], level=params["log_level"], format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') prepend_log_msg = file_path + ".dannce_train " @@ -1299,7 +1305,11 @@ def dannce_predict(params: Dict): Args: params (Dict): Paremeters dictionary. """ + import pdb + pdb.set_trace() # Setup Logging for com_train + if not os.path.exists(os.path.dirname(params["log_dest"])): + os.makedirs(os.path.dirname(params["log_dest"])) logging.basicConfig(filename=params["log_dest"], level=params["log_level"], format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') prepend_log_msg = file_path + ".dannce_predict " diff --git a/setup.py b/setup.py index 6a779ef..b902979 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,17 @@ """Setup file for dannce.""" from setuptools import setup, find_packages +def get_name(): + import os + curr_dir = os.path.dirname(__file__) + print ("Current_Directory = ", curr_dir) + os.environ['DANNCE_HOME'] = curr_dir + print (os.environ['DANNCE_HOME']) + + return "dannce" + setup( - name="dannce", + name=get_name(), version="1.2.0", packages=find_packages(), install_requires=[ @@ -17,8 +26,8 @@ "attrs", "multiprocess", "opencv-python", - "tensorflow==2.6.0", - "keras==2.6.*", # Required to resolve pip keras install bug for tf 2.6 + "tensorflow==2.4.0", + # "keras==2.6.*", # Required to resolve pip keras install bug for tf 2.6 "psutil", ], # scripts=[], From 64d66a93399b6125a495f30a783849219d598929 Mon Sep 17 00:00:00 2001 From: Anshuman Sabath Date: Fri, 15 Apr 2022 17:08:19 -0400 Subject: [PATCH 6/9] Add checks for log_dir location. Create if does not exist. --- cluster/duke.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/duke.yaml b/cluster/duke.yaml index 049f05f..a0e9fcb 100644 --- a/cluster/duke.yaml +++ b/cluster/duke.yaml @@ -10,4 +10,4 @@ com_predict: "--job-name=predictCom -p gpu-common,dsplus-gpu --mem=10000 -t 3-00 com_multi_predict: "--job-name=predictCom -p gpu-common,dsplus-gpu --mem=10000 -t 0-03:00 --gres=gpu:1 -N 1 -n 8 --account=plusds" # Setup functions (optional, set to "" if no setup is required. Trailing ; is required) -setup: ". ~/.bashrc; conda activate dannce_cuda1;" \ No newline at end of file +setup: ". ~/.bashrc; conda activate dannce_tf26;" \ No newline at end of file From 88ed45725f9e773eaa928b2f4fededb597a6f9ba Mon Sep 17 00:00:00 2001 From: data-hound Date: Tue, 19 Apr 2022 13:12:47 -0400 Subject: [PATCH 7/9] Added logging messages to cli.py. Included datetime in the default log file destination. --- configs/dannce_mouse_config.yaml | 2 +- dannce/__init__.py | 3 ++- dannce/cli.py | 9 +++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/configs/dannce_mouse_config.yaml b/configs/dannce_mouse_config.yaml index 4f36d85..797f4f3 100755 --- a/configs/dannce_mouse_config.yaml +++ b/configs/dannce_mouse_config.yaml @@ -36,4 +36,4 @@ max_num_samples: 1000 dannce_finetune_weights: ./DANNCE/weights/weights.rat.MAX/ log_level: DEBUG -log_dest: ../logs/dannce-04-15-22.log \ No newline at end of file +# log_dest: ../logs/dannce-04-15-22.log \ No newline at end of file diff --git a/dannce/__init__.py b/dannce/__init__.py index c50f221..5abd6d1 100755 --- a/dannce/__init__.py +++ b/dannce/__init__.py @@ -1,3 +1,4 @@ +from datetime import datetime """Dannce module and default parameters""" # Default parameters, which can be superseded by CL arguments or # config files @@ -47,7 +48,7 @@ "norm_method":"layer", "slurm_config": None, "log_level": "INFO", - "log_dest": "../logs/dannce.log", + "log_dest": "../../logs/dannce_"+datetime.now().strftime("%b%d_%Y")+ ".log", } _param_defaults_dannce = { "metric": ["euclidean_distance_3D"], diff --git a/dannce/cli.py b/dannce/cli.py index 2b55925..17d9096 100755 --- a/dannce/cli.py +++ b/dannce/cli.py @@ -18,6 +18,7 @@ import argparse import yaml from typing import Dict, Text +import logging def load_params(param_path: Text) -> Dict: @@ -856,7 +857,11 @@ def combine(base_params: Dict, clargs: argparse.Namespace, dannce_net: bool) -> base_params[k] = v elif v is not None: base_params[k] = v - + + if not os.path.exists(os.path.dirname(base_params["log_dest"])): + os.makedirs(os.path.dirname(base_params["log_dest"])) + logging.basicConfig(filename=base_params["log_dest"], level=base_params["log_level"], + format='%(asctime)s %(levelname)s:%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') for k, v in base_params.items(): - print("{} set to: {}".format(k, v)) + logging.info("{} set to: {}".format(k, v)) return base_params From 903cbe4b834f879c02fbc83b59eb02915232d8d6 Mon Sep 17 00:00:00 2001 From: Anshuman Sabath Date: Thu, 12 May 2022 17:36:56 -0400 Subject: [PATCH 8/9] Remove debug points --- dannce/engine/processing.py | 2 +- setup.py | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/dannce/engine/processing.py b/dannce/engine/processing.py index 78b10cf..1e25da8 100755 --- a/dannce/engine/processing.py +++ b/dannce/engine/processing.py @@ -70,7 +70,7 @@ def plot_out(imo, lo, imn): # Plot all training images and save # create new directory for images if necessary debugdir = os.path.join(params["com_train_dir"], outdir) - import pdb; pdb.set_trace() + logging.info(prepend_log_msg + "Saving debug images to: " + debugdir) if not os.path.exists(debugdir): os.makedirs(debugdir) diff --git a/setup.py b/setup.py index b902979..5ce8efb 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,8 @@ """Setup file for dannce.""" from setuptools import setup, find_packages -def get_name(): - import os - curr_dir = os.path.dirname(__file__) - print ("Current_Directory = ", curr_dir) - os.environ['DANNCE_HOME'] = curr_dir - print (os.environ['DANNCE_HOME']) - - return "dannce" - setup( - name=get_name(), + name="dannce", version="1.2.0", packages=find_packages(), install_requires=[ From 0dcdf7180024f4a73863fcf30c9b200604d2bbcd Mon Sep 17 00:00:00 2001 From: Anshuman Sabath Date: Thu, 12 May 2022 17:42:45 -0400 Subject: [PATCH 9/9] Changing to tf2.6 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5ce8efb..6a779ef 100644 --- a/setup.py +++ b/setup.py @@ -17,8 +17,8 @@ "attrs", "multiprocess", "opencv-python", - "tensorflow==2.4.0", - # "keras==2.6.*", # Required to resolve pip keras install bug for tf 2.6 + "tensorflow==2.6.0", + "keras==2.6.*", # Required to resolve pip keras install bug for tf 2.6 "psutil", ], # scripts=[],