Skip to content

Commit

Permalink
outlier_correction_mp
Browse files Browse the repository at this point in the history
  • Loading branch information
@sronilsson
sronilsson committed Nov 8, 2024
1 parent 963c392 commit 5c1e814
Show file tree
Hide file tree
Showing 11 changed files with 557 additions and 88 deletions.
Binary file added docs/_static/img/spatial_density.webp
View file
Binary file not shown.
4 changes: 2 additions & 2 deletions docs/nb/CLI Example 1.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,9 +438,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python_3.6",
"display_name": "simba",
"language": "python",
"name": "python_3.6"
"name": "simba"
},
"language_info": {
"codemirror_mode": {
Expand Down
184 changes: 140 additions & 44 deletions docs/nb/outlier_correction.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,20 @@
},
{
"cell_type": "code",
"execution_count": 23,
"execution_count": 1,
"id": "cb913766",
"metadata": {},
"outputs": [],
"source": [
"from simba.outlier_tools.outlier_corrector_movement import OutlierCorrecterMovement\n",
"from simba.outlier_tools.outlier_corrector_location import OutlierCorrecterLocation\n",
"from simba.utils.cli import set_outlier_correction_criteria_cli\n",
"from simba.pose_importers.dlc_importer_csv import import_multiple_dlc_tracking_csv_file"
"from simba.utils.cli.cli_tools import set_outlier_correction_criteria_cli\n",
"from simba.pose_importers.dlc_importer_csv import import_dlc_csv_data"
]
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 2,
"id": "f7cc63fa",
"metadata": {},
"outputs": [],
Expand All @@ -41,16 +41,15 @@
"# DATA, AND (III) THE ATTRIBUTES OF OUR NEW VIDEOS (FPS ETC.)\n",
"\n",
"## Define the path to our SimBA project config ini\n",
"CONFIG_PATH = '/Users/simon/Desktop/envs/troubleshooting/notebook_example/project_folder/project_config.ini'\n",
"CONFIG_PATH = r\"C:\\troubleshooting\\two_black_animals_14bp\\project_folder\\project_config.ini\"\n",
"\n",
"## Define the path to the directory holding our new DLC CSV pose-estimation data\n",
"DATA_DIR = '/Users/simon/Desktop/envs/troubleshooting/notebook_example/data'\n",
"DATA_DIR = r\"C:\\troubleshooting\\two_black_animals_14bp\\dlc_data\"\n",
"\n",
"## Define if / how you want to interpolate missing pose-estimation data,\n",
"## and if/how you want to smooth the new pose estimation data: here we do neither.\n",
"INTERPOLATION_SETTING = 'None' # OPTIONS: 'None', Animal(s): Nearest', 'Animal(s): Linear', 'Animal(s): Quadratic','Body-parts: Nearest', 'Body-parts: Linear', 'Body-parts: Quadratic'\n",
"SMOOTHING_SETTING = None # OPTIONS: 'Gaussian', 'Savitzky Golay'\n",
"SMOOTHING_TIME = None # TIME IN MILLISECOND\n",
"INTERPOLATION_SETTING = None # OPTIONS: 'None', Animal(s): Nearest', 'Animal(s): Linear', 'Animal(s): Quadratic','Body-parts: Nearest', 'Body-parts: Linear', 'Body-parts: Quadratic'\n",
"SMOOTHING_SETTING = None # OPTIONS: {'time_window': 500, 'method': 'savitzky-golay'}, {'time_window': 500, 'method': 'gaussian'}\n",
"\n",
"## Define the fps and the pixels per millimeter of the incoming data: has to be the same for all new videos.\n",
"## if you have varying fps / px per millimeter / resolutions, then use gui (2023/05)\n",
Expand All @@ -71,89 +70,186 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 3,
"id": "c3d8767e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Importing Aqu_FFJ_Cre_721 to SimBA project...\n",
"Pose-estimation data for video Aqu_FFJ_Cre_721 imported to SimBA project (elapsed time: 0.1718s)...\n",
"Importing Aqu_FFJ_Cre_723 to SimBA project...\n",
"Pose-estimation data for video Aqu_FFJ_Cre_723 imported to SimBA project (elapsed time: 0.1681s)...\n",
"Importing Aqu_FFJ_Cre_722 to SimBA project...\n",
"Pose-estimation data for video Aqu_FFJ_Cre_722 imported to SimBA project (elapsed time: 0.1617s)...\n",
"SIMBA COMPLETE: Imported 3 pose estimation file(s) (elapsed time: 0.5078s) \tcomplete\n"
"Importing Test_1 to SimBA project...\n",
"Pose-estimation data for video Test_1 imported to SimBA project (elapsed time: 0.0546s)...\n",
"Importing Test_2 to SimBA project...\n",
"Pose-estimation data for video Test_2 imported to SimBA project (elapsed time: 0.052s)...\n",
"Importing Test_3 to SimBA project...\n",
"Pose-estimation data for video Test_3 imported to SimBA project (elapsed time: 0.0463s)...\n",
"Importing Test_4 to SimBA project...\n",
"Pose-estimation data for video Test_4 imported to SimBA project (elapsed time: 0.0431s)...\n",
"Importing Test_5 to SimBA project...\n",
"Pose-estimation data for video Test_5 imported to SimBA project (elapsed time: 0.0482s)...\n",
"Importing Test_6 to SimBA project...\n",
"Pose-estimation data for video Test_6 imported to SimBA project (elapsed time: 0.0583s)...\n",
"Importing Test_7 to SimBA project...\n",
"Pose-estimation data for video Test_7 imported to SimBA project (elapsed time: 0.0503s)...\n",
"Importing Test_8 to SimBA project...\n",
"Pose-estimation data for video Test_8 imported to SimBA project (elapsed time: 0.05s)...\n",
"Importing Test_9 to SimBA project...\n",
"Pose-estimation data for video Test_9 imported to SimBA project (elapsed time: 0.058s)...\n",
"SIMBA COMPLETE: Imported 9 pose estimation file(s) to directory (elapsed time: 0.4758s) \tcomplete\n"
]
}
],
"source": [
"# WE RUN THE DATA IMPORTER FOR OUR DIRECTORY OF FILES\n",
"## This imports your DLC files in the ``DATA_DIR`` according to the smoothing / interpolation settings defined above\n",
"import_multiple_dlc_tracking_csv_file(config_path=CONFIG_PATH,\n",
" interpolation_setting=INTERPOLATION_SETTING,\n",
" smoothing_setting=SMOOTHING_SETTING,\n",
" smoothing_time=SMOOTHING_TIME,\n",
" data_dir=DATA_DIR)"
"\n",
"import_dlc_csv_data(config_path=CONFIG_PATH,\n",
" interpolation_settings=INTERPOLATION_SETTING,\n",
" smoothing_settings=SMOOTHING_SETTING,\n",
" data_path=DATA_DIR)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": 5,
"id": "38306f7e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing video Aqu_FFJ_Cre_721. Video 1/3...\n",
"Corrected movement outliers for file Aqu_FFJ_Cre_721 (elapsed time: 0.2929s)...\n",
"Processing video Aqu_FFJ_Cre_723. Video 2/3...\n",
"Corrected movement outliers for file Aqu_FFJ_Cre_723 (elapsed time: 0.2713s)...\n",
"Processing video Aqu_FFJ_Cre_722. Video 3/3...\n",
"Corrected movement outliers for file Aqu_FFJ_Cre_722 (elapsed time: 0.2674s)...\n",
"SIMBA COMPLETE: Log for corrected \"movement outliers\" saved in project_folder/logs (elapsed time: 0.8572s) \tcomplete\n",
"Processing video Aqu_FFJ_Cre_721. Video 1/3..\n",
"Corrected location outliers for file Aqu_FFJ_Cre_721 (elapsed time: 49.6797s)...\n",
"Processing video Aqu_FFJ_Cre_723. Video 2/3..\n",
"Corrected location outliers for file Aqu_FFJ_Cre_723 (elapsed time: 24.645s)...\n",
"Processing video Aqu_FFJ_Cre_722. Video 3/3..\n",
"Corrected location outliers for file Aqu_FFJ_Cre_722 (elapsed time: 15.1142s)...\n",
"SIMBA COMPLETE: Log for corrected \"location outliers\" saved in project_folder/logs (elapsed time: 89.4743s) \tcomplete\n"
"SIMBA COMPLETE: Outlier parameters set (elapsed time: 0.003s) \tcomplete\n"
]
}
],
"source": [
"#We set the outlier criteria in the project_config.ini and run the outlier correction. NOTE: You can also set this manually in the project_config.ini or thrugh\n",
"#We set the outlier criteria in the project_config.ini NOTE: You can also set this manually in the project_config.ini or thrugh\n",
"# the SimBA GUI. If this has already been done, there is **no need** to call `set_outlier_correction_criteria_cli`.\n",
"set_outlier_correction_criteria_cli(config_path=CONFIG_PATH,\n",
" aggregation=AGGREGATION_METHOD,\n",
" body_parts=BODY_PARTS,\n",
" movement_criterion=MOVEMENT_CRITERION,\n",
" location_criterion=LOCATION_CRITERION)\n",
"\n",
"\n",
" location_criterion=LOCATION_CRITERION)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "ff58e186",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing video Test_1. Video 1/9...\n",
"Corrected movement outliers for file Test_1 (elapsed time: 0.435s)...\n",
"Processing video Test_2. Video 2/9...\n",
"Corrected movement outliers for file Test_2 (elapsed time: 0.1166s)...\n",
"Processing video Test_3. Video 3/9...\n",
"Corrected movement outliers for file Test_3 (elapsed time: 0.1158s)...\n",
"Processing video Test_4. Video 4/9...\n",
"Corrected movement outliers for file Test_4 (elapsed time: 0.114s)...\n",
"Processing video Test_5. Video 5/9...\n",
"Corrected movement outliers for file Test_5 (elapsed time: 0.1183s)...\n",
"Processing video Test_6. Video 6/9...\n",
"Corrected movement outliers for file Test_6 (elapsed time: 0.1124s)...\n",
"Processing video Test_7. Video 7/9...\n",
"Corrected movement outliers for file Test_7 (elapsed time: 0.1151s)...\n",
"Processing video Test_8. Video 8/9...\n",
"Corrected movement outliers for file Test_8 (elapsed time: 0.117s)...\n",
"Processing video Test_9. Video 9/9...\n",
"Corrected movement outliers for file Test_9 (elapsed time: 0.1152s)...\n",
"SIMBA COMPLETE: Log for corrected \"movement outliers\" saved in C:\\troubleshooting\\two_black_animals_14bp\\project_folder\\logs (elapsed time: 1.3693s) \tcomplete\n",
"Processing video Test_1. Video 1/9..\n",
"Corrected location outliers for file Test_1 (elapsed time: 0.8654s)...\n",
"Processing video Test_2. Video 2/9..\n",
"Corrected location outliers for file Test_2 (elapsed time: 0.877s)...\n",
"Processing video Test_3. Video 3/9..\n",
"Corrected location outliers for file Test_3 (elapsed time: 0.8534s)...\n",
"Processing video Test_4. Video 4/9..\n",
"Corrected location outliers for file Test_4 (elapsed time: 0.8611s)...\n",
"Processing video Test_5. Video 5/9..\n",
"Corrected location outliers for file Test_5 (elapsed time: 0.8581s)...\n",
"Processing video Test_6. Video 6/9..\n",
"Corrected location outliers for file Test_6 (elapsed time: 0.8512s)...\n",
"Processing video Test_7. Video 7/9..\n",
"Corrected location outliers for file Test_7 (elapsed time: 0.8616s)...\n",
"Processing video Test_8. Video 8/9..\n",
"Corrected location outliers for file Test_8 (elapsed time: 0.8641s)...\n",
"Processing video Test_9. Video 9/9..\n",
"Corrected location outliers for file Test_9 (elapsed time: 0.8626s)...\n",
"SIMBA COMPLETE: Log for corrected \"location outliers\" saved in project_folder/logs (elapsed time: 7.8084s) \tcomplete\n"
]
}
],
"source": [
"# Finally, we run the outlier correction (NOTE: SEE CELL BELOW FOR ALTERNATIVE WAY OF RUNNING OUTLIER CORRECTION ACROSS MULTIPLE CORES)\n",
"_ = OutlierCorrecterMovement(config_path=CONFIG_PATH).run()\n",
"_ = OutlierCorrecterLocation(config_path=CONFIG_PATH).run()"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"id": "47975a2a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Video Test_1 complete...\n",
"Video Test_2 complete...\n",
"Video Test_3 complete...\n",
"Video Test_4 complete...\n",
"Video Test_5 complete...\n",
"Video Test_6 complete...\n",
"Video Test_7 complete...\n",
"Video Test_8 complete...\n",
"Video Test_9 complete...\n",
"SIMBA COMPLETE: Log for corrected \"movement outliers\" saved in C:\\troubleshooting\\two_black_animals_14bp\\project_folder\\logs (elapsed time: 4.2945s) \tcomplete\n",
"Video Test_1 complete...\n",
"Video Test_2 complete...\n",
"Video Test_3 complete...\n",
"Video Test_4 complete...\n",
"Video Test_5 complete...\n",
"Video Test_6 complete...\n",
"Video Test_7 complete...\n",
"Video Test_8 complete...\n",
"Video Test_9 complete...\n",
"SIMBA COMPLETE: Log for corrected \"location outliers\" saved in project_folder/logs (elapsed time: 4.0219s) \tcomplete\n"
]
}
],
"source": [
"# OPTIONAL: If you find that the outlier correction - as run in the immediate above cell - is slow, we could run outlier \n",
"# correction over multiple cores. If you choose this approach, make sure you are running the latest version of SimBA.\n",
"# You can update SimBA by running `pip install simba-uw-tf-dev --upgrade`\n",
"\n",
"from simba.outlier_tools.outlier_corrector_location_mp import OutlierCorrecterLocationMultiprocess\n",
"from simba.outlier_tools.outlier_corrector_movement_mp import OutlierCorrecterMovementMultiProcess\n",
"\n",
"_ = OutlierCorrecterMovementMultiProcess(config_path=CONFIG_PATH).run()\n",
"_ = OutlierCorrecterLocationMultiprocess(config_path=CONFIG_PATH).run()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "61fb78fa",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:simba_dev] *",
"display_name": "simba",
"language": "python",
"name": "conda-env-simba_dev-py"
"name": "simba"
},
"language_info": {
"codemirror_mode": {
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
# Setup configuration
setuptools.setup(
name="Simba-UW-tf-dev",
version="2.2.8",
version="2.3.1",
author="Simon Nilsson, Jia Jie Choong, Sophia Hwang",
author_email="[email protected]",
description="Toolkit for computer classification and analysis of behaviors in experimental animals",
Expand Down
8 changes: 2 additions & 6 deletions simba/mixins/config_reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,15 +152,11 @@ def __init__(
self.outlier_corrected_movement_dir + f"/*.{self.file_type}"
)
self.cpu_cnt, self.cpu_to_use = find_core_cnt()
self.machine_results_paths = glob.glob(
self.machine_results_dir + f"/*.{self.file_type}"
)
self.machine_results_paths = glob.glob(self.machine_results_dir + f"/*.{self.file_type}")
self.logs_path = os.path.join(self.project_path, "logs")
self.body_parts_path = os.path.join(self.project_path, Paths.BP_NAMES.value)
check_file_exist_and_readable(file_path=self.body_parts_path)
self.body_parts_lst = (
pd.read_csv(self.body_parts_path, header=None).iloc[:, 0].to_list()
)
self.body_parts_lst = (pd.read_csv(self.body_parts_path, header=None).iloc[:, 0].to_list())
self.body_parts_lst = [x for x in self.body_parts_lst if str(x) != "nan"]
self.get_body_part_names()
self.get_bp_headers()
Expand Down
33 changes: 33 additions & 0 deletions simba/mixins/statistics_mixin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4182,3 +4182,36 @@ def normalized_google_distance(x: np.ndarray, y: np.ndarray) -> float:
return -1.0
else:
return N / D

def symmetry_index(x: np.ndarray, y: np.ndarray, agg_type: Literal['mean', 'median'] = 'mean') -> float:

"""
Calculate the Symmetry Index (SI) between two arrays of measurements, `x` and `y`, over a given time series.
The Symmetry Index quantifies the relative difference between two measurements at each time point, expressed as a percentage.
The function returns either the mean or median Symmetry Index over the entire series, based on the specified aggregation type.
Zero indicates perfect symmetry. Positive values pepresent increasing asymmetry between the two measurements.
:param np.ndarray x: A 1-dimensional array of measurements from one side (e.g., left side), representing a time series or sequence of measurements.
:param np.ndarray y: A 1-dimensional array of measurements from the other side (e.g., right side), of the same length as `x`.
:param Literal['mean', 'median'] agg_type: The aggregation method used to summarize the Symmetry Index across all time points.
:return: The aggregated Symmetry Index over the series, either as the mean or median SI.
:rtype: float
:example:
>>> x = np.random.randint(0, 155, (100,))
>>>y = np.random.randint(0, 155, (100,))
>>> symmetry_index(x=x, y=y)
"""

check_valid_array(data=x, source=f'{Statistics.symmetry_index.__name__} x', accepted_ndims=(1,), min_axis_0=1,
accepted_dtypes=Formats.NUMERIC_DTYPES.value)
check_valid_array(data=x, source=f'{Statistics.symmetry_index.__name__} y', accepted_ndims=(1,), min_axis_0=1,
accepted_axis_0_shape=[x.shape[0]], accepted_dtypes=Formats.NUMERIC_DTYPES.value)
check_str(name=f'{Statistics.symmetry_index.__name__} agg_type', value=agg_type, options=('mean', 'median'))
si_values = np.abs(x - y) / (0.5 * (x + y)) * 100
if agg_type == 'mean':
return np.float32(np.nanmean(si_values))
else:
return np.float32(np.nanmedian(si_values))

17 changes: 3 additions & 14 deletions simba/outlier_tools/outlier_corrector_location.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ class OutlierCorrecterLocation(ConfigReader):
:param Union[str, os.PathLike] config_path: path to SimBA project config file in Configparser format
Examples
----------
:example:
>>> _ = OutlierCorrecterLocation(config_path='MyProjectConfig').run()
"""

Expand All @@ -54,18 +53,8 @@ def __init__(self,
self.outlier_bp_dict = {}
for animal_name in self.animal_bp_dict.keys():
self.outlier_bp_dict[animal_name] = {}
self.outlier_bp_dict[animal_name]["bp_1"] = read_config_entry(
self.config,
ConfigKey.OUTLIER_SETTINGS.value,
"location_bodypart1_{}".format(animal_name.lower()),
"str",
)
self.outlier_bp_dict[animal_name]["bp_2"] = read_config_entry(
self.config,
ConfigKey.OUTLIER_SETTINGS.value,
"location_bodypart2_{}".format(animal_name.lower()),
"str",
)
self.outlier_bp_dict[animal_name]["bp_1"] = read_config_entry(self.config, ConfigKey.OUTLIER_SETTINGS.value, "location_bodypart1_{}".format(animal_name.lower()),"str")
self.outlier_bp_dict[animal_name]["bp_2"] = read_config_entry(self.config, ConfigKey.OUTLIER_SETTINGS.value, "location_bodypart2_{}".format(animal_name.lower()), "str")

def __find_location_outliers(self):
for animal_name, animal_data in self.bp_dict.items():
Expand Down
Loading

0 comments on commit 5c1e814

Please sign in to comment.