From 9611b8e0747c42e495d553340e67bf566ad84ce9 Mon Sep 17 00:00:00 2001 From: Ziv Yaniv Date: Fri, 20 Jan 2023 14:38:21 -0500 Subject: [PATCH 1/6] Adding script to convert fluorescent_probes.csv to markdown. Convert table to markdown and sort on excitation and emission. --- pyproject.toml | 3 +- .../fluorescent_probes_csv_2_md.py | 95 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 src/ibex_imaging_knowledge_base_utilities/fluorescent_probes_csv_2_md.py diff --git a/pyproject.toml b/pyproject.toml index cdf0b69..dda30b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "ibex_imaging_knowledge_base_utilities" -version = "0.2.0" +version = "0.3.0" authors = [{ name="Ziv Yaniv", email="zivyaniv@nih.gov" }, ] description = "Utility scripts used for maintaining the IBEX Imaging Community Knowledge-Base" @@ -30,5 +30,6 @@ dependencies = [ [project.scripts] bib2md = "ibex_imaging_knowledge_base_utilities.bib2md:main" reagent_resources_csv_2_md_url = "ibex_imaging_knowledge_base_utilities.reagent_resources_csv_2_md_url:main" +fluorescent_probes_csv_2_md = "ibex_imaging_knowledge_base_utilities.fluorescent_probes_csv_2_md:main" update_index_md_stats = "ibex_imaging_knowledge_base_utilities.update_index_md_stats:main" validate_zenodo_json = "ibex_imaging_knowledge_base_utilities.validate_zenodo_json:main" diff --git a/src/ibex_imaging_knowledge_base_utilities/fluorescent_probes_csv_2_md.py b/src/ibex_imaging_knowledge_base_utilities/fluorescent_probes_csv_2_md.py new file mode 100644 index 0000000..ff35d90 --- /dev/null +++ b/src/ibex_imaging_knowledge_base_utilities/fluorescent_probes_csv_2_md.py @@ -0,0 +1,95 @@ +# ========================================================================= +# +# Copyright Ziv Yaniv +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ========================================================================= + +import pandas as pd +import argparse +import sys +import pathlib +from .argparse_types import file_path +import json + +""" +This script converts the IBEX knowledge-base fluorescent_probes.csv file to markdown. + +This script is automatically run when modifications to the fluorescent_probes.csv file is merged +into the main branch of the ibex_knowledge_base repository (see .github/workflows/data2md.yml). + +Assumption: The fluorescent_probes.csv file is valid. It conforms to the expected format (empty entries denoted +by the string "NA"). +""" + +def fluorescent_probe_csv_to_md( + template_file_path, + csv_file_path, + output_dir +): + """ + Convert the IBEX knowledge-base fluorescent probe csv file to markdown. Output is written to a + file named fluorescent_probes.md in the output directory. The template_file_path file is expected + to contain the string + {probe_table} which is replaced with the contents of the actual table. + """ + # Read the dataframe and keep entries that are "NA", don't convert to nan + df = pd.read_csv(csv_file_path, dtype=str, keep_default_na=False) + df.sort_values(by=["Excitation Max (nm)", "Emission Max (nm)"], inplace=True) + with open(template_file_path, "r") as fp: + input_md_str = fp.read() + with open(output_dir / "fluorescent_probes.md", "w") as fp: + fp.write( + input_md_str.format(probe_table=df.to_markdown(index=False)) + ) + + +def main(argv=None): + if argv is None: # script was invoked from commandline + argv = sys.argv[1:] + parser = argparse.ArgumentParser( + description="Convert knowledge-base fluorescent probes file from csv to md and sort according to excitation and emission." + ) + parser.add_argument( + "md_template_file", + type=file_path, + help='Path to template markdown file which contains the string "{probe_table}".', + ) + parser.add_argument( + "csv_file", type=file_path, help="Path to the fluorescent_probes.csv file." + ) + parser.add_argument( + "output_dir", + type=dir_path, + help="Path to the output directory (the fluorescent_probes.md file is written to this directory).", + ) + args = parser.parse_args(argv) + + try: + return fluorescent_probe_csv_to_md( + template_file_path=args.md_template_file, + csv_file_path=args.csv_file, + output_dir=args.output_dir + ) + except Exception as e: + print( + f"{e}", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From bb897f88d06e5d9b7b5c92bcec7501d12d9b6150 Mon Sep 17 00:00:00 2001 From: Ziv Yaniv Date: Fri, 20 Jan 2023 14:40:36 -0500 Subject: [PATCH 2/6] Modify reagent_resources_csv_2_md_url script to use markdown input. Use template markdown file and also sort table according to Target Name / Protein Biomarker column. --- .../reagent_resources_csv_2_md_url.py | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src/ibex_imaging_knowledge_base_utilities/reagent_resources_csv_2_md_url.py b/src/ibex_imaging_knowledge_base_utilities/reagent_resources_csv_2_md_url.py index 625eea2..ce18e73 100644 --- a/src/ibex_imaging_knowledge_base_utilities/reagent_resources_csv_2_md_url.py +++ b/src/ibex_imaging_knowledge_base_utilities/reagent_resources_csv_2_md_url.py @@ -55,8 +55,6 @@ by the string "NA"). """ -md_header = "\n\n" - def short_circuit_requests_get(url, params=None, **kwargs): res = requests.Response() @@ -161,15 +159,20 @@ def uniprots_to_md(uniprots_str, uniprot_md_str): def csv_to_md_with_url( - csv_file_path, supporting_material_root_dir, vendor_to_website_json_file_path + template_file_path, + csv_file_path, + supporting_material_root_dir, + vendor_to_website_json_file_path, ): """ - Convert the IBEX knowledge-base csv file to markdown and add links to the supporting + Convert the IBEX knowledge-base reagent resources csv file to markdown and add links to the supporting material files. Output is written to a file named markdown.md in the parent directory - of the supporting_material_root_dir. + of the supporting_material_root_dir. The md_template_path file is expected to contain the + string {reagent_resources_table} which is replaced with the contents of the actual table. """ # Read the dataframe and keep entries that are "NA", don't convert to nan df = pd.read_csv(csv_file_path, dtype=str, keep_default_na=False) + df.sort_values(by=["Target Name / Protein Biomarker"], inplace=True) supporting_material_path = pathlib.PurePath(supporting_material_root_dir).name if not df.empty: print("Start linking to supporting material...") @@ -210,8 +213,13 @@ def csv_to_md_with_url( print(f"Vendor ({k}) not found in {vendor_to_website_json_file_path}.") return 1 print("Finished linking to vendor websites...") + + with open(template_file_path, "r") as fp: + input_md_str = fp.read() with open(supporting_material_root_dir.parent / "reagent_resources.md", "w") as fp: - fp.write("# Reagent Resources\n\n" + md_header + df.to_markdown(index=False)) + fp.write( + input_md_str.format(reagent_resources_table=df.to_markdown(index=False)) + ) return 0 @@ -222,18 +230,23 @@ def main(argv=None): description="Convert knowledge-base reagent resources file from csv to md and add hyperlinks." ) parser.add_argument( - "csv_file", type=file_path, help="Path to the reagent_resources.csv file." + "md_template_file", + type=file_path, + help='Path to template markdown file which contains the string "{reagent_resources_table}".', ) parser.add_argument( - "supporting_material_root_dir", - type=dir_path, - help="Path to the directory containing the supporting materials files.", + "csv_file", type=file_path, help="Path to the reagent_resources.csv file." ) parser.add_argument( "vendor_to_website", type=file_path, help="JSON file containing the mapping between vendor name and website", ) + parser.add_argument( + "supporting_material_root_dir", + type=dir_path, + help="Path to the directory containing the supporting materials files.", + ) parser.add_argument( "--skip_url_validation", action="store_true", @@ -245,7 +258,10 @@ def main(argv=None): if args.skip_url_validation: requests.get = short_circuit_requests_get return csv_to_md_with_url( - args.csv_file, args.supporting_material_root_dir, args.vendor_to_website + template_file_path=args.md_template_file, + csv_file_path=args.csv_file, + supporting_material_root_dir=args.supporting_material_root_dir, + vendor_to_website_json_file_path=args.vendor_to_website, ) except Exception as e: print( From da62534e53323242178f60066332f1c3e52e8974 Mon Sep 17 00:00:00 2001 From: Ziv Yaniv Date: Fri, 20 Jan 2023 15:02:16 -0500 Subject: [PATCH 3/6] Change the computed stats on the landing page. Compute number_of_contributors, number_of_validated_reagents, number_of_fluorescent_probes, number_of_tissues. --- .../update_index_md_stats.py | 55 +++++++++++++------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/src/ibex_imaging_knowledge_base_utilities/update_index_md_stats.py b/src/ibex_imaging_knowledge_base_utilities/update_index_md_stats.py index 827e070..246c591 100644 --- a/src/ibex_imaging_knowledge_base_utilities/update_index_md_stats.py +++ b/src/ibex_imaging_knowledge_base_utilities/update_index_md_stats.py @@ -46,36 +46,59 @@ def update_index_stats(input_md, input_csv, output_file): with open(input_md, "r") as fp: input_md_str = fp.read() stats_dictionary = compute_stats_dictionary(input_csv) - stats_dictionary[ - "do_not_edit_message" - ] = "\n\n" with open(output_file, "w") as fp: fp.write(input_md_str.format(**stats_dictionary)) +def entry2list(entry): + """ + Replace a string entry with a If the entry is + nan a null string or "NA" return an empty list. + Otherwise, the string is split using the semicolon as + the separator character, leading and trailing whitespace is + removed from the substrings. + """ + if pd.isna(entry) or entry.strip() == "": + return set() + else: + res_list = [v.strip() for v in entry.split(";") if v.strip() != ""] + res = set(res_list) + if len(res_list) != len(res): + raise ValueError(f"entry with duplicate values - {entry}") + return res + + def compute_stats_dictionary(input_csv): stats_dict = {} df = pd.read_csv(input_csv, dtype=str, keep_default_na=False) - stats_dict["number_of_contributors"] = df["Contributor"].nunique() - stats_dict["number_of_recommended_antibodies"] = df["Catalog Number"][ - df["Recommend"] == "Yes" - ].nunique() - stats_dict["number_of_not_recommended_antibodies"] = df["Catalog Number"][ - df["Recommend"] == "No" - ].nunique() - stats_dict["number_of_fluorophores"] = len( + # Compute number of contributors, both original and folks that + # replicated the validation and either agree or disagree with the + # original contribution. The original contributor added the ORCID + # to the "Agree" column and the "Contributor" column, so no need to + # look at the "Contributor" column. + all_contributions = df["Agree"].tolist() + df["Disagree"].tolist() + all_unique_contributors = set( + [ + v.strip() + for x in all_contributions + for v in x.split(";") + if v.strip() != "NA" + ] + ) + stats_dict["number_of_contributors"] = len(all_unique_contributors) + stats_dict["number_of_validated_reagents"] = len(df) + stats_dict["number_of_fluorescent_probes"] = len( df["Conjugate"][ ~df["Conjugate"].isin( [ "NA", - "AF594", - "eF615", - "Hoechst", - "JOJO-1", "Unconjugated", - "PE/Dazzle AF594", "Biotin", "HRP", + "UT014", + "UT015", + "UT016", + "UT019", ] ) ].unique() From dc7136f7191b55ecfb23f58aa034279e47436af9 Mon Sep 17 00:00:00 2001 From: Ziv Yaniv Date: Fri, 20 Jan 2023 15:04:28 -0500 Subject: [PATCH 4/6] Update tests for all the modified and new scripts. --- tests/data/fluorescent_probes.csv | 67 +++++++++++++++++++++++++++++ tests/data/fluorescent_probes.md.in | 10 +++++ tests/data/index.md.in | 39 ++++++++++------- tests/test_scripts.py | 38 ++++++++++++++-- 4 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 tests/data/fluorescent_probes.csv create mode 100644 tests/data/fluorescent_probes.md.in diff --git a/tests/data/fluorescent_probes.csv b/tests/data/fluorescent_probes.csv new file mode 100644 index 0000000..d99dfb4 --- /dev/null +++ b/tests/data/fluorescent_probes.csv @@ -0,0 +1,67 @@ +Fluorescent Probe,Excitation Max (nm),Emission Max (nm),Signal Inactivation Conditions IBEX2D Manual +Hoechst,350,461,Does not bleach +VioGreen,388,520,1 mg/ml LiBH4 15 minutes +VioBlue,400,452,1 mg/ml LiBH4 15 minutes +Spark Violet 538,400,538,1 mg/ml LiBH4 15 minutes +StarBright Violet 670,401,667,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +StarBright Violet 710,402,713,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +BV421,405,421,1 mg/ml LiBH4 15 minutes + Light +eF450,405,450,1 mg/ml LiBH4 15 minutes +BV510,405,510,1 mg/ml LiBH4 15 minutes + Light +BV570,405,570,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +Pacific Blue,410,455,1 mg/ml LiBH4 15 minutes +Cyan Fluorescent Protein (CFP),435,485,Does not bleach +StarBright Blue 700,473,703,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +PerCP-Vio 700,482,704,Not tested +Green Fluorescent Protein (GFP),488,510,Does not bleach +iF488,488,530,1 mg/ml LiBH4 15 minutes +AF488,490,525,1 mg/ml LiBH4 15 minutes +AF488 (Plus),490,525,1 mg/ml LiBH4 15 minutes +FITC,490,525,1 mg/ml LiBH4 30 minutes +CL490,491,515,1 mg/ml LiBH4 15 minutes +Spark Blue 574,506,574,1 mg/ml LiBH4 15 minutes +iF514,511,527,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +Yellow Fluorescent Protein (YFP),513,527,Does not bleach +JOJO-1,530,544,Does not bleach +AF532,532,554,1 mg/ml LiBH4 15 minutes +iF532,537,560,1 mg/ml LiBH4 15 minutes +CL550,550,575,1 mg/ml LiBH4 15 minutes +AF555,555,580,1 mg/ml LiBH4 15 minutes +AF555 (Plus),555,580,1 mg/ml LiBH4 15 minutes +Spark YG 570,555,570,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +Red Fluorescent Protein (RFP),555,584,Does not bleach +AF546,556,573,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +eF570,556,569,1 mg/ml LiBH4 15 minutes +iF555,557,570,1 mg/ml LiBH4 15 minutes +PE,565,578,1 mg/ml LiBH4 15 minutes +RY586,565,586,Not tested +PE/iF594,565,606,1 mg/ml LiBH4 15 minutes +PE/Dazzle AF594,565,610,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +PE-Vio 615,565,619,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +PE-Vio 770,565,775,1 mg/ml LiBH4 15 minutes +AF568,578,603,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +iF594,588,604,1 mg/ml LiBH4 15 minutes +AF594,590,617,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment. +CL594,593,614,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +CF594,593,615,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +eF615,595,615,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +Texas Red,596,615,Does not bleach within 15 minutes of 1 mg/ml LiBH4 treatment +AF633,631,650,1 mg/ml LiBH4 15 minutes +eF660,633,669,1 mg/ml LiBH4 15 minutes +AF647,650,665,1 mg/ml LiBH4 15 minutes +AF647 (Plus),650,665,1 mg/ml LiBH4 15 minutes +APC-Vio 770,652,775,Not tested +CL650,655,676,1 mg/ml LiBH4 15 minutes +iF647,656,670,1 mg/ml LiBH4 15 minutes +AF660,662,690,1 mg/ml LiBH4 15 minutes +AF680,679,702,1 mg/ml LiBH4 15 minutes +iF680,684,701,1 mg/ml LiBH4 15 minutes +Spark Red 718,687,718,1 mg/ml LiBH4 15 minutes +Vio Bright R720,695,720,1 mg/ml LiBH4 15 minutes +AF700,702,723,1 mg/ml LiBH4 15 minutes +AF750,749,775,1 mg/ml LiBH4 15 minutes +iF750,757,779,1 mg/ml LiBH4 15 minutes +BL759/780,759,780,1 mg/ml LiBH4 15 minutes +DL755,776,754,1 mg/ml LiBH4 15 minutes +AF790,784,814,1 mg/ml LiBH4 15 minutes +AF800 (Plus),786,790,1 mg/ml LiBH4 15 minutes diff --git a/tests/data/fluorescent_probes.md.in b/tests/data/fluorescent_probes.md.in new file mode 100644 index 0000000..e53e721 --- /dev/null +++ b/tests/data/fluorescent_probes.md.in @@ -0,0 +1,10 @@ +# Fluorescent Probes Tested by the IBEX Imaging Community + + + +Summary of fluorescent probes tested by the IBEX Imaging Community. Inactivation conditions are method specific. + +For the original IBEX2D manual method that uses 1 mg/ml of LiBH4. The time, concentration of LiBH4, and method (continuous exchange, bleaching in the presence of light) may vary by user. + + +{probe_table} diff --git a/tests/data/index.md.in b/tests/data/index.md.in index 83295f7..6301910 100644 --- a/tests/data/index.md.in +++ b/tests/data/index.md.in @@ -2,21 +2,20 @@ [![Creative Commons License](https://i.creativecommons.org/l/by/4.0/88x31.png)](http://creativecommons.org/licenses/by/4.0/)      [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md) -{do_not_edit_message} + + - - - + + - - - + + @@ -24,10 +23,9 @@ # Iterative Bleaching Extends Multiplexity (IBEX) Knowledge-Base -The Iterative Bleaching Extends Multiplexity (IBEX) imaging method is an iterative immunolabeling and chemical bleaching method that enables highly multiplexed imaging in diverse tissues. The method was originally developed in the Laboratory of Dr. [Ronald Germain](https://irp.nih.gov/pi/ronald-germain) in the Lymphocyte Biology Section, [Laboratory of Immune System Biology](https://www.niaid.nih.gov/research/lab-immune-system-biology) at the US National Institutes of Health and described in the following publications: +The Iterative Bleaching Extends Multiplexity (IBEX) imaging method is an iterative immunolabeling and chemical bleaching method that enables highly multiplexed imaging in diverse tissues. Development of the [IBEX method](https://doi.org/10.1038/s41596-021-00644-9) and [related software](https://github.com/niaid/imaris_extensions) was led by Dr. Andrea Radtke and Dr. Ziv Yaniv. [IBEX](https://doi.org/10.1073/pnas.2018488117) and related methods, [Ce3D](https://doi.org/10.1073/pnas.1708981114), [Ce3D-IBEX](https://doi.org/10.1111/imr.13052), [Opal-plex](https://doi.org/10.1073/pnas.2018488117), were originally developed in the laboratory of [Dr. Ronald N. Germain](https://irp.nih.gov/pi/ronald-germain), the US National Institutes of Health. + -* A. J. Radtke et al., "IBEX: A versatile multiplex optical imaging approach for deep phenotyping and spatial analysis of cells in complex tissues", Proc. Natl. Acad. Sci. USA, 117(52): 33455-33465, 2020, [doi: 10.1073/pnas.2018488117](https://doi.org/10.1073/pnas.2018488117). -* A. J. Radtke et al., "IBEX: an iterative immunolabeling and chemical bleaching method for high-content imaging of diverse tissues", Nat. Protoc., 17(2):378-401, 2022, [doi: 10.1038/s41596-021-00644-9](https://doi.org/10.1038/s41596-021-00644-9). ## Overview The IBEX Imaging Community is an international group of scientists committed to sharing knowledge related to multiplexed imaging in a transparent and collaborative manner. This open, global repository is a central resource for reagents, protocols, panels, publications, software, and datasets. In addition to IBEX, we support standard, single cycle multiplexed imaging (Multiplexed 2D imaging), volume imaging of cleared tissues with clearing enhanced 3D (Ce3D), highly multiplexed 3D imaging (Ce3D-IBEX), and extension of the IBEX dye inactivation protocol to the Leica Cell DIVE (Cell DIVE-IBEX). @@ -54,10 +52,21 @@ Unlike publications, in which only successful work is described, the goal of thi Our practice of sharing recommended and not recommended reagents prevents other researchers from wasting time and effort on work that is known to fail. The significant material cost of validating antibodies and developing imaging panels is well-described [[Hickey et al. 2021](https://doi.org/10.1038/s41592-021-01316-y)]. By stewarding our time and resources, we make science more equitable, allowing investigators around the world to prioritize community-validated reagents for their work. ### Commitment to excellence -As scientists we strive to enable others to reproduce our work, confirming or refuting our results, thus making science self-correcting. Self-correction does not happen by default, it requires an explicit effort on our part [[Ioannidis 2012](https://doi.org/10.1177/1745691612464056)]. We are committed to reporting details that are critical for the success of a reagent, e.g., antigen retrieval conditions, detergent in blocking buffer, best conjugates/formats, and alternative clones, if applicable. We therefor include information related to antibody validation such as positive and negative controls and other relevant details in the detailed **notes section** of the supporting materials files. +As scientists we strive to enable others to reproduce our work, confirming or refuting our results, thus making science self-correcting. Self-correction does not happen by default, it requires an explicit effort on our part [[Ioannidis 2012](https://doi.org/10.1177/1745691612464056)]. We are committed to reporting details that are critical for the success of a reagent, e.g., antigen retrieval conditions, detergent in blocking buffer, best conjugates/formats, and alternative clones, if applicable. We, therefore, include information related to antibody validation such as positive and negative controls and other relevant details in the detailed notes section of the supporting materials files, as illustrated [here](./supporting_material/CD21_PE/0000-0003-4379-8967.md). ### Power of iteration -This repository contains the *current state of knowledge* with respect to the IBEX imaging method and associated protocols. The authoritative and versioned knowledge-base is available on Zenodo. This knowledge-base, like the method, is iterative. Help improve this community resource by contributing: -1. A question on the [discussion forum](https://github.com/IBEXImagingCommunity/ibex_imaging_knowledge_base/discussions). -1. A publication utilizing the IBEX imaging protocol (current list of [publications](https://IBEXImagingCommunity.github.io/ibex_imaging_knowledge_base/publications.html)). -1. Confirm previous reagent results or contribute new ones (see [how to contribute](contrib.md)). +This repository contains the *current state of knowledge* with respect to the IBEX imaging method and associated protocols. The authoritative and versioned knowledge-base is available on Zenodo. This knowledge-base, like the method, is iterative. Help advance and refine this community resource by: +. +1. Asking a question on the [discussion forum](https://github.com/IBEXImagingCommunity/ibex_imaging_knowledge_base/discussions). +1. Adding a publication utilizing the IBEX imaging protocol to the list of [publications](https://IBEXImagingCommunity.github.io/ibex_imaging_knowledge_base/publications.html). +1. Confirming previous reagent results or contribute new ones (see [how to contribute](contrib.md)). + +# Supported by + +

+ National Institute of Allergy and Infectious Diseases Logo National Cancer Institute Logo  Schroeder Allergy and Immunology Research Institute Logo
Chan Zuckerberg Initiative Logo Wellcome Trust Logo
+

+ +--- + +Banner created by Autumn Yarmosh and David M. Sullivan. diff --git a/tests/test_scripts.py b/tests/test_scripts.py index c45eb07..cb5edf4 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -5,6 +5,8 @@ from ibex_imaging_knowledge_base_utilities.reagent_resources_csv_2_md_url import ( csv_to_md_with_url, ) +from ibex_imaging_knowledge_base_utilities.fluorescent_probes_csv_2_md import fluorescent_probe_csv_to_md + from ibex_imaging_knowledge_base_utilities.bib2md import bibfile2md from ibex_imaging_knowledge_base_utilities.update_index_md_stats import ( update_index_stats, @@ -33,18 +35,20 @@ def files_md5(self, file_path_list): class TestCSV2MD(BaseTest): @pytest.mark.parametrize( - "csv_file_name, supporting_material_root_dir, vendor_to_website_json_file_path, result_md5hash", + "md_template_file_name, csv_file_name, supporting_material_root_dir, vendor_to_website_json_file_path, result_md5hash", [ ( + "reagent_resources.md.in", "reagent_resources.csv", "supporting_material", "vendor_urls.json", - "4a918d9274950a9ce091310cdb2903c2", + "eaaff9000872870cfd0712ecc372f622", ) ], ) def test_csv_2_md_with_url( self, + md_template_file_name, csv_file_name, supporting_material_root_dir, vendor_to_website_json_file_path, @@ -52,6 +56,7 @@ def test_csv_2_md_with_url( ): csv_to_md_with_url( + self.data_path / md_template_file_name, self.data_path / csv_file_name, self.data_path / supporting_material_root_dir, self.data_path / vendor_to_website_json_file_path, @@ -61,6 +66,33 @@ def test_csv_2_md_with_url( == result_md5hash ) +class TestFluorescentProbesCSV2MD(BaseTest): + @pytest.mark.parametrize( + "md_template_file_name, csv_file_name, result_md5hash", + [ + ( + "fluorescent_probes.md.in", + "fluorescent_probes.csv", + "5a1133df3230beb235bc3e4eda324d90", + ) + ], + ) + def test_fluorescent_probe_csv_to_md( + self, + md_template_file_name, + csv_file_name, + result_md5hash, + tmp_path + ): + fluorescent_probe_csv_to_md( + template_file_path = self.data_path / md_template_file_name, + csv_file_path = self.data_path / csv_file_name, + output_dir = tmp_path + ) + assert ( + self.files_md5([tmp_path / "fluorescent_probes.md"]) + == result_md5hash + ) class TestBib2MD(BaseTest): @pytest.mark.parametrize( @@ -81,7 +113,7 @@ def test_bib_2_md(self, bib_file_name, csl_file_name, result_md5hash, tmp_path): class TestUpdateIndexMDStats(BaseTest): @pytest.mark.parametrize( "input_md_file_name, csv_file_name, result_md5hash", - [("index.md.in", "reagent_resources.csv", "4bb071331c7fe7945e0759e9c95bbd12")], + [("index.md.in", "reagent_resources.csv", "776c99aec2968209d2e351e63e6b325a")], ) def test_update_index_stats( self, input_md_file_name, csv_file_name, result_md5hash, tmp_path From a1b18ee20957bb5423d662542378115966112c02 Mon Sep 17 00:00:00 2001 From: Ziv Yaniv Date: Fri, 20 Jan 2023 15:05:40 -0500 Subject: [PATCH 5/6] Updating changelog before release. --- CHANGELOG.md | 13 ++++++++++ .../fluorescent_probes_csv_2_md.py | 22 ++++++---------- tests/test_scripts.py | 25 ++++++++----------- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90a834a..645787a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,19 @@ needed. This is equivalent to summarizing all activity on a feature branch versu ## Unreleased +## v0.3.0 + +### Added +* fluorescent_probes_csv_2_md - script for creating the knowledge-base fluorescent_probes markdown page from the fluorescent_probes.csv. + +### Changed +* reagent_resources_csv_2_md_url - In addition to the reagent_resources.csv we now use a template file into which the table is written. Allows us to modify the descriptive text without modifying code. Additionally, the table is sorted on the "Target Name / Protein Biomarker" column. +* update_index_md_stats - Change the computed statistics to: + 1. number_of_contributors - count both original contributors and folks that replicated the work. + 1. number_of_validated_reagents - count rows in the reagent_resources.csv. + 1. number_of_fluorescent_probes - count number of unique entries in conjugate column of the reagent_resources.csv (ignoring NA, Unconjugated, Biotin, HRP, UT014, UT015, UT016, UT019). + 1. number_of_tissues - count unique combinations of Target_Species-Target_Tissue-Tissue_State. + ## v0.2.0 ### Added diff --git a/src/ibex_imaging_knowledge_base_utilities/fluorescent_probes_csv_2_md.py b/src/ibex_imaging_knowledge_base_utilities/fluorescent_probes_csv_2_md.py index ff35d90..6844408 100644 --- a/src/ibex_imaging_knowledge_base_utilities/fluorescent_probes_csv_2_md.py +++ b/src/ibex_imaging_knowledge_base_utilities/fluorescent_probes_csv_2_md.py @@ -19,9 +19,8 @@ import pandas as pd import argparse import sys -import pathlib -from .argparse_types import file_path -import json +from .argparse_types import file_path, dir_path + """ This script converts the IBEX knowledge-base fluorescent_probes.csv file to markdown. @@ -29,15 +28,12 @@ This script is automatically run when modifications to the fluorescent_probes.csv file is merged into the main branch of the ibex_knowledge_base repository (see .github/workflows/data2md.yml). -Assumption: The fluorescent_probes.csv file is valid. It conforms to the expected format (empty entries denoted +Assumption: The fluorescent_probes.csv file is valid. It conforms to the expected format (empty entries denoted by the string "NA"). """ -def fluorescent_probe_csv_to_md( - template_file_path, - csv_file_path, - output_dir -): + +def fluorescent_probe_csv_to_md(template_file_path, csv_file_path, output_dir): """ Convert the IBEX knowledge-base fluorescent probe csv file to markdown. Output is written to a file named fluorescent_probes.md in the output directory. The template_file_path file is expected @@ -50,16 +46,14 @@ def fluorescent_probe_csv_to_md( with open(template_file_path, "r") as fp: input_md_str = fp.read() with open(output_dir / "fluorescent_probes.md", "w") as fp: - fp.write( - input_md_str.format(probe_table=df.to_markdown(index=False)) - ) + fp.write(input_md_str.format(probe_table=df.to_markdown(index=False))) def main(argv=None): if argv is None: # script was invoked from commandline argv = sys.argv[1:] parser = argparse.ArgumentParser( - description="Convert knowledge-base fluorescent probes file from csv to md and sort according to excitation and emission." + description="Convert knowledge-base fluorescent probes file from csv to md and sort according to excitation and emission." # noqa E501 ) parser.add_argument( "md_template_file", @@ -80,7 +74,7 @@ def main(argv=None): return fluorescent_probe_csv_to_md( template_file_path=args.md_template_file, csv_file_path=args.csv_file, - output_dir=args.output_dir + output_dir=args.output_dir, ) except Exception as e: print( diff --git a/tests/test_scripts.py b/tests/test_scripts.py index cb5edf4..67d1eb8 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -5,7 +5,9 @@ from ibex_imaging_knowledge_base_utilities.reagent_resources_csv_2_md_url import ( csv_to_md_with_url, ) -from ibex_imaging_knowledge_base_utilities.fluorescent_probes_csv_2_md import fluorescent_probe_csv_to_md +from ibex_imaging_knowledge_base_utilities.fluorescent_probes_csv_2_md import ( + fluorescent_probe_csv_to_md, +) from ibex_imaging_knowledge_base_utilities.bib2md import bibfile2md from ibex_imaging_knowledge_base_utilities.update_index_md_stats import ( @@ -35,7 +37,7 @@ def files_md5(self, file_path_list): class TestCSV2MD(BaseTest): @pytest.mark.parametrize( - "md_template_file_name, csv_file_name, supporting_material_root_dir, vendor_to_website_json_file_path, result_md5hash", + "md_template_file_name, csv_file_name, supporting_material_root_dir, vendor_to_website_json_file_path, result_md5hash", # noqa E501 [ ( "reagent_resources.md.in", @@ -66,6 +68,7 @@ def test_csv_2_md_with_url( == result_md5hash ) + class TestFluorescentProbesCSV2MD(BaseTest): @pytest.mark.parametrize( "md_template_file_name, csv_file_name, result_md5hash", @@ -78,21 +81,15 @@ class TestFluorescentProbesCSV2MD(BaseTest): ], ) def test_fluorescent_probe_csv_to_md( - self, - md_template_file_name, - csv_file_name, - result_md5hash, - tmp_path + self, md_template_file_name, csv_file_name, result_md5hash, tmp_path ): fluorescent_probe_csv_to_md( - template_file_path = self.data_path / md_template_file_name, - csv_file_path = self.data_path / csv_file_name, - output_dir = tmp_path - ) - assert ( - self.files_md5([tmp_path / "fluorescent_probes.md"]) - == result_md5hash + template_file_path=self.data_path / md_template_file_name, + csv_file_path=self.data_path / csv_file_name, + output_dir=tmp_path, ) + assert self.files_md5([tmp_path / "fluorescent_probes.md"]) == result_md5hash + class TestBib2MD(BaseTest): @pytest.mark.parametrize( From 3045065aaf4817acce0ae129af14f2b9bcacb2ce Mon Sep 17 00:00:00 2001 From: Ziv Yaniv Date: Fri, 20 Jan 2023 15:30:07 -0500 Subject: [PATCH 6/6] Adding missing data file for testing. --- tests/data/reagent_resources.md.in | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/data/reagent_resources.md.in diff --git a/tests/data/reagent_resources.md.in b/tests/data/reagent_resources.md.in new file mode 100644 index 0000000..cfb0306 --- /dev/null +++ b/tests/data/reagent_resources.md.in @@ -0,0 +1,9 @@ +# Reagent Resources + + + +To view validation images, notes, and accompanying publications for a listed reagent: click the links in the Agree and Disagree columns. Reagents can also be searched by catalog number or other identifiers using the Find function of the web browser. + +Additional information on fluorescent probes tested by the IBEX Imaging Community can be found [here](fluorescent_probes.md). + +{reagent_resources_table}

{number_of_contributors}

{number_of_recommended_antibodies}

{number_of_not_recommended_antibodies}

{number_of_fluorophores}

{number_of_validated_reagents}

{number_of_fluorescent_probes}

{number_of_tissues}

Contributors

Antibodies Recommended

Antibodies
Not Recommended

Fluorophores
IBEX Approved

Community Validated
Reagents

Fluorescent Probes
Tested

Tissues