New official release: 3.4

lgmgeo · Feb 20, 2024 · 22774c6 · 22774c6
1 parent ba923c9
commit 22774c6
Show file tree

Hide file tree

Showing 242 changed files with 3,434 additions and 3,932 deletions.
diff --git a/.gitignore b/.gitignore
@@ -18,12 +18,10 @@ Python/
 share/python3/phenogenius/
 
 # Unit tests
-tests/data/check_all_public_test_directory.log
-tests/data/privateTest_*
-tests/data/test_*/command_Tcl.log
-tests/data/test_*/output/
-tests/data/ToDo
-tests/data/test_37_bracketedToAngularNotation/Rodrigo_tests_ignore
+tests/AnnotSV/privateTest_*
+tests/AnnotSV/test_*/command_Tcl.log
+tests/AnnotSV/test_*/output/
+tests/AnnotSV/ToDo
 
 # VariantConvert
 share/python3/variantconvert/pipinstall.flag

diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
 ############################################################################################################
-# AnnotSV 3.3.9                                                                                            #
+# AnnotSV 3.4                                                                                              #
 #                                                                                                          #
 # AnnotSV: An integrated tool for Structural Variations annotation and ranking                             #
 #                                                                                                          #
@@ -33,13 +33,14 @@ ETCDIR               := $(PREFIX)/etc
 SHAREDIR             := $(PREFIX)/share
 DOCDIR               := $(SHAREDIR)/doc
 BASHDIR              := $(SHAREDIR)/bash
+TESTSDIR             := $(PREFIX)/tests
 TCLVERSION           := tcl$(shell echo 'puts $${tcl_version};exit 0' | tclsh)
 TCLDIRDISTRIBUTED    := share/tcl
 TCLDIR               := $(SHAREDIR)/$(TCLVERSION)
 PYTHONDIRDISTRIBUTED := share/python3
 PYTHONDIR            := $(SHAREDIR)/python3
 ANNOTSV              := AnnotSV
-VERSION              := 3.3.6
+VERSION              := 3.4
 RM                   := /bin/rm
 RMDIR                := /bin/rmdir
 MKDIR                := install -d
@@ -118,8 +119,9 @@ install-variantconvert:
 	@echo "---------------------------"
 	touch $(DESTDIR)$(PYTHONDIR)/variantconvert/pipinstall.flag
 	chmod 777 $(DESTDIR)$(PYTHONDIR)/variantconvert/pipinstall.flag
-	pip3 install -e $(DESTDIR)$(PYTHONDIR)/variantconvert/. || pip install -e $(DESTDIR)$(PYTHONDIR)/variantconvert/. || rm -f $(DESTDIR)$(PYTHONDIR)/variantconvert/pipinstall.flag
-	@echo ""
+	pip3 install -e $(DESTDIR)$(PYTHONDIR)/variantconvert/. &> ./tmp.variantconvert.txt || pip install -e $(DESTDIR)$(PYTHONDIR)/variantconvert/. &> ./tmp.variantconvert.txt || rm -f $(DESTDIR)$(PYTHONDIR)/variantconvert/pipinstall.flag
+	rm -f ./tmp.variantconvert.txt
+	ls -l $(DESTDIR)$(PYTHONDIR)/variantconvert/pipinstall.flag || echo "variantconvert not installed"
 	touch $(DESTDIR)$(PYTHONDIR)/variantconvert/configs/GRCh37/annotsv3_from_bed.local.json
 	touch $(DESTDIR)$(PYTHONDIR)/variantconvert/configs/GRCh38/annotsv3_from_bed.local.json
 	$(CHMOD) $(DESTDIR)$(PYTHONDIR)/variantconvert/configs/GRCh37/annotsv3_from_bed.local.json
@@ -165,15 +167,15 @@ install-human-annotation: Annotations_Human_$(VERSION).tar.gz install-exomiser
 	@echo ""
 	@echo "--> Human annotation installed"
 
-install-exomiser-1: 2202_phenotype.zip
+install-exomiser-1: 2309_phenotype.zip
 	@echo ""
 	@echo "Installation of Exomiser data:"
 	@echo ""
-	$(MKDIR) -p $(DESTDIR)$(SHAREDIR)/$(ANNOTSV)/Annotations_Exomiser/2202
-	tar -xf 2202_hg19.tar.gz -C $(DESTDIR)$(SHAREDIR)/$(ANNOTSV)/Annotations_Exomiser/2202/
-	unzip 2202_phenotype.zip -d $(DESTDIR)$(SHAREDIR)/$(ANNOTSV)/Annotations_Exomiser/2202/
-	$(RM) -rf 2202_phenotype.zip
-	$(RM) -rf 2202_hg19.tar.gz
+	$(MKDIR) -p $(DESTDIR)$(SHAREDIR)/$(ANNOTSV)/Annotations_Exomiser/2309
+	tar -xf 2309_hg19.tar.gz -C $(DESTDIR)$(SHAREDIR)/$(ANNOTSV)/Annotations_Exomiser/2309/
+	unzip 2309_phenotype.zip -d $(DESTDIR)$(SHAREDIR)/$(ANNOTSV)/Annotations_Exomiser/2309/
+	$(RM) -rf 2309_phenotype.zip
+	$(RM) -rf 2309_hg19.tar.gz
 
 install-exomiser-2:
 	install -p -m 0755 $(PROPERTIES) $(DESTDIR)$(ETCDIR)/$(ANNOTSV)
@@ -203,7 +205,7 @@ Annotations_%.tar.gz:
 	@echo ""
 	@echo "Download Exomiser supporting data files:"
 	@echo ""
-	curl -C - -LO https://www.lbgi.fr/~geoffroy/Annotations/2202_hg19.tar.gz
+	curl -C - -LO https://www.lbgi.fr/~geoffroy/Annotations/2309_hg19.tar.gz
 	curl -C - -LO https://data.monarchinitiative.org/exomiser/data/$@
 
 
@@ -232,14 +234,15 @@ uninstall1:
 	$(RM) -rf $(DESTDIR)$(SHAREDIR)/$(ANNOTSV)
 	$(RM) -rf $(DESTDIR)$(BASHDIR)/$(ANNOTSV)
 	$(RM) -rf $(DESTDIR)$(ETCDIR)/$(ANNOTSV)
+	$(RM) -rf $(DESTDIR)$(TESTSDIR)/$(ANNOTSV)
 	$(RM) -rf $(DESTDIR)$(PREFIX)/Makefile
 	$(RM) -rf $(DESTDIR)$(PREFIX)/README.md
 	$(RM) -rf $(DESTDIR)$(PREFIX)/Scoring_Criteria_AnnotSV_*.xlsx
 	$(RM) -rf $(DESTDIR)$(PREFIX)/.git
 	$(RM) -rf $(DESTDIR)$(PREFIX)/.gitignore
 
 uninstall2:
-	$(RMDIR) --ignore-fail-on-non-empty $(DESTDIR)$(BINDIR) $(DESTDIR)$(BASHDIR) $(DESTDIR)$(TCLDIR) $(DESTDIR)$(PYTHONDIR) $(DESTDIR)$(DOCDIR) $(DESTDIR)$(SHAREDIR) $(DESTDIR)$(ETCDIR)
+	$(RMDIR) --ignore-fail-on-non-empty $(DESTDIR)$(BINDIR) $(DESTDIR)$(BASHDIR) $(DESTDIR)$(TCLDIR) $(DESTDIR)$(PYTHONDIR) $(DESTDIR)$(DOCDIR) $(DESTDIR)$(SHAREDIR) $(DESTDIR)$(ETCDIR) $(DESTDIR)$(TESTSDIR)
 
 uninstall3:
 	$(RMDIR) --ignore-fail-on-non-empty $(DESTDIR)$(PREFIX)

diff --git a/README.AnnotSV_3.3.6.pdf b/README.AnnotSV_3.3.6.pdf
diff --git a/README.AnnotSV_3.4.pdf b/README.AnnotSV_3.4.pdf
diff --git a/README.AnnotSV_latest.pdf b/README.AnnotSV_latest.pdf
diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 <p align="center">
-    <img src="docs/images/AnnotSV_logo.png" width="500">
+    <img src="share/doc/AnnotSV/Images/AnnotSV_logo.png" width="500">
 
 <br />
 
@@ -12,12 +12,12 @@
 # Table of contents
 
 - ## AnnotSV Annotations Engine
-    - ### [README](README.AnnotSV_latest.pdf)
-    - ### [Getting Started](docs/quickstart.md)
-    - ### [Ranking](docs/ranking.md)
-    - ### [Annotations](docs/annotations.md)
+    - ### [README](README.AnnotSV_3.4.pdf)
+    - ### [Getting Started](share/doc/AnnotSV/quickstart.md)
+    - ### [Ranking](share/doc/AnnotSV/ranking.md)
+    - ### [Annotations](share/doc/AnnotSV/annotations.md)
 - ## AnnotSV Webserver
-    - ### [Overview](docs/webserver_overview.md)
+    - ### [Overview](share/doc/AnnotSV/webserver_overview.md)
 <br />
 
 # Collaborative work

diff --git a/Scoring_Criteria_AnnotSV_latest.xlsx b/Scoring_Criteria_AnnotSV_latest.xlsx
diff --git a/Scoring_Criteria_AnnotSV_v3.3.xlsx b/Scoring_Criteria_AnnotSV_v3.3.xlsx
diff --git a/Scoring_Criteria_AnnotSV_v3.4.xlsx b/Scoring_Criteria_AnnotSV_v3.4.xlsx
diff --git a/bin/AnnotSV b/bin/AnnotSV
@@ -1,7 +1,7 @@
 #!/usr/bin/env tclsh
 
 ############################################################################################################
-# AnnotSV 3.3.9                                                                                            #
+# AnnotSV 3.4                                                                                              #
 #                                                                                                          #
 # AnnotSV: An integrated tool for Structural Variations annotation and ranking                             #
 #                                                                                                          #

diff --git a/changeLog.txt b/changeLog.txt
@@ -1,5 +1,5 @@
 ############################################################################################################
-# AnnotSV 3.3.9                                                                                            #
+# AnnotSV 3.4                                                                                              #
 #                                                                                                          #
 # An integrated tool for Structural Variations annotation                                                  #
 #                                                                                                          #
@@ -10,24 +10,29 @@
 
 For more details, please see the README file.
 
-February,  , 2024, AnnotSV version 3.4
-	- Fix a bug when setting the cytoband annotation (issue 210)
-	- Add 2 new features: Closest_left and Closest_right
-		=> AnnotSV expands the SV up to 5 megabases in both direction (left and right) and then tries to find neighboring genes
-		   In both directions, the closest gene to the SV is reported
-	- Add new phenotype prioritization method: PhenoGenius (Yauy et al., 2023, doi: 10.1101/2022.07.29.22278181)
-		- Add 3 new features: PhenoGenius_score, PhenoGenius_phenotype and PhenoGenius_specificity
-	- Gene names are now sorted by genomic coordinates and no longer sorted alphabetically ("Gene_name" feature)
+February, 16, 2024, AnnotSV version 3.4
+	- Human annotation update
 	- Add new regulatory elements annotations:
 		- Activity-by-Contact (ABC) model annotations (doi: 10.1038/s41586-021-03446-x)
 		- Massively Parallel Reporter assays (MPRA) annotations (doi: 10.1038/s41467-019-11526-w)
 	- Add novel benign SV annotation sources:
 		- from gnomad v4 (GRCh38) (issue 201)
 		- from the Human Pangenome Reference Consortium (HPRC) dataset (PACBIO long reads, GRCh38 only) (issue 202)
 		- from dbVar (NCBI’s database of genomic structural variation)
+	- Add new phenotype prioritization method: PhenoGenius (Yauy et al., 2023, doi: 10.1101/2022.07.29.22278181)
+		- Add 3 new features: PhenoGenius_score, PhenoGenius_phenotype and PhenoGenius_specificity
+	- Add 4 new features: 
+		- Closest_left and Closest_right
+		  AnnotSV expands the SV up to 5 megabases in both direction (left and right) and then tries to find neighboring genes
+		  In both directions, the closest gene to the SV is reported
+		- NCBI_gene_ID 
+		- Tx_version (transcript version)
+	- Gene names are now sorted by genomic coordinates and no longer sorted alphabetically ("Gene_name" feature)
+	- Fix a bug when setting the cytoband annotation (issue 210)
 	- Fix misleading OMIM annotation (issues 156, 132)
-	- Add "NCBI_gene_ID" and "Tx_version" annotations
+	- Update of the Makefile
 	- Update of the "contact" recommendations in output (from email to GitHub)
+	- Reorganization of the GitHub .md files
 
 January, 15, 2024, AnnotSV version 3.3.9
 	- Fix a bug when setting the "hpoVersion" variable (issue 207)

diff --git a/docs/images/AnnotSV_logo.png b/docs/images/AnnotSV_logo.png
diff --git a/etc/AnnotSV/application.properties b/etc/AnnotSV/application.properties
@@ -26,8 +26,8 @@ server.servlet.application-display-name=Exomiser Prioritiser Server
 #Absolute system path where the exomiser data is installed
 #exomiser.data-directory=${project.build.testOutputDirectory}
 exomiser.data-directory=YYYY
-exomiser.phenotype.data-version=2202
-exomiser.hg19.data-version=2202
+exomiser.phenotype.data-version=2309
+exomiser.hg19.data-version=2309
 exomiser.phenotype.random-walk-preload=true
 
 #Actuator configuration

diff --git a/share/bash/AnnotSV/checkPhenoGeniusInstall.sh b/share/bash/AnnotSV/checkPhenoGeniusInstall.sh
@@ -40,12 +40,12 @@ fi
 
 cd $ANNOTSV/share/python3/phenogenius/PhenoGenius
 
-if [ `poetry run python phenogenius_cli.py --help | grep -c -- "--hpo_list"` ]
+if [ `poetry run python phenogenius_cli.py --help | grep -c -- "--hpo_list"` == "1" ]
 then
 	exit 0
 else
 	poetry install &> ../poetry_install.log ;# Can be run from different servers
-	if [ `poetry run python phenogenius_cli.py --help | grep -c -- "--hpo_list"` ]
+	if [ `poetry run python phenogenius_cli.py --help | grep -c -- "--hpo_list"` == 1 ]
 	then
 		exit 0
 	else
@@ -55,3 +55,4 @@ else
 	fi
 fi
 
+
diff --git a/share/doc/AnnotSV/Example/test_GRCh37.annotated.example.tsv b/share/doc/AnnotSV/Example/test_GRCh37.annotated.example.tsv
diff --git a/share/doc/AnnotSV/Images/AnnotSV_logo.png b/share/doc/AnnotSV/Images/AnnotSV_logo.png
diff --git a/docs/images/AnnotSV_overview.jpg → ...e/doc/AnnotSV/Images/AnnotSV_overview.jpg b/docs/images/AnnotSV_overview.jpg → ...e/doc/AnnotSV/Images/AnnotSV_overview.jpg
diff --git a/docs/images/gpl.png → share/doc/AnnotSV/Images/gpl.png b/docs/images/gpl.png → share/doc/AnnotSV/Images/gpl.png
diff --git a/docs/annotations.md → share/doc/AnnotSV/annotations.md b/docs/annotations.md → share/doc/AnnotSV/annotations.md
@@ -2,6 +2,6 @@
 
 AnnotSV requires different data sources for the annotation and the ranking of SV. In order to provide a ready to start installation of AnnotSV, each annotation source (that do not require a commercial license) is automatically downloaded during the installation.
 
-The aim of each of these sources is fully explained in the [README](../README.AnnotSV_latest.pdf) file.
+The aim of each of these sources is fully explained in the [README](../../../README.AnnotSV_3.4.pdf) file.
 
-Annotation can be performed using either the GRCh37 or GRCh38 build of the human genome (user defined). Some of the annotations are linked to the gene name and thus provided independently of the genome build.
+Annotation can be performed using either the GRCh37 or GRCh38 build of the human genome (user defined). Some of the annotations are linked to the gene name and thus provided independently of the genome build.
diff --git a/docs/quickstart.md → share/doc/AnnotSV/quickstart.md b/docs/quickstart.md → share/doc/AnnotSV/quickstart.md
@@ -1,7 +1,7 @@
 
 # Quick Installation
 
-Requirements are fully explained in the [README](../README.AnnotSV_latest.pdf) file.
+Requirements are fully explained in the [README](../../../README.AnnotSV_3.4.pdf) file.
 
 1. The sources can be cloned to any directory:
 ```

diff --git a/docs/ranking.md → share/doc/AnnotSV/ranking.md b/docs/ranking.md → share/doc/AnnotSV/ranking.md
@@ -16,4 +16,4 @@ In order to assist the clinical interpretation of SV, AnnotSV provides on top of
 
 # Method
 
-The comprehensive and detailed scoring guidelines are available in the [Scoring_Criteria_AnnotSV](../Scoring_Criteria_AnnotSV_latest.xlsx) file (see Table1 for loss SV and Table2 for gain SV).
+The comprehensive and detailed scoring guidelines are available in the [Scoring_Criteria_AnnotSV](../../../Scoring_Criteria_AnnotSV_v3.4.xlsx) file (see Table1 for loss SV and Table2 for gain SV).
diff --git a/docs/webserver_overview.md → share/doc/AnnotSV/webserver_overview.md b/docs/webserver_overview.md → share/doc/AnnotSV/webserver_overview.md
@@ -11,10 +11,10 @@ The user can add optional sample related information such as the HPO terms to he
 ## Annotations
 AnnotSV compiles functionally, regulatory and clinically **relevant information** and aims at providing annotations useful to i) **interpret SV potential pathogenicity and ii) filter out SV potential false positives.**
 <br />
-These **relevant annotations** are detailed in the [README](../README.AnnotSV_latest.pdf).
+These **relevant annotations** are detailed in the [README](../../../README.AnnotSV_3.4.pdf).
 
 ## Phenotype-driven prioritization
-To link the patient's phenotypic data to the already available knowledge for each gene, AnnotSV provide a phenotype driven prioritization module based on the [HPO (Human Phenotype Ontology) dataset](https://pubmed.ncbi.nlm.nih.gov/30476213/) and [Exomiser](https://www.nature.com/articles/nprot.2015.124) to score each SV (from 0 to 1.0). We generally recommend a score above 0.7 for a known disease gene and 0.5 for a gene not yet related to a disease.
+To link the patient's phenotypic data to the already available knowledge for each gene, AnnotSV provide a phenotype driven prioritization module based on the [HPO (Human Phenotype Ontology) dataset](https://pubmed.ncbi.nlm.nih.gov/30476213/).
 
 ## Ranking
 In addition, AnnotSV provides on top of the annotations **a ranking score** to assess SV pathogenicity.
@@ -40,4 +40,4 @@ Every SV are reported, even those not covering a gene. This type of annotation g
 - An annotation of the SV **“split” by gene**:
 This type of annotation gives an opportunity to focus on each gene overlapped by the SV. Thus, when a SV spans over several genes, the output will contain as many annotations lines as covered genes (cf example in FAQ). This latter annotation is extremely powerful to shorten the identification of mutation implicated in a specific gene.
 
-The annotations columns available in the output file are detailed in the [README](../README.AnnotSV_latest.pdf).
+The annotations columns available in the output file are detailed in the [README](../../../README.AnnotSV_3.4.pdf).
diff --git a/share/tcl/AnnotSV/AnnotSV-benignsv.tcl b/share/tcl/AnnotSV/AnnotSV-benignsv.tcl
@@ -1,5 +1,5 @@
 ############################################################################################################
-# AnnotSV 3.3.9                                                                                            #
+# AnnotSV 3.4                                                                                              #
 #                                                                                                          #
 # AnnotSV: An integrated tool for Structural Variations annotation and ranking                             #
 #                                                                                                          #
@@ -1342,7 +1342,6 @@ proc checkCMRI_benignFile {genomeBuild} {
         if {$L_toWriteGain ne {}} {
             WriteTextInFile [join $L_toWriteGain "\n"] $benignGainFile_Tmp
         }
-        puts "\t       ([llength $L_toWriteIns] INS + [llength $L_toWriteInv] INV)"
         if {$L_toWriteIns ne {}} {
             WriteTextInFile [join $L_toWriteIns "\n"]  $benignInsFile_Tmp
         }

diff --git a/share/tcl/AnnotSV/AnnotSV-candidategenes.tcl b/share/tcl/AnnotSV/AnnotSV-candidategenes.tcl
@@ -1,5 +1,5 @@
 ############################################################################################################
-# AnnotSV 3.3.9                                                                                            #
+# AnnotSV 3.4                                                                                              #
 #                                                                                                          #
 # AnnotSV: An integrated tool for Structural Variations annotation and ranking                             #
 #                                                                                                          #

diff --git a/share/tcl/AnnotSV/AnnotSV-clingen.tcl b/share/tcl/AnnotSV/AnnotSV-clingen.tcl
@@ -1,5 +1,5 @@
 ############################################################################################################
-# AnnotSV 3.3.9                                                                                            #
+# AnnotSV 3.4                                                                                              #
 #                                                                                                          #
 # AnnotSV: An integrated tool for Structural Variations annotation and ranking                             #
 #                                                                                                          #
@@ -21,6 +21,16 @@
 # along with this program; If not, see <http://www.gnu.org/licenses/>.                                     #
 ############################################################################################################
 
+
+# ClinGenFileDownloaded:
+########################
+# ftp://ftp.clinicalgenome.org/ClinGen_gene_curation_list_GRCh37.tsv
+# Some genes are on chrom X AND chrom Y:
+# CSF2RA  1438    Xp22.33 and Yp11.2      chrX:1387693-1428828    30      Gene associated with autosomal recessive phenotype       0       No evidence available       2021-08-23       MONDO:0012580
+# CSF2RA  1438    Xp22.33 and Yp11.2      chrY:1268814-1325218    30      Gene associated with autosomal recessive phenotype       0       No evidence available       2021-08-23       MONDO:0012580
+# => be carreful to avoid redundancy
+
+
 ## - Check if the "ClinGen_gene_curation_list_GRCh37.tsv" file exists.
 ## - Check and create if necessary the 'date'_ClinGenAnnotations.tsv file.
 proc checkClinGenFile {} {
@@ -72,7 +82,11 @@ proc checkClinGenFile {} {
             set gene [lindex $Ls $i_gene]
             set HI [lindex $Ls $i_HI]
             set TS [lindex $Ls $i_TS]
-
+
+			# To avoir redundancy for genes (CSF2RA and VAMP7) presents on chrom X AND chrom Y
+			if {[info exists seen($gene)]} {continue}
+			set seen($gene) 1
+
             if {$HI eq "Not yet evaluated"} {set HI ""}
             if {$TS eq "Not yet evaluated"} {set TS ""}
 

diff --git a/share/tcl/AnnotSV/AnnotSV-closestgenes.tcl b/share/tcl/AnnotSV/AnnotSV-closestgenes.tcl
@@ -1,5 +1,5 @@
 ############################################################################################################
-# AnnotSV 3.3.9                                                                                            #
+# AnnotSV 3.4                                                                                              #
 #                                                                                                          #
 # AnnotSV: An integrated tool for Structural Variations annotation and ranking                             #
 #                                                                                                          #

diff --git a/share/tcl/AnnotSV/AnnotSV-config.tcl b/share/tcl/AnnotSV/AnnotSV-config.tcl
@@ -1,5 +1,5 @@
 ############################################################################################################
-# AnnotSV 3.3.9                                                                                            #
+# AnnotSV 3.4                                                                                              #
 #                                                                                                          #
 # AnnotSV: An integrated tool for Structural Variations annotation and ranking                             #
 #                                                                                                          #