Finish Release-1.0.0_beta6

Improved configuration, compilation and linking process.  Added options to control whether or not to build junctools, and whether to dynamically link in boost.  Improved the python detection.

Added column in tab output to handle sample number

Made GTF loading more flexible in junctools

Fixed a bug interpreting mapsplice junction files

Fixed a bug handling block sizes in bed files

Fixed several travis issues

.gitignore

@@ -36,10 +36,8 @@ callgrind*
 /.pydevproject
 *.cproject
 *.project
-/portcullis-0.*/
 /scripts/__pycache__/
 *.travis.yml.swp
 *.h
 *.ac~
-/portcullis-1.0.0/
-/portcullis-1.0.0_beta2/
+/portcullis-*/

.travis.yml

@@ -5,6 +5,10 @@ matrix:
         - os: linux
           sudo: required
           compiler: gcc
+        - os: linux
+          sudo: required
+          compiler: gcc
+          env: COMPILER=GCC5
         - os: osx
           compiler: clang
 
@@ -28,11 +32,14 @@ before_script:
 
 # Build portcullis and run tests
 script:
+- if [[ "$COMPILER" == "GCC5" ]]; then export CXX="g++-5" && export CC="gcc-5"; elif [[ "$TRAVIS_OS_NAME" == "linux" ]]; then export CXX="g++-4.9" && export CC="gcc-4.9"; fi   
 - export PATH="$TRAVIS_BUILD_DIR/exdeps/miniconda/bin:$PATH"
 - source activate test-environment
-- export LD_LIBRARY_PATH="$TRAVIS_BUILD_DIR/exdeps/miniconda/envs/test-environment/lib:$LD_LIBRARY_PATH"
+#- export LD_LIBRARY_PATH="$TRAVIS_BUILD_DIR/exdeps/miniconda/envs/test-environment/lib:$LD_LIBRARY_PATH"
+- gcc --version
+- g++ --version
 - ./configure --disable-silent-rules
-- make
-- make check
-- make distcheck
-
+- make V=1
+- make V=1 check
+#- make V=1 distcheck
+- cat tests/test-suite.log

.travis/before_install.sh

@@ -5,18 +5,19 @@ if [[ $TRAVIS_OS_NAME == 'osx' ]]; then
 else
     sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
     sudo apt-get update -qq
+    sudo apt-get install -qq libstdc++6 libc6-dev
     if [ "$COMPILER" == "GCC5" ]; then 
-        sudo apt-get install -qq g++-5
+        sudo apt-get install -qq gcc-5 g++-5
         sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 100
         sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 100
         export CXX="g++-5"
-        export CC="gcc-5"        
+        export CC="gcc-5"
     else 
-        sudo apt-get install -qq g++-4.9
+        sudo apt-get install -qq gcc-4.9 g++-4.9
         sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.9 100
         sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 100
         export CXX="g++-4.9"
         export CC="gcc-4.9"
     fi
     gcc --version
-fi
+fi

.travis/install.sh

@@ -24,9 +24,13 @@ else
 	tar -xf boost.tar.gz
 	cd boost_1_59_0
 	sudo ./bootstrap.sh --with-libraries=chrono,timer,program_options,filesystem,system
-	if [[ "$COMPILER" == "GCC5" ]]; then 
+	if [[ "$COMPILER" == "GCC5" ]]; then
+                export CXX="g++-5"
+                export CC="gcc-5"
 		sudo ./b2 -d0 --toolset=gcc-5 install; 
-	else 
+	else
+                export CXX="g++-4.9"
+                export CC="gcc-4.9"
 		sudo ./b2 -d0 --toolset=gcc-4.9 install; 
 	fi
 	cd ..
@@ -42,6 +46,7 @@ bash miniconda.sh -b -p $TRAVIS_BUILD_DIR/exdeps/miniconda; export PATH="$TRAVIS
 hash -r
 conda config --set always_yes yes --set changeps1 no
 conda update -q conda
+conda install libgcc
 conda info -a
 conda create -q -n test-environment python=3.5 anaconda
 source activate test-environment;

configure.ac

@@ -4,7 +4,7 @@
 
 # Autoconf setup
 AC_PREREQ([2.68])
-AC_INIT([portcullis],[1.0.0_beta5],[[email protected]],[portcullis],[http://www.earlham.ac.uk])
+AC_INIT([portcullis],[1.0.0_beta6],[[email protected]],[portcullis],[http://www.earlham.ac.uk])
 AC_CONFIG_SRCDIR([src/portcullis.cc])
 AC_CONFIG_AUX_DIR([build-aux])
 AC_CONFIG_MACRO_DIR([m4])
@@ -95,26 +95,34 @@ if test "$with_samtools" == no ; then
     AC_MSG_ERROR([samtools not found])
 fi 
 
-#AM_PATH_PYTHON([], [], AC_MSG_ERROR([Python interpreter not found.]))
+AC_ARG_ENABLE([junctools],
+    AS_HELP_STRING([--disable-junctools], [Disable build of python package junctools]), make_junctools="no", make_junctools="yes")
 
-AX_PYTHON_DEVEL([>= '3.1'])
+if [[ "${make_junctools}" == "yes" ]]; then
+
+    AX_PYTHON_DEVEL([>= '3.1'])
 
-if [[ -z "${PYTHON_VERSION}" ]]; then
-    AC_MSG_ERROR([Python3 not detected. If running on a debian system try "sudo apt-get install python3-dev".])
-fi    
+    if [[ -z "${PYTHON_VERSION}" ]]; then
+        AC_MSG_ERROR([Python3 not detected. If running on a debian system try "sudo apt-get install python3-dev".])
+    fi    
 
-if [[ -z "${PYTHON_EXTRA_LIBS}" ]]; then
-    AC_MSG_ERROR([Python3 detected but Python3 development library was not found. If running on a debian system try "sudo apt-get install python3-dev".])
-fi
+    if [[ -z "${PYTHON_EXTRA_LIBS}" ]]; then
+        AC_MSG_ERROR([Python3 detected but Python3 development library was not found. If running on a debian system try "sudo apt-get install python3-dev".])
+    fi
 
-pybin=python${PYTHON_VERSION}
+    pybin=python${PYTHON_VERSION}
 
-# Check for required python libraries
-AX_PYTHON_MODULE(numpy, [AC_MSG_ERROR([numpy not found])], ${pybin})
-AX_PYTHON_MODULE(matplotlib, [AC_MSG_ERROR([matplotlib not found])], ${pybin})
-AX_PYTHON_MODULE(scipy, [AC_MSG_ERROR([scipy not found])], ${pybin})
-AX_PYTHON_MODULE(sphinx, [AC_MSG_ERROR([sphinx not found])], ${pybin})
-AX_PYTHON_MODULE(sklearn, [AC_MSG_ERROR([sklearn not found])], ${pybin})     
+    # Check for required python libraries
+    #AX_PYTHON_MODULE(numpy, [AC_MSG_ERROR([numpy not found])], ${pybin})
+    #AX_PYTHON_MODULE(matplotlib, [AC_MSG_ERROR([matplotlib not found])], ${pybin})
+    #AX_PYTHON_MODULE(scipy, [AC_MSG_ERROR([scipy not found])], ${pybin})
+    #AX_PYTHON_MODULE(sphinx, [AC_MSG_ERROR([sphinx not found])], ${pybin})
+    #AX_PYTHON_MODULE(sklearn, [AC_MSG_ERROR([sklearn not found])], ${pybin})
+
+
+fi
+AM_CONDITIONAL([MAKE_JUNCTOOLS], [test x$make_junctools = xyes])
+AC_SUBST([MAKE_JUNCTOOLS])
 
 
 # Check for sphinx executable to build documentation
@@ -139,13 +147,10 @@ fi
 AM_CONDITIONAL([MAKE_DOCS], [test x$sphinx = xyes])
 AC_SUBST([MAKE_DOCS])
 
-AC_ARG_ENABLE([junctools],
-    AS_HELP_STRING([--disable-junctools], [Disable build of python package junctools]))
 
-AM_CONDITIONAL([MAKE_JUNCTOOLS], [test x$enable_junctools != xno])
-AC_SUBST([MAKE_JUNCTOOLS])
 
 ## Check for boost (required)
+AC_ARG_ENABLE([dynamic-boost], AS_HELP_STRING([--enable-dynamic-boost], [Boost is statically linked by default.  Use this option if you would prefer to dynamically link boost]), dynboost="yes", dynboost="no")
 AX_BOOST_BASE([1.52],, [AC_MSG_ERROR([Boost not found.  Please ensure that boost is properly built and the BOOST_ROOT environment variable is set.  Alternatively you can override BOOST_ROOT with the --with-boost option.])])
 AX_BOOST_SYSTEM
 AX_BOOST_FILESYSTEM
@@ -154,8 +159,26 @@ AX_BOOST_PROGRAM_OPTIONS
 AX_BOOST_TIMER
 
 
-# Combine BOOST variables
-BOOST_LIBS="${BOOST_TIMER_STATIC_LIB} ${BOOST_CHRONO_STATIC_LIB} ${BOOST_FILESYSTEM_STATIC_LIB} ${BOOST_PROGRAM_OPTIONS_STATIC_LIB} ${BOOST_SYSTEM_STATIC_LIB}"
+# Combine BOOST variables (apart for BOOST_TEST)
+BOOST_DYN_LIBS="${BOOST_TIMER_LIB} ${BOOST_CHRONO_LIB} ${BOOST_FILESYSTEM_LIB} ${BOOST_PROGRAM_OPTIONS_LIB} ${BOOST_SYSTEM_LIB}"
+AC_SUBST([BOOST_DYN_LIBS])
+
+BOOST_STATIC_LIBS="${BOOST_TIMER_STATIC_LIB} ${BOOST_CHRONO_STATIC_LIB} ${BOOST_FILESYSTEM_STATIC_LIB} ${BOOST_PROGRAM_OPTIONS_STATIC_LIB} ${BOOST_SYSTEM_STATIC_LIB} "
+AC_SUBST([BOOST_STATIC_LIBS])
+
+# Prefer static linking of boost.  If one or more static libs are not present 
+# (this happens with some default boost installations on some environments) then
+# use dynamic libs instead.
+if [[ "${dynboost}" == "yes" ]]; then
+    BOOST_LIBS="${BOOST_DYN_LIBS}"
+else
+    if [[ -z "${BOOST_TIMER_STATIC_LIB}" ]] || [[ -z "${BOOST_CHRONO_STATIC_LIB}" ]] || [[ -z "${BOOST_FILESYSTEM_STATIC_LIB}" ]] || [[ -z "${BOOST_PROGRAM_OPTIONS_STATIC_LIB}" ]] || [[ -z "${BOOST_SYSTEM_STATIC_LIB}" ]]; then
+        AC_MSG_WARN([Not all static boost libraries could be found.  Will use dynamic libraries instead.])
+        BOOST_LIBS="${BOOST_DYN_LIBS}"
+    else
+        BOOST_LIBS="${BOOST_STATIC_LIBS}"
+    fi
+fi
 AC_SUBST([BOOST_LIBS])
 
 LIBS="${SAVED_LIBS}"
@@ -166,23 +189,28 @@ LIBS="${SAVED_LIBS}"
 AM_CXXFLAGS="-g -g3 -gdwarf-4"
 AC_SUBST([AM_CXXFLAGS])
 
-#AM_CPPFLAGS="${BOOST_CPPFLAGS} ${PYTHON_CPPFLAGS} ${DLIB_CPPFLAGS} ${ZLIB_CPPFLAGS}"
 AM_CPPFLAGS="${BOOST_CPPFLAGS} ${ZLIB_CPPFLAGS}"
 AC_SUBST([AM_CPPFLAGS])
 
-#AM_LDFLAGS="${BOOST_LDFLAGS} ${PYTHON_EXTRA_LDFLAGS} ${ZLIB_LDFLAGS} ${PTHREAD_CFLAGS}"
 AM_LDFLAGS="${BOOST_LDFLAGS} ${PTHREAD_CFLAGS} ${ZLIB_LDFLAGS}"
 AC_SUBST([AM_LDFLAGS])
 
 # Order is important!
-#AM_LIBS="${M_LIB} ${DL_LIB} ${BOOST_LIBS} ${PYTHON_EXTRA_LIBS} ${PYTHON_LIBS} ${RT_LIB} ${ZLIB_LIB} ${PTHREAD_LIBS}"
 AM_LIBS="${M_LIB} ${DL_LIB} ${BOOST_LIBS} ${RT_LIB} ${ZLIB_LIB} ${PTHREAD_LIBS}"
 AC_SUBST([AM_LIBS])
 
 AC_OUTPUT
 
+if [[ "$make_junctools" == "no" ]]; then
+    AC_MSG_NOTICE([Junctools not created at user request])    
+fi
+
 if [[ "${sphinx}" == "no" ]]; then
     AC_MSG_WARN([Sphinx not detected, or version is too old.  Wont create documentation.  You can still find the documentation online at: https://portcullis.readthedocs.io/en/latest/])    
+fi
+
+if [[ "${dynboost}" == "yes" ]]; then
+    AC_MSG_NOTICE([Boost will be dynamically linked to your executable])
 else
-    AC_MSG_NOTICE([Building documentation with sphinx])
-fi
+    AC_MSG_NOTICE([Boost will be statically linked to your executable])
+fi

deps/htslib-1.3/Makefile.am

@@ -4,14 +4,14 @@ ACLOCAL_AMFLAGS = -I m4
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = phtslib.pc
 
-HTSLIB_CPPFLAGS = -I$(top_srcdir) -I$(srcdir) @AM_CPPFLAGS@ @CPPFLAGS@
+HTSLIB_CPPFLAGS = -I$(top_srcdir) -I$(srcdir) @AM_CPPFLAGS@
 
 # Create portcullis library
 lib_LTLIBRARIES = libphts.la
 
 libphts_la_LDFLAGS = -version-info 4:1:3
 libphts_la_CPPFLAGS = $(HTSLIB_CPPFLAGS)
-libphts_la_CXXFLAGS = -O3 @AM_CXXFLAGS@
+libphts_la_CFLAGS = -O3 @AM_CFLAGS@
 libphts_la_SOURCES = \
 	bgzf.c \
 	faidx.c \
@@ -111,19 +111,19 @@ check_PROGRAMS = \
 	test/fieldarith \
 	test/sam
 
-HTSLIB_TEST_CXXFLAGS = -O0 @AM_CXXFLAGS@
+HTSLIB_TEST_CFLAGS = -O0 @AM_CFLAGS@
 HTSLIB_LIBS = libphts.la @AM_LIBS@
 
 test_fieldarith_SOURCES = test/fieldarith.c
-test_fieldarith_CXXFLAGS = $(HTSLIB_TEST_CXXFLAGS)
+test_fieldarith_CFLAGS = $(HTSLIB_TEST_CFLAGS)
 test_fieldarith_CPPFLAGS = -I.. $(HTSLIB_CPPFLAGS)
-test_fieldarith_LDFLAGS = @AM_LDFLAGS@ @LDFLAGS@
+test_fieldarith_LDFLAGS = @AM_LDFLAGS@
 test_fieldarith_LDADD = $(HTSLIB_LIBS)
 
 test_sam_SOURCES = test/sam.c
-test_sam_CXXFLAGS = $(HTSLIB_TEST_CXXFLAGS)
+test_sam_CFLAGS = $(HTSLIB_TEST_CFLAGS)
 test_sam_CPPFLAGS = $(HTSLIB_CPPFLAGS)
-test_sam_LDFLAGS = @AM_LDFLAGS@ @LDFLAGS@
+test_sam_LDFLAGS = @AM_LDFLAGS@
 test_sam_LDADD = $(HTSLIB_LIBS)
 
 

deps/htslib-1.3/configure.ac

@@ -57,8 +57,8 @@ AX_CHECK_ZLIB
 
 LIBS=${SAVE_LIBS}
 
-AM_CXXFLAGS="-g -g3 -gdwarf-4 -Wno-deprecated-declarations"
-AC_SUBST([AM_CXXFLAGS])
+AM_CFLAGS="-g -g3 -gdwarf-4 -Wno-deprecated-declarations -Wno-pointer-bool-conversion"
+AC_SUBST([AM_CFLAGS])
 
 AM_CPPFLAGS=${ZLIB_CPPFLAGS}
 AC_SUBST([AM_CPPFLAGS])

deps/ranger-0.3.8/Makefile.am

@@ -50,9 +50,8 @@ library_include_HEADERS = \
 
 libranger_la_CPPFLAGS = \
 	-isystem $(top_srcdir)/include \
-	@AM_CPPFLAGS@ \
-	@CPPFLAGS@
+	@AM_CPPFLAGS@ 
 
-libranger_la_CXXFLAGS = -g -g3 -gdwarf-4 -O3 -pedantic @AM_CXXFLAGS@
+libranger_la_CXXFLAGS = -O3 -pedantic -ansi -Wall -Wextra -std=c++11 -Wno-unused-parameter @AM_CXXFLAGS@
 
 

doc/Makefile

@@ -48,6 +48,7 @@ DISTFILES = $(DPS)/index.rst \
 	    $(DPS)/requirements.rst \
 	    $(DPS)/junctools.rst \
 	    $(DPS)/using.rst \
+	    $(DPS)/faq.rst \
 	    $(DPI)/hamming.png \
 	    $(DPI)/portcullis_logo.png \
 	    $(DPA)/portcullis_genomescience2016_poster.pdf \

doc/Makefile.in

@@ -48,6 +48,7 @@ DISTFILES = $(DPS)/index.rst \
 	    $(DPS)/requirements.rst \
 	    $(DPS)/junctools.rst \
 	    $(DPS)/using.rst \
+	    $(DPS)/faq.rst \
 	    $(DPI)/hamming.png \
 	    $(DPI)/portcullis_logo.png \
 	    $(DPA)/portcullis_genomescience2016_poster.pdf \

doc/source/conf.py

@@ -53,9 +53,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.0.0_beta5'
+version = '1.0.0_beta6'
 # The full version, including alpha/beta/rc tags.
-release = '1.0.0_beta5'
+release = '1.0.0_beta6'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/source/faq.rst

@@ -0,0 +1,26 @@
+.. _faq:
+
+
+Frequently Asked Questions
+==========================
+
+Can I reduce portcullis' memory usage?
+--------------------------------------
+
+Portcullis is optimised to reduce memory usage where possible, so in most cases
+memory usage should not be an issue if you were able to align your reads in the first
+place.  However, if you are encountering out of memory issues, there are a few factors 
+that can influence memory usage in portcullis, particularly during the junction
+analysis stage.  The first is the number of threads used, more threads require more
+memory.  You can therefore reduce memory usage at the expense of runtime.  Second,
+we have an ability to process additional metrics in the junction analysis stage
+called ``--extra``, this can require large amounts of memory, so unless you really
+need this option on (there should be no direct impact on the default filtering
+setup) the switch this off.  Finally, the main driver for memory usage is the depth
+of the dataset, specifically highly covered junctions.  Should have have tens of 
+thousands of reads supporting a single junction, all these reads must be kept in
+memory.  You can therefore reduce memory usage by pre-processing the BAM files to
+either cap the depth (you can use the `Kmer Analysis Toolkit <https://github.com/TGAC/KAT>`_ 
+to identify high coverage kmers then remove reads associated with those kmers), 
+or downsample using ``samtools view -s``. 
+

doc/source/index.rst

@@ -32,6 +32,7 @@ Contents:
     using
     metrics
     junctools
+    faq
 
 
 

doc/source/junctools.rst

@@ -149,6 +149,8 @@ junction or a false positive.
     information.  However, should you wish to disable this feature you can do so
     with the ``--use_strand`` option.
 
+.. _junctools_convert:
+
 Convert
 -------