Merge for 2.16.0 release

.github/workflows/flake.yml

@@ -9,7 +9,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.8]
+        python-version: [3.9]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -21,4 +21,4 @@ jobs:
     - name: Test
       run: |
         pip install flake8
-        flake8 tools modules/parallel/pyext modules/test/pyext modules/mmcif/pyext
+        flake8 tools modules/parallel/pyext modules/test/pyext modules/mmcif/pyext modules/statistics/pyext modules/pepdock/pyext

CMakeLists.txt

@@ -442,7 +442,7 @@ endif(IMP_DOXYGEN_FOUND)
 file(WRITE ${CMAKE_BINARY_DIR}/IMPConfig.cmake
      "set(IMP_INSTALL_PREFIX \"${CMAKE_BINARY_DIR}\" )\n"
      "set(IMP_TOOLS_DIR \"${CMAKE_SOURCE_DIR}/tools\" )\n"
-     "set(IMP_MODULES_DIR \"${CMAKE_SOURCE_DIR}/tools/cmake\" )\n"
+     "set(IMP_USE_DIR \"${CMAKE_SOURCE_DIR}/tools/cmake\" )\n"
      "set(IMP_BUILD_INFO_DIR \"${CMAKE_BINARY_DIR}/build_info\" )\n"
      "set(IMP_INCLUDE_DIR \"${CMAKE_BINARY_DIR}/include\" )\n"
      "set(IMP_BIN_DIR \"${CMAKE_BINARY_DIR}/bin\" )\n"
@@ -451,14 +451,14 @@ file(WRITE ${CMAKE_BINARY_DIR}/IMPConfig.cmake
      "set(IMP_DATA_DIR \"${CMAKE_BINARY_DIR}/data\" )\n"
      "set(IMP_SWIG_DIR \"${CMAKE_BINARY_DIR}/swig\" )\n"
      "set(RMF_SWIG_DIR \"${CMAKE_SOURCE_DIR}/modules/rmf/dependency/RMF/swig\" )\n"
-     "set(RMF_MODULES_DIR \"${CMAKE_SOURCE_DIR}/cmake_modules\" )\n"
+     "set(IMP_MODULES_DIR \"${CMAKE_SOURCE_DIR}/cmake_modules\" )\n"
      "set(RMF_INCLUDE_PATH \"${RMF_INCLUDE_PATH}\" )\n"
-     "set(IMP_USE_FILE \"\${IMP_MODULES_DIR}/UseIMP.cmake\" )\n")
+     "set(IMP_USE_FILE \"\${IMP_USE_DIR}/UseIMP.cmake\" )\n")
 # Installed locations
 file(WRITE ${CMAKE_BINARY_DIR}/cmake/IMPConfig.cmake
      "set(IMP_INSTALL_PREFIX \"${CMAKE_INSTALL_PREFIX}\" )\n"
      "set(IMP_TOOLS_DIR \"${CMAKE_INSTALL_FULL_TOOLSDIR}\" )\n"
-     "set(IMP_MODULES_DIR \"${CMAKE_INSTALL_FULL_CMAKEDIR}\" )\n"
+     "set(IMP_USE_DIR \"${CMAKE_INSTALL_FULL_CMAKEDIR}\" )\n"
      "set(IMP_BUILD_INFO_DIR \"${CMAKE_INSTALL_FULL_BUILDINFODIR}\" )\n"
      "set(IMP_INCLUDE_DIR \"${CMAKE_INSTALL_FULL_INCLUDEDIR}\" )\n"
      "set(IMP_BIN_DIR \"${CMAKE_INSTALL_FULL_BINDIR}\" )\n"
@@ -467,9 +467,9 @@ file(WRITE ${CMAKE_BINARY_DIR}/cmake/IMPConfig.cmake
      "set(IMP_DATA_DIR \"${CMAKE_INSTALL_FULL_DATADIR}/IMP\" )\n"
      "set(IMP_SWIG_DIR \"${CMAKE_INSTALL_FULL_SWIGDIR}\" )\n"
      "set(RMF_SWIG_DIR \"${CMAKE_INSTALL_FULL_SWIGDIR}\" )\n"
-     "set(RMF_MODULES_DIR \"${CMAKE_INSTALL_FULL_CMAKEDIR}\" )\n"
+     "set(IMP_MODULES_DIR \"${CMAKE_INSTALL_FULL_CMAKEDIR}\" )\n"
      "set(RMF_INCLUDE_PATH \"${CMAKE_INSTALL_FULL_INCLUDEDIR}\" )\n"
-     "set(IMP_USE_FILE \"\${IMP_MODULES_DIR}/UseIMP.cmake\" )\n")
+     "set(IMP_USE_FILE \"\${IMP_USE_DIR}/UseIMP.cmake\" )\n")
 list(REMOVE_DUPLICATES IMP_ALL_DEPENDS_VARS)
 foreach(cmakefile ${CMAKE_BINARY_DIR}/IMPConfig.cmake
                   ${CMAKE_BINARY_DIR}/cmake/IMPConfig.cmake)

ChangeLog.md

@@ -1,6 +1,48 @@
 ChangeLog {#changelog}
 =========
 
+# 2.16.0 - 2021-12-16 # {#changelog_2_16_0}
+- OpenCubicSpline now throws a ValueException for out-of-range values, to
+  be consistent with ClosedCubicSpline (previously it threw ModelException).
+- SAXS tools (such as `compute_rg`, `foxs`) can now read input structures
+  in mmCIF format.
+- Most IMP functions that return arrays of integer or floating point values,
+  or lists of ParticleIndexes, now return NumPy arrays if IMP is built with
+  NumPy. Code that adds lists or searches for particle pairs in a list may
+  need to be modified as NumPy arrays are slightly different from Python lists.
+- IMP::pmi now handles MSE (selenomethionine) residues in input structures;
+  structure is read for such residues and they are considered equivalent to
+  regular MET in the FASTA sequence.
+- The Windows .exe installer now supports Python 3.10.
+- Scoring function evaluation can now be done more efficiently in some cases
+  by using information on which particles have moved since the last evaluation.
+  This behavior can be turned on (by default it is off) using
+  IMP::core::MonteCarlo::set_score_moved() or by setting the `score_moved`
+  parameter to IMP::pmi::macros::ReplicaExchange0.
+- The orientation-dependent scoring function IMP::score_functor::OrientedSoap
+  now caches the system topology and so should be roughly twice as fast in
+  typical applications.
+- IMP::pmi::macros::BuildSystem::add_state() now assigns multi-character
+  chain IDs by default, so that it is no longer limited to creating 62
+  molecules.
+- IMP::pmi::output::Output now reports a ValueError if asked to write out
+  a PDB file containing multi-character chain IDs, rather than silently
+  truncating them.
+- The unused IMP::piecewise_linear_distribution class has been removed.
+  Use boost::piecewise_linear_distribution instead.
+- The deprecated methods
+  IMP::{Singleton,Pair,Triplet,Quad}Predicate::get_value() have been removed.
+  Use the get_value_index() methods instead.
+- The deprecated methods IMP::SingletonContainer::get_particles(),
+  IMP::PairContainer::get_particle_pairs(),
+  IMP::TripletContainer::get_particle_triplets(), and
+  IMP::QuadContainer::get_particle_quads() have been removed. Use the
+  get_contents() method instead.
+- The deprecated IMP::isd::Weight::get_number_of_states() method has been
+  removed. Use get_number_of_weights() instead.
+- We no longer provide packages for Ubuntu 16.04 LTS (Xenial Xerus), as it
+  reached end of life in April 2021.
+
 # 2.15.0 - 2021-06-02 # {#changelog_2_15_0}
 - The [Homebrew](https://brew.sh/) Mac packages now support Apple Silicon.
 - We no longer provide RPMs for CentOS 6, since it has reached end of life.
@@ -549,15 +591,15 @@ tools/git/developer_tools/init_git if you have an existing clone.
 - 11/3: IMP::container::InContainerPairFilter and friends now treat pairs as unordered. A flag was added to control this.
 - 9/5: IMP::base::Object classes now have a method IMP::base::Object::clear_caches() that clears all cache data in the object. This is not universally implemented, but now provides a correct path for doing that. The IMP_LIST() type macros call that method when their contents changes.
 ## Summer 2012
-- 7/3: IMP_PERIODIC_OPTIMIZER_STATE() was superceded by the class IMP::core::PeriodicOptimizerState as it is rather cleaner than the macro.
+- 7/3: IMP_PERIODIC_OPTIMIZER_STATE() was superseded by the class IMP::core::PeriodicOptimizerState as it is rather cleaner than the macro.
 - 7/2: in order to make various base classes in IMP more Python friendly, macros IMP_PROTECTED_METHOD(), IMP_PROTECTED_CONSTRUCTOR() were added that properly expose such methods to Python. The standards checks now complain if \c protected is used an a class in the API.
 - 6/24: Executables from \c module/bin are now put in \b build/module_bin/module. This removes a recurring source of errors in the build script due to the origin and destination directory have the same name in scons.
 - 6/24: Benchmarks are now put int \c module/benchmark and build to \c build/benchmark/module to remove a recurring source of errors.
 - 6/18: The IMP::benchmark module supports the GPerfTools profilers when available. Benchmark executables (things in \c bin whose name starts with \c benchmark_) that use the IMP_BENCHMARK() macro have command line flags to turn on and cpu and memory profiling.
 - 6/12: The naming rules for .i files that are in module \c pyext directories changed. They should now be \c IMP_modulename.name.i instead of \c IMP_modulename_name.i. This is to better support underscores in names.
 ## Winter 2011/2012
 - 20/4: A method, IMP::atom::read_pdb(IMP::base::TextInput, int, IMP::atom::Hierarchy), was added to load coordinates from a pdb file into an existing IMP::atom::Hierarchy.
-- 4/4: the filter types were superceded by the predicate types. You will only see a difference if you develop a filter class (eg IMP::PairFilter -derived class) or directly call methods on filters (specifically IMP::PairFilter::get_contains()).
+- 4/4: the filter types were superseded by the predicate types. You will only see a difference if you develop a filter class (eg IMP::PairFilter -derived class) or directly call methods on filters (specifically IMP::PairFilter::get_contains()).
 - 29/3: IMP::Particle::get_has_model() has been changed to IMP::ModelObject::get_is_part_of_model() to be consistent with other classes.
 - 28/3: IMP::core::Mover changed slightly, adding an extra method IMP::core::Mover::get_output_particles() and requiring the IMP::Model be passed to the constructor. All movers in IMP SVN were updated without any difficulty. This allows movers to use information contained in the dependency graph amongst particles so they can, for example, weight moves based on scores.
 - 26/3: IMP::SingletonContainerInput etc were added. Methods that take these as their arguments can be passed either containers or lists of particles (or tuples), as convenient. All methods that take IMP::SingletonContainer objects should eventually move over to taking this (changing just requires changing the argument from a pointer to an Input).
@@ -584,7 +626,7 @@ tools/git/developer_tools/init_git if you have an existing clone.
 - IMP::compatibility::checked_vector was renamed to IMP::compatibility::vector and made available to IMP as IMP::vector. Typedefs should use this type (IMP::vector).
 - Various IMP::algebra plural typedefs which had been using std::vector (since they never were updated), have now been moved to IMP::vector.
 - IMP::statistics::HistogramD now supports display with \c matplotlib.
-- IMP::statistics::HistogramD supercedes IMP::statistics::Histogram
+- IMP::statistics::HistogramD supersedes IMP::statistics::Histogram
 - predicate base classes have been added: IMP::SingletonPredicate, IMP::PairPredicate etc. These take particles and return an integer. Current functionality using them includes IMP::core::PredicateSingletonScore that uses a predicate to choose which score to apply.
 - the core generic restraint and score state support was moved to the kernel to avoid breaking various scores outside of IMP svn.
 - the IMP::display module was refactored and moved up to be a dependency of IMP::core and IMP::atom rather than vice versa. The main observable results are that you should now use IMP::atom::HierarchyGeometry, IMP::atom::SelectionGeometry, IMP::atom::HierarchiesGeometry, IMP::core::XYZRGeometry, IMP::atom::BondGeometry etc instead of finding them in IMP::display. This change means that, display can be used internally when testing and debugging functionality in IMP::core and IMP::atom and that tests that use IMP::display don't need to build all of IMP::atom.
@@ -653,7 +695,7 @@ tools/git/developer_tools/init_git if you have an existing clone.
 - the name of the clustering methods have been changed to create_ from get_ since they create objects.
 - the names of the recently added metric based clustering support has been changed to Metric rather than Distance.
 - IMP now has the ability to have certain tests designated as expensive and to skip those tests when test-fast is run. The motivation for this is that the tests take a really long time making it impractical to run before every commit. And it is hard to tell which tests have not been run when running with the necessary multiple build processes. My thought is that tests that take about a second or less on a debug build should be left along (this is probably  >95% of the tests), but the few that take longer should be marked as expensive. To do that, rename the test to expensive_text_XXX.py (from test_XXX.py) and add expensive_python_tests= env.IMPModuleGetExpensivePythonTests() to the IMPModuleTest call in the test SConscript.
-- the fuction IMP::display::create_restraint_geometry() was added to do what its name says.
+- the function IMP::display::create_restraint_geometry() was added to do what its name says.
 - runtime dimensional geometric objects have been added. These include IMP::algebra::VectorKD, IMP::algebra::BoundingBoxKD and IMP::algebra::SphereKD. The IMP::statistics::Embedding classes now use IMP::algebra::VectorKD instead of Floats. The sparse grids also support variable dimensions as to the nearest neighbor searches.
 - IMP::atom::DopePairScore has been added to IMP.
 - IMP::atom::CHARMMStereochemistryRestraint provides a high-level simplified

VERSION

@@ -1 +1 @@
-2.15.0
+2.16.0

cmake_modules/IMPFindC++11.cmake

@@ -29,8 +29,14 @@ if(IMP_CXX11)
       set(IMP_CXX11_FLAGS "--std=c++0x" CACHE INTERNAL "" FORCE)
     endif()
   elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
-    # Modern clang uses C++14 by default; don't force older C++11
-    if(APPLE AND DARWIN_VERSION GREATER 18)
+    execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version
+                    OUTPUT_VARIABLE CLANG_VERSION)
+    if(CLANG_VERSION MATCHES "clang version ([0-9.]+)")
+      set(CLANG_VERSION ${CMAKE_MATCH_1})
+    endif()
+    message(STATUS "clang version: ${CLANG_VERSION}")
+    # Modern clang (6 or later) uses C++14 by default; don't force older C++11
+    if(CLANG_VERSION VERSION_GREATER 6.0)
       message(STATUS "Using clang C++11 (or later) support")
     # c++11's std::move (which boost/CGAL use) doesn't work until
     # OS X 10.9 (Darwin version 13)

doc/manual/biosystem.md

@@ -20,7 +20,9 @@ recommend that you manage your application as a GitHub repository so that
 
 We recommend the following contents for your repository (see the
 [Nup84 repository](https://github.com/integrativemodeling/nup84)
-for an example):
+for an example, or
+[this journal article](https://doi.org/10.1002/1873-3468.14067) for more
+general recommendations):
 
  - subdirectories containing
    - your modeling protocol (generally one or more Python scripts).
@@ -80,3 +82,9 @@ for an example):
      - `parallel`: if set, the tests will be run in an MPI environment, with
        the given number of cores available (by default, a serial environment
        is used)
+
+Publication or deposition generally require a [DOI](https://www.doi.org/).
+We generally do this by uploading a snapshot of the GitHub repository to
+[Zenodo](https://zenodo.org), alongside other input/output datasets that
+aren't deposited in a specialist repository such as
+[PDB](https://www.wwpdb.org/), [EMDB](https://www.ebi.ac.uk/emdb/) etc.

doc/manual/code_conventions.md

@@ -18,7 +18,7 @@ tabs. [cleanup_code.py](\ref dev_tools_cleanup_code) can help you do this format
 automatically.
 
 All Python code should conform to the [Python style
-guide](http://www.python.org/dev/peps/pep-0008/).  In essence this
+guide](https://www.python.org/dev/peps/pep-0008/).  In essence this
 translates to 4-space indents, no tabs, and similar class, method and
 variable naming to the C++ code. You can check that your Python code
 is correctly formatted using the [flake8](https://pypi.org/project/flake8/)

doc/manual/extdepends.md

@@ -1,9 +1,15 @@
 Adding external dependencies {#extdepends}
 ============================
 
-You can have an %IMP module depend on some external library.
-See the files `modules/kernel/dependencies.py` and
-`modules/kernel/dependency/Log4CXX.description` for examples.
+[TOC]
+
+# Introduction # {#extdep_intro}
+
+You can have an %IMP module depend on some external library, for example
+to take advantage of a method implemented in that library, or to link
+%IMP with another piece of software at runtime.
+
+# Considerations # {#extdep_consider}
 
 Think very carefully before introducing a new external dependency. Each
 additional dependency makes it harder for people to use your code.
@@ -16,3 +22,47 @@ Generally if you need to add a new dependency you should probably also put
 your code in a new module, rather than adding it to an existing module. That
 way, people that elect not to install that dependency will only be deprived
 of your code, not of the existing module.
+
+# Simple dependencies # {#extdep_simple}
+
+The simplest way to add a C/C++ dependency `foo` is to create a file
+`foo.description` in the `dependency` subdirectory of your module. This is a
+simple text file containing several variables:
+
+- `headers`: a colon-separated list of any C/C++ headers that need to be
+  included to use the dependency.
+- `libraries`: a colon-separated list of any libraries that need to be
+  linked against to use the dependency. (There is a similar `extra_libraries`
+  variable, if needed, for libraries that aren't part of the dependency but
+  that need to also be linked in order for it to work.)
+- `body`: a fragment of C++ code that uses the dependency.
+
+When CMake is run, it will use the variables in `foo.description` to build
+a small test program in order to make sure the dependency is available and
+that it works. See `modules/kernel/dependency/GPerfTools.description` for
+an example.
+
+# More complex dependencies # {#extdep_complex}
+
+For more complex dependencies, you can also create a `foo.cmake` file
+containing arbitrary CMake instructions to configure the dependency. See for
+example `modules/rmf/dependency/RMF.cmake`.
+
+# Using the dependency # {#extdep_using}
+
+The next step in adding a `foo` dependency is to list `foo` in the module's
+`dependencies.py` file. `foo` can be listed either in that file's
+`required_dependencies` variable or in `optional_dependencies`, both of
+which are colon-separated lists of external dependencies. See for example
+`modules/kernel/dependencies.py`.
+
+If placed in `required_dependencies`, the module cannot be built unless your
+dependency is found. The module will be automatically linked against the
+dependency.
+
+If placed in `optional_dependencies`, the module can be built either with
+or without the dependency. If the dependency is available, a preprocessor
+macro will be set when the module is built; protect your code with that
+macro. For example, IMP::score_functor can be built with or without the
+HDF5 library. Any code that requires HDF5 is conditional on the
+`IMP_SCORE_FUNCTOR_USE_HDF5` preprocessor macro.

doc/manual/install_windows.md

@@ -43,8 +43,8 @@ we employed is as follows:
 
   - Install Microsoft Visual Studio Express (it is free, but registration with
     Microsoft is required).
-  - Get and install [cmake](http://www.cmake.org).
-  - Get [Python](http://www.python.org)
+  - Get and install [cmake](https://cmake.org).
+  - Get [Python](https://www.python.org)
     (make sure you get the
     32-bit version if you're going to build %IMP for 32-bit Windows).
   - Get and install the
@@ -96,7 +96,7 @@ we employed is as follows:
     [libTAU](https://integrativemodeling.org/libTAU.html)
      - Copy `libTAU.lib` to `TAU.lib` to help cmake find it.
   - (Optional) Get the [OpenCV source code](http://opencv.org/)
-    and build it by [following these instructions](http://docs.opencv.org/doc/tutorials/introduction/windows_install/windows_install.html)
+    and build it by [following these instructions](https://docs.opencv.org/3.4.15/d3/d52/tutorial_windows_install.html#tutorial_windows_install_build)
      - Copy each `opencv_*.lib` to a similar file without the version extension
        (e.g. copy `opencv_ml244.lib` to `opencv_ml.lib`) to help cmake find it
   - Set PATH, INCLUDE, and/or LIB environment variables so that the compiler