Merge branch 'develop' of github.com:jhamman/RVIC into develop

.travis.yml

@@ -26,4 +26,6 @@ script:
   - pip freeze
   - echo $PATH
   - python -c 'import rvic; print rvic.__file__'
-  - py.test  # erroring here for unknown path reasons...
+  - cd tests
+  - pwd
+  - python run_tests.py unit

README.md

@@ -1,6 +1,8 @@
 # RVIC Streamflow Routing Model
 
-The RVIC Streamflow Routing model is a simple source to sink routing model.  The model represents each grid cell by a node in the channel network.  Unit hydrographs are developed that described the time distribution of flow from each source grid cell to a corresponding sink grid cell.  The development of the unit hydrographs is done as a pre process step (i.e. `rvic_model parameters`).  The final step is the convolution of the unit hydrographs with fluxes from a land surface model, typically VIC (i.e. `rvic_model convolution`).  
+[![Build Status](https://travis-ci.org/jhamman/RVIC.svg?branch=develop)](https://travis-ci.org/jhamman/RVIC)
+
+The RVIC Streamflow Routing model is a simple source to sink routing model.  The model represents each grid cell by a node in the channel network.  Unit hydrographs are developed that described the time distribution of flow from each source grid cell to a corresponding sink grid cell.  The development of the unit hydrographs is done as a pre process step (i.e. `rvic parameters`).  The final step is the convolution of the unit hydrographs with fluxes from a land surface model, typically VIC (i.e. `rvic convolution`).  
 
 ### Usage
 See the [RVIC Wiki Page](https://github.com/jhamman/RVIC/wiki/RVIC-Wiki)

ci/requirements-2.7-dev.txt

@@ -3,4 +3,5 @@ matplotlib==1.3.1
 netCDF4==1.0.8
 numpy==1.8.1
 scipy==0.13.3
+pandas==0.13.1
 pytest==2.5.2

ci/requirements-2.7.txt

@@ -2,3 +2,5 @@ matplotlib==1.3.1
 netCDF4==1.0.8
 numpy==1.8.1
 scipy==0.13.3
+pandas==0.13.1
+pytest==2.5.2

config/rvic_parameters_example.cfg

@@ -98,6 +98,7 @@ LATITUDE_VAR: lat
 FLOW_DISTANCE_VAR: Flow_Distance
 FLOW_DIRECTION_VAR: Flow_Direction
 BASIN_ID_VAR: Basin_ID
+SOURCE_AREA_VAR: Source_Area
 
 #-- Velocity and diffusion  --#
 # The velocity and diffusion parameters may either be specified as variables in

rvic/convert.py

@@ -3,7 +3,7 @@
 """
 
 from logging import getLogger
-from core.log import init_logger, LOG_NAME
+from core.log import init_logger, close_logger, LOG_NAME
 from core.utilities import make_directories, copy_inputs, read_domain
 from core.utilities import tar_inputs
 from core.convert import read_station_file, read_uhs_files, move_domain
@@ -143,6 +143,8 @@ def uhs2param_final(outlets, dom_data, new_dom_data, config_dict, directories):
     log.info('Location of Inputs: %s', inputs_tar)
     log.info('Location of Log: %s', log_tar)
     log.info('Location of Parmeter File %s', param_file)
+
+    close_logger()
     # ---------------------------------------------------------------- #
     return
 # -------------------------------------------------------------------- #

rvic/convolution.py

@@ -14,8 +14,9 @@
 Major updates to the...
 """
 import os
+from collections import OrderedDict
 from logging import getLogger
-from core.log import init_logger, LOG_NAME
+from core.log import init_logger, close_logger, LOG_NAME
 from core.utilities import make_directories, read_domain
 from core.utilities import write_rpointer, tar_inputs
 from core.variables import Rvar
@@ -168,7 +169,7 @@ def convolution_init(config_file):
     # Setup history Tape(s) and Write Initial Outputs
     history = config_dict['HISTORY']
     numtapes = int(history['RVICHIST_NTAPES'])
-    hist_tapes = {}
+    hist_tapes = OrderedDict()
 
     # make sure history file fields are all in list form
     if numtapes == 1:
@@ -185,14 +186,14 @@ def convolution_init(config_file):
                             RvicDomainFile=os.path.split(domain['FILE_NAME'])[1])
 
     for j in xrange(numtapes):
-        tapename = 'Tape.%i' % j
+        tapename = 'Tape.{0}'.format(j)
         log.info('setting up History %s', tapename)
         hist_tapes[tapename] = Tape(time_handle.time_ord,
                                     options['CASEID'],
                                     rout_var,
                                     tape_num=j,
                                     fincl=['streamflow'],
-                                    mfilt=int(history['RVICHIST_MFILT'][j]),
+                                    mfilt=history['RVICHIST_MFILT'][j],
                                     ndens=int(history['RVICHIST_NDENS'][j]),
                                     nhtfrq=int(history['RVICHIST_NHTFRQ'][j]),
                                     avgflag=history['RVICHIST_AVGFLAG'][j],
@@ -220,7 +221,7 @@ def convolution_init(config_file):
 
 # -------------------------------------------------------------------- #
 def convolution_run(hist_tapes, data_model, rout_var, dom_data, time_handle,
-                 directories, config_dict):
+                    directories, config_dict):
     """
     Main run loop for RVIC model.
     """
@@ -347,6 +348,8 @@ def convolution_final(time_handle, hist_tapes):
 
     log.info('Done with rvic convolution.')
     log.info('Location of Log: %s', log_tar)
+
+    close_logger()
     # ---------------------------------------------------------------- #
     return
 # -------------------------------------------------------------------- #

rvic/core/aggregate.py

@@ -4,7 +4,6 @@
 """
 
 import numpy as np
-from scipy.spatial import cKDTree
 from collections import OrderedDict
 from share import FILLVALUE_F
 from utilities import find_nearest, latlon2yx
@@ -21,8 +20,6 @@
 # -------------------------------------------------------------------- #
 # Find target cells for pour points
 def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict):
-# def make_agg_pairs(lons, lats, dom_lon, dom_lat, dom_ids,
-                   # fdr_lons, fdr_lats, fdr_srcarea, agg_type='agg'):
     """
     Group pour points by domain grid outlet cell
     """

rvic/core/config.py

@@ -3,6 +3,8 @@
 
 """
 
+import os
+from collections import OrderedDict
 from ConfigParser import SafeConfigParser
 
 
@@ -33,10 +35,10 @@ def read_config(config_file):
     config.optionxform = str
     config.read(config_file)
     sections = config.sections()
-    dict1 = {}
+    dict1 = OrderedDict()
     for section in sections:
         options = config.options(section)
-        dict2 = {}
+        dict2 = OrderedDict()
         for option in options:
             dict2[option] = config_type(config.get(section, option))
         dict1[section] = dict2
@@ -60,14 +62,44 @@ def config_type(value):
             return False
         elif value in ['none', 'None', 'NONE', '']:
             return None
+        elif isint(value):
+            return int(value)
+        elif isfloat(value):
+            return float(value)
         else:
-            try:
-                return float(value)
-            except:
-                return value
+            return os.path.expandvars(value)
     else:
         try:
             return map(float, val_list)
+        except:
+            pass
+        try:
+            return map(int, val_list)
         except:
             return val_list
 # -------------------------------------------------------------------- #
+
+
+# -------------------------------------------------------------------- #
+def isfloat(x):
+    """Test of value is a float"""
+    try:
+        a = float(x)
+    except ValueError:
+        return False
+    else:
+        return True
+# -------------------------------------------------------------------- #
+
+
+# -------------------------------------------------------------------- #
+def isint(x):
+    """Test if value is an integer"""
+    try:
+        a = float(x)
+        b = int(a)
+    except ValueError:
+        return False
+    else:
+        return a == b
+# -------------------------------------------------------------------- #

rvic/core/history.py

@@ -8,9 +8,12 @@
         - initialization:  sets tape options, determines filenames, etc.
         - update: method that incorporates new fluxes into the history tape.
         - __next_update_out_data: method to determine when to update the
-        outdata container
-
+        out_data container
+        - __next_write_out_data: method to determine when to write the out_data
+        container
+        - finish: method to close all remaining history tapes.
 """
+
 import os
 import numpy as np
 from netCDF4 import Dataset, date2num, num2date, stringtochar
@@ -46,7 +49,7 @@ def __init__(self, time_ord, caseid, Rvar, tape_num=0,
         self._tape_num = tape_num
         self._time_ord = time_ord        # Days since basetime
         self._caseid = caseid            # Case ID and prefix for outfiles
-        self._fincl = fincl              # Fields to include in history file
+        self._fincl = list(fincl)        # Fields to include in history file
         self._mfilt = mfilt              # Maximum number of time samples
         self._ndens = ndens
         if self._ndens == 1:             # Output file precision
@@ -78,6 +81,7 @@ def __init__(self, time_ord, caseid, Rvar, tape_num=0,
         else:
             # If monthly
             self._out_data_stepsize = None  # varies by month
+        log.debug('_out_data_stepsize: ', self._out_data_stepsize)
         # ------------------------------------------------------------ #
 
         # ------------------------------------------------------------ #
@@ -91,7 +95,7 @@ def __init__(self, time_ord, caseid, Rvar, tape_num=0,
                 raise ValueError('Must include grid lons / lats if '
                                  'outtype == grid')
         else:
-            self._out_data_shape = self._num_outlets
+            self._out_data_shape = (self._num_outlets, )
         # ------------------------------------------------------------ #
 
         # ------------------------------------------------------------ #
@@ -181,6 +185,8 @@ def __init__(self, time_ord, caseid, Rvar, tape_num=0,
         # Determine when the update of out_data should be
         self.__next_update_out_data()
         # ------------------------------------------------------------ #
+
+        log.debug(self.__repr__())
     # ---------------------------------------------------------------- #
 
     # ---------------------------------------------------------------- #
@@ -190,18 +196,18 @@ def __str__(self):
 
     def __repr__(self):
         parts = ['------- Summary of History Tape Settings -------',
-                 '\t# caseid:       {0}s'.format(self._caseid),
-                 '\t# fincl:        {0}s'.format(','.join(self._fincl)),
-                 '\t# nhtfrq:       {0}s'.format(self._nhtfrq),
-                 '\t# mfilt:        {0}s'.format(self._mfilt),
-                 '\t# ncprec:       {0}s'.format(self._ncprec),
-                 '\t# avgflag:      {0}s'.format(self._avgflag),
-                 '\t# fname_format: {0}s'.format(self._fname_format),
-                 '\t# file_format:  {0}s'.format(self._file_format),
-                 '\t# outtype:      {0}s'.format(self._outtype),
-                 '\t# out_dir:      {0}s'.format(self._out_dir),
-                 '\t# calendar:     {0}s'.format(self._calendar),
-                 '\t# units:        {0}s'.format(self._units),
+                 '\t# caseid:       {0}'.format(self._caseid),
+                 '\t# fincl:        {0}'.format(','.join(self._fincl)),
+                 '\t# nhtfrq:       {0}'.format(self._nhtfrq),
+                 '\t# mfilt:        {0}'.format(self._mfilt),
+                 '\t# ncprec:       {0}'.format(self._ncprec),
+                 '\t# avgflag:      {0}'.format(self._avgflag),
+                 '\t# fname_format: {0}'.format(self._fname_format),
+                 '\t# file_format:  {0}'.format(self._file_format),
+                 '\t# outtype:      {0}'.format(self._outtype),
+                 '\t# out_dir:      {0}'.format(self._out_dir),
+                 '\t# calendar:     {0}'.format(self._calendar),
+                 '\t# units:        {0}'.format(self._units),
                  '  ------- End of History Tape Settings -------']
         return '\n'.join(parts)
     # ---------------------------------------------------------------- #
@@ -276,7 +282,10 @@ def write_initial(self):
 
     # ---------------------------------------------------------------- #
     def __next_write_out_data(self):
-        """ """
+        """determine the maximum size of out_data"""
+
+        log.debug('determining size of out_data')
+
         self._out_data_i = 0            # position counter for out_data array
 
         # ------------------------------------------------------------ #
@@ -297,7 +306,6 @@ def __next_write_out_data(self):
 
                 # calculate the mfilt value
                 mfilt = int(round((b1 - b0) / self._out_data_stepsize))
-
         elif self._mfilt == 'month':
             if self._nhtfrq == 0:
                 mfilt = 1
@@ -311,19 +319,19 @@ def __next_write_out_data(self):
 
                 # calculate the mfilt value
                 mfilt = int(round((b1 - b0) / self._out_data_stepsize))
-
         elif self._mfilt == 'day':
-            if self._nhtfrq == 0:
+            if self._nhtfrq != 0:
+                b1 = b0 + 1.0
+            else:
                 raise ValueError('Incompatable values for NHTFRQ and MFILT')
-            b1 = b0 + 1.0
 
             # calculate the mfilt value
             mfilt = int(round((b1 - b0) / self._out_data_stepsize))
-
         else:
-            mfilt = self._mfilt
+            mfilt = int(self._mfilt)
         # ------------------------------------------------------------ #
 
+        # ------------------------------------------------------------ #
         if mfilt < 1:
             mfilt = 1
 
@@ -334,14 +342,21 @@ def __next_write_out_data(self):
 
         shape = (mfilt, ) + self._out_data_shape
 
+        log.debug('out_data shape: %s', shape)
+        log.debug('_out_data_write: %s', self._out_data_write)
+
         for field in self._fincl:
             self._out_data[field] = np.zeros(shape, dtype=np.float64)
+
+        self._out_data_has_values = False
     # ---------------------------------------------------------------- #
 
     # ---------------------------------------------------------------- #
     # fill in out_data
     def __update_out_data(self):
 
+        self._out_data_has_values = True
+
         # ------------------------------------------------------------ #
         # Update the _out_data fields
         for field in self._fincl:
@@ -368,12 +383,15 @@ def __update_out_data(self):
             self._out_data_i = 0
         else:
             self._out_data_i += 1
+            log.debug('out_data counter is %s of %s', self._out_data_i,
+                      self._out_data_write)
     # ---------------------------------------------------------------- #
 
     # ---------------------------------------------------------------- #
     def finish(self):
         """write out_data"""
-        if self._out_data_i > 0:
+        log.debug('finishing tape %s', self._tape_num)
+        if self._out_data_has_values:
             if self._outtype == 'grid':
                 self.__write_grid()
             else:

rvic/core/log.py

@@ -78,3 +78,15 @@ def init_logger(log_dir='./', log_level='DEBUG', verbose=False):
 
     return logger
 # -------------------------------------------------------------------- #
+
+
+def close_logger():
+    """Close the handlers of the logger"""
+    log = logging.getLogger(LOG_NAME)
+    x = list(log.handlers)
+    for i in x:
+        log.removeHandler(i)
+        i.flush()
+        i.close()
+        sys.stdout = sys.__stdout__
+        sys.stderr = sys.__stderr__

rvic/core/param_file.py

@@ -64,6 +64,10 @@ def finish_params(outlets, dom_data, config_dict, directories):
         subset_length = routing['BASIN_FLOWDAYS']*SECSPERDAY/routing['OUTPUT_INTERVAL']
         log.info('Not subsetting because either SUBSET_DAYS is null or '
                  'SUBSET_DAYS<BASIN_FLOWDAYS')
+        for i, (cell_id, outlet) in enumerate(outlets.iteritems()):
+            outlets[cell_id].offset = np.zeros(outlet.unit_hydrograph.shape[1],
+                                               dtype=np.int16)
+        full_time_length = outlet.unit_hydrograph.shape[0]
     # ---------------------------------------------------------------- #
 
     # ---------------------------------------------------------------- #