Merge branch 'release_candidate' for release 0.3.0 into master

.travis.yml

@@ -2,8 +2,7 @@ cache: apt
 sudo: true
 language: python
 python:
-  - "2.7"
-  - "3.3"
+  - 2.7
 before_install:
   - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; # grab miniconda
   - bash miniconda.sh -b -p $HOME/miniconda # install miniconda
@@ -13,12 +12,21 @@ before_install:
   - conda install pyyaml
   - conda info -a # and print the info
 
-install:
-  - conda env create --file environment.yml #set up the environment thats in the file
-  - source activate test-environment # activate it
-  - python setup.py install # install ptypy
+env:
+  - TEST_ENV_NAME=core_dependencies
+  - TEST_ENV_NAME=full_dependencies
+
+install: True
 
 script:
+  - conda env create --file ${TEST_ENV_NAME}.yml #set up the environment thats in the file
+  - source activate ${TEST_ENV_NAME} # activate it
+  - conda install pytest # additional dependencies for the tests
+  - pip install pytest-cov
+  - pip install coveralls
+  - echo $PYTHONPATH
+  - conda list
+  - python setup.py install # install ptypy
   - py.test ptypy/test -v --cov ptypy --cov-report term-missing # now run the tests
 
 after_script:

benchmark/model_speed.py

@@ -0,0 +1,16 @@
+
+from ptypy import utils as u
+from ptypy.core import Ptycho, Vanilla
+
+p = u.Param()
+p.verbose_level = 3
+p.io = u.Param()
+p.io.home = "/tmp/ptypy/"
+p.scans = u.Param()
+p.scans.MF = u.Param()
+p.scans.MF.data= u.Param()
+p.scans.MF.name = 'Vanilla'
+p.scans.MF.data.name = 'QuickScan'
+p.scans.MF.data.num_frames = 50000
+p.scans.MF.data.shape = 32
+Ptycho(p,level=2)

core_dependencies.yml

@@ -0,0 +1,8 @@
+name: core_dependencies
+channels:
+  - conda-forge
+dependencies:
+  - python=2.7
+  - numpy
+  - scipy
+  - h5py

doc/Makefile

@@ -3,8 +3,13 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = ~/.local/bin/sphinx-build
+
+ifeq ($(SPHINXBUILD),"")
+    SPHINXBUILD   = ~/.local/bin/sphinx-build
+endif
+
 #SPHINXBUILD   = /usr/bin/sphinx-build
+
 PAPER         =
 BUILDDIR      = build
 

doc/conf.py

@@ -21,11 +21,11 @@
 #sys.path.insert(0, os.path.abspath('.'))
 
 # generate paramters.rst and other rst
-execfile('parameters2rst.py')
-execfile('tmp2rst.py')
-execfile('version.py')
 import subprocess
-subprocess.call(['python', 'script2rst.py'])  # We need this to have a clean sys.argv
+subprocess.check_call(['python', 'script2rst.py'])  # We need this to have a clean sys.argv
+subprocess.check_call(['python','parameters2rst.py'])
+subprocess.check_call(['python','tmp2rst.py'])
+execfile('version.py')
 
 # -- General configuration ------------------------------------------------
 
@@ -42,73 +42,63 @@
 ]
 
 
+def truncate_docstring(app, what, name, obj, options, lines):
+    """
+    Remove the Default parameter entries.
+    """
+    if not hasattr(obj, 'DEFAULT'):
+        return
+    if any(l.strip().startswith('Defaults:') for l in lines):
+        while True:
+            if lines.pop(-1).strip().startswith('Defaults:'):
+                break
+
+
 def remove_mod_docstring(app, what, name, obj, options, lines):
     from ptypy import utils as u
-    import numpy as np
+    from ptypy import defaults_tree
     u.verbose.report.headernewline='\n\n'
     searchstr = ':py:data:'
 
-    def get_refs(dct,pd,depth=2, indent = ''):
-        if depth<0:
+    def get_refs(dct, pd, depth=2, indent=''):
+        if depth < 0:
             return
 
         for k, value in dct.iteritems():
             ref = ', see :py:data:`~%s`' % pd.children[k].entry_point if pd.children.has_key(k) else ''
-            if hasattr(value,'items'):
+            if hasattr(value, 'items'):
                 v = str(value.__class__.__name__)
-            elif str(value)==value:
-                v='"%s"' % value
+            elif str(value) == value:
+                v = '"%s"' % value
             else:
-                v=str(value)
+                v = str(value)
 
-            lines.append(indent+'* *' +k+'* = ``'+v+'``' +ref)#+'\n')
+            lines.append(indent + '* *' + k + '* = ``' + v + '``' + ref)
 
-            if hasattr(value,'items'):
-                #lines.append('\n\n')
+            if hasattr(value, 'items'):
                 lines.append("")
-                get_refs(value,pd.children[k],depth=depth-1, indent = indent+'  ')
+                get_refs(value, pd.children[k], depth=depth-1, indent=indent+'  ')
                 lines.append("")
-                #lines.append('\n\n')
 
-    #if name.find('DEFAULT')>=0:
     if isinstance(obj, u.Param) or isinstance(obj, dict):
-        keys = obj.keys()
         pd = None
 
-        """
-        # auto_matching
-        for entry,pdesc in u.validator.entry_points_Param.iteritems():
-            chkeys = ':'.join([k.split('.')[-1] for k in pdesc.children.keys()])
-            #print chkeys
-            #print keys
-            matches = [key in chkeys for key in keys]
-            #print matches
-            print np.mean(matches)
-            if np.mean(matches)>0.8:
-                print 'Param match'
-                e=entry
-                print e
-                pd = pdesc
-                break
-        """
         for l in lines:
             start = l.find(searchstr)
             if start > -1:
                 newstr = l[start:]
                 newstr = newstr.split('`')[1]
                 newstr = newstr.replace('~', '')
-                #print newstr, what, name, options
-                pd = u.validator.entry_points_dct.get(newstr,None)
+                pd = defaults_tree.get(newstr)
                 break
 
         if pd is not None:
-            #lines.append('Match with :py:data:`.%s` \n\n' %pd.entry_point)
             get_refs(obj, pd, depth=2, indent='')
-            #print lines
 
 
 def setup(app):
     app.connect('autodoc-process-docstring', remove_mod_docstring)
+    app.connect('autodoc-process-docstring', truncate_docstring)
 
 
 napoleon_use_ivar = True

doc/html_templates/ptypysphinx/download.html

@@ -32,5 +32,5 @@
 <ul class="simple">
 <li>I13, <a href="https://www.diamond.ac.uk">Diamond</a></li>
 <li>ID16, <a href="https://www.esrf.eu">ESRF</a></li>
-<li>NanoScopium, <a href="https://www.synchrotron-soleil.fr">Soleil</a></li>
+<!--<li>NanoScopium, <a href="https://www.synchrotron-soleil.fr">Soleil</a></li>-->
 </ul>

doc/index.rst

@@ -44,7 +44,7 @@ Highlights
   overcoming partial coherence or related phenomena.
 
 * **On-the-fly** reconstructions (while data is being acquired) using the
-  the :any:`PtyScan` class in the linking mode :ref:`linking mode<case_flyscan>` 
+  the :py:class:`PtyScan` class in the linking mode :ref:`linking mode<case_flyscan>` 
 
 
 Quicklinks
@@ -68,10 +68,10 @@ Quicklinks
 
 .. rubric:: Footnotes
 
-.. [#Enders2016] B.Enders and P.Thibault, **Proc. R. Soc. A** 472, 20160640 (2016), `doi <http://dx.doi.org/10.1098/rspa.2016.0640>`_
+.. [#Enders2016] B.Enders and P.Thibault, **Proc. R. Soc. A** 472, 20160640 (2016), `doi <http://dx.doi.org/10.1098/rspa.2016.0640>`__
 
-.. [#Thi2013] P.Thibault and A.Menzel, **Nature** 494, 68 (2013), `doi <http://dx.doi.org/10.1038/nature11806>`_
+.. [#Thi2013] P.Thibault and A.Menzel, **Nature** 494, 68 (2013), `doi <http://dx.doi.org/10.1038/nature11806>`__
 
-.. [#ml] P.Thibault and M.Guizar-Sicairos, **New J. of Phys.** 14, 6 (2012), `doi <http://dx.doi.org/10.1126/science.1158573>`_
+.. [#ml] P.Thibault and M.Guizar-Sicairos, **New J. of Phys.** 14, 6 (2012), `doi <http://dx.doi.org/10.1126/science.1158573>`__
 
-.. [#dm] P.Thibault, M.Dierolf *et al.*, **New J. of Phys. 14**, 6 (2012), `doi <http://dx.doi.org/10.1088/1367-2630/14/6/063004>`_
+.. [#dm] P.Thibault, M.Dierolf *et al.*, **New J. of Phys. 14**, 6 (2012), `doi <http://dx.doi.org/10.1088/1367-2630/14/6/063004>`__

doc/parameters2rst.py

@@ -1,5 +1,5 @@
 
-from ptypy import utils as u
+from ptypy import defaults_tree
 
 prst = open('rst/parameters.rst','w')
 
@@ -9,32 +9,52 @@
 Header+= '************************\n\n'
 prst.write(Header)
 
-names = u.validator.parameter_descriptions
-for name,desc in u.validator.parameter_descriptions.iteritems():
-    if name=='':
+for path, desc in defaults_tree.descendants:
+
+    types = desc.type
+    default = desc.default
+    lowlim, uplim = desc.limits
+    is_wildcard = (desc.name == '*')
+
+    if is_wildcard:
+        path = path.replace('*', desc.parent.name[:-1] + '_00')
+
+    if path == '':
         continue
-    if hasattr(desc,'children') and desc.parent is u.validator.pdroot:
-        prst.write('\n'+name+'\n')
-        prst.write('='*len(name)+'\n\n')
-    if hasattr(desc,'children') and desc.parent.parent is u.validator.pdroot:
-        prst.write('\n'+name+'\n')
-        prst.write('-'*len(name)+'\n\n')
-
-    prst.write('.. py:data:: '+name)
-    prst.write('('+', '.join([t for t in desc.type])+')')
+    if desc.children and desc.parent is desc.root:
+        prst.write('\n'+path+'\n')
+        prst.write('='*len(path)+'\n\n')
+    if desc.children and desc.parent.parent is desc.root:
+        prst.write('\n'+path+'\n')
+        prst.write('-'*len(path)+'\n\n')
+
+    prst.write('.. py:data:: '+path)
+
+    if desc.is_symlink:
+        tp = 'Param'
+    else:
+        tp = ', '.join([str(t) for t in types])
+    prst.write(' ('+tp+')')
     prst.write('\n\n')
-    num = str(desc.ID) if hasattr(desc,'ID') else "None"
-    prst.write('   *('+num+')* '+desc.shortdoc+'\n\n')
-    prst.write('   '+desc.longdoc.replace('\n','\n   ')+'\n\n')
-    prst.write('   *default* = ``'+str(desc.default))
-    if desc.lowlim is not None and desc.uplim is not None:
-        prst.write(' (>'+str(desc.lowlim)+', <'+str(desc.uplim)+')``\n')
-    elif desc.lowlim is not None and desc.uplim is None:
-        prst.write(' (>'+str(desc.lowlim)+')``\n')
-    elif desc.lowlim is None and desc.uplim is not None:
-        prst.write(' (<'+str(desc.uplim)+')``\n')
+
+    if is_wildcard:
+        prst.write('   *Wildcard*: multiple entries with arbitrary names are accepted.\n\n')
+
+    prst.write('   '+desc.help+'\n\n')
+    prst.write('   '+desc.doc.replace('<newline>','\n').replace('\n', '\n   ')+'\n\n')
+
+    if desc.is_symlink:
+        prst.write('   *default* = '+':py:data:`'+desc.path+'`\n')
     else:
-        prst.write('``\n')
+        prst.write('   *default* = ``'+repr(default))
+        if lowlim is not None and uplim is not None:
+            prst.write(' (>'+str(lowlim)+', <'+str(uplim)+')``\n')
+        elif lowlim is not None and uplim is None:
+            prst.write(' (>'+str(lowlim)+')``\n')
+        elif lowlim is None and uplim is not None:
+            prst.write(' (<'+str(uplim)+')``\n')
+        else:
+            prst.write('``\n')
 
     prst.write('\n')
 prst.close()

doc/rst/ptypy.core.rst

@@ -44,14 +44,6 @@ ptypy.core.manager module
     :undoc-members:
     :show-inheritance:
 
-ptypy.core.model module
------------------------
-
-.. automodule:: ptypy.core.model
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 ptypy.core.paths module
 -----------------------
 

doc/rst/ptypy.debug.rst

@@ -0,0 +1,29 @@
+ptypy.debug package
+===================
+
+Submodules
+----------
+
+ptypy.debug.embedded_shell module
+---------------------------------
+
+.. automodule:: ptypy.debug.embedded_shell
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+ptypy.debug.ipython_kernel module
+---------------------------------
+
+.. automodule:: ptypy.debug.ipython_kernel
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ptypy.debug
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/rst/ptypy.rst

@@ -13,6 +13,7 @@ Subpackages
     ptypy.resources
     ptypy.simulations
     ptypy.utils
+    ptypy.debug
 
 Module contents
 ---------------

doc/rst/ptypy.utils.rst

@@ -12,14 +12,6 @@ ptypy.utils.array_utils module
     :undoc-members:
     :show-inheritance:
 
-ptypy.utils.embedded_shell module
----------------------------------
-
-.. automodule:: ptypy.utils.embedded_shell
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 ptypy.utils.math_utils module
 -----------------------------
 
@@ -76,10 +68,10 @@ ptypy.utils.scripts module
     :undoc-members:
     :show-inheritance:
 
-ptypy.utils.validator module
-----------------------------
+ptypy.utils.descriptor module
+-----------------------------
 
-.. automodule:: ptypy.utils.validator
+.. automodule:: ptypy.utils.descriptor
     :members:
     :undoc-members:
     :show-inheritance: