From 805b8220acdf502275e543e3ac182918a5ccbf53 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 18:29:09 -0400 Subject: [PATCH 01/12] Updates to python3 and updates setup script --- .gitignore | 4 ++++ .travis.yml | 34 ++++++++++++++++++++++++++++++++++ setup.py | 30 +++++++++++++++++++----------- stile/file_io.py | 2 +- stile/hsc/base_tasks.py | 5 ++--- stile/stile_utils.py | 8 ++++---- stile/sys_tests.py | 18 +++++++++--------- stile/treecorr_utils.py | 4 ++-- 8 files changed, 75 insertions(+), 30 deletions(-) create mode 100644 .travis.yml diff --git a/.gitignore b/.gitignore index d342bea..d8598c7 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ nosetests.xml doc/_build doc/html doc/latex + +.ropeproject/ +.eggs/ +*.bak diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..0f57130 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,34 @@ +language: python + +matrix: + include: + - os: linux + python: 2.7 + sudo: required + env: TOXENV=py27 + - os: linux + sudo: required + python: 3.6 + env: TOXENV=py36 + - os: osx + language: generic + env: TOXENV=py27 + - os: osx + language: generic + env: TOXENV=py36 + +script: +- python setup.py install +- nosetests tests/test_binning.py tests/test_stile_utils.py + +after_success: +- coveralls + +deploy: + provider: pypi + user: msimet + password: + secure: secure_token + on: + tags: true + branch: master diff --git a/setup.py b/setup.py index 701bca7..9357c3c 100755 --- a/setup.py +++ b/setup.py @@ -1,23 +1,31 @@ #!/usr/bin/env python +from setuptools import setup -from distutils.core import setup -try: - import treecorr -except ImportError: - import warnings - warnings.warn("treecorr package cannot be imported. Installation will proceed, but you may "+ - "wish to install it if you would like to use the correlation functions within "+ - "Stile.") - +# read the contents of the README file +with open('README.md', encoding="utf-8") as f: + long_description = f.read() setup(name='Stile', version='0.1', description='Stile: Systematics Tests in Lensing pipeline', author='The Stile team', - requirements=['numpy'], + install_requires=['numpy', 'treecorr', 'astropy', 'matplotlib'], author_email='melanie.simet@gmail.com', url='https://github.com/msimet/Stile', packages=['stile', 'stile.hsc'], scripts=['bin/StileVisit.py', 'bin/StileVisitNoTract.py', 'bin/StileCCD.py', - 'bin/StileCCDNoTract.py', 'bin/StilePatch.py', 'bin/StileTract.py'] + 'bin/StileCCDNoTract.py', 'bin/StilePatch.py', 'bin/StileTract.py'], + test_suite='nose.collector', + tests_require=['nose'], + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Scientific/Engineering :: Physics' + ] ) diff --git a/stile/file_io.py b/stile/file_io.py index f2efd55..178d2cc 100644 --- a/stile/file_io.py +++ b/stile/file_io.py @@ -13,7 +13,7 @@ has_fits = False import numpy import os -import stile_utils +from . import stile_utils def ReadFITSImage(file_name, hdu=0): diff --git a/stile/hsc/base_tasks.py b/stile/hsc/base_tasks.py index 413c067..ee06258 100644 --- a/stile/hsc/base_tasks.py +++ b/stile/hsc/base_tasks.py @@ -337,8 +337,7 @@ def generateColumns(self, dataRef, catalog, mask_tuple, raw_cols, extra_col_dict # offset for (x,y) if extra_col_dict has a column 'CCD'. Currently getMm() returns values # in pixel. When the pipeline is updated, we should update this line as well. - if dataRef.dataId.has_key('ccd') and extra_col_dict.has_key( - 'CCD') and ('x' in raw_cols or 'y' in raw_cols): + if 'ccd' in dataRef.dataId and 'CCD' in extra_col_dict and ('x' in raw_cols or 'y' in raw_cols): xy0 = cameraGeomUtils.findCcd(dataRef.getButler().mapper.camera, cameraGeom.Id( dataRef.dataId.get('ccd')) ).getPositionFromPixel(afwGeom.PointD(0., 0.)).getMm() @@ -832,7 +831,7 @@ def run(self, visit, dataRefList): catalogs.append(dataRef.get(self.catalog_type, immediate=True, flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)) except RuntimeError as e: - print e, ', skip this patch' + print(e, ', skip this patch') catalogs = [self.removeFlaggedObjects(catalog) for catalog in catalogs] sys_data_list = [] extra_col_dicts = [{} for catalog in catalogs] diff --git a/stile/stile_utils.py b/stile/stile_utils.py index ae66143..98af2e1 100644 --- a/stile/stile_utils.py +++ b/stile/stile_utils.py @@ -10,7 +10,7 @@ def Parser(): """ Returns an argparse Parser object with input args used by Stile and TreeCorr. """ - import treecorr_utils + from . import treecorr_utils import argparse p = argparse.Parser(parent=treecorr_utils.Parser()) #TODO: add, obviously, EVERYTHING ELSE @@ -111,7 +111,7 @@ def __init__(self, simple_stats): self.simple_stats = simple_stats for stat in self.simple_stats: init_str = 'self.' + stat + '=None' - exec init_str + exec(init_str) self.percentiles = None self.values = None @@ -127,11 +127,11 @@ def __str__(self): # Loop over simple statistics and print them, if not None. Generically if one is None then # all will be, so just check one. test_str = "test_val = self."+("%s"%self.simple_stats[0]) - exec test_str + exec(test_str) if test_val is not None: for stat in self.simple_stats: this_string = 'this_val = self.'+stat - exec this_string + exec(this_string) ret_str += '\t%s: %f\n'%(stat, this_val) ret_str += '\n' diff --git a/stile/sys_tests.py b/stile/sys_tests.py index 077306b..3a8bfcb 100644 --- a/stile/sys_tests.py +++ b/stile/sys_tests.py @@ -31,7 +31,7 @@ import numpy import stile -import stile_utils +from . import stile_utils try: import treecorr from treecorr.corr2 import corr2_valid_params @@ -445,24 +445,24 @@ def getCF(self, correlation_function_type, data, data2=None, 'data and random, and random2 if data2.') elif correlation_function_type in ['gg', 'm2', 'kk']: if random or random2: - print "Warning: randoms ignored for this correlation function type" + print("Warning: randoms ignored for this correlation function type") elif correlation_function_type in ['ng', 'nm', 'nk']: if data2 is None: raise ValueError('Must include data2 for this correlation function type') if random2 is not None: - print "Warning: random2 ignored for this correlation function type" + print("Warning: random2 ignored for this correlation function type") elif correlation_function_type == 'norm': if data2 is None: raise ValueError('Must include data2 for this correlation function type') if random is None: raise ValueError('Must include random for this correlation function type') if random2 is None: - print "Warning: random2 ignored for this correlation function type" + print("Warning: random2 ignored for this correlation function type") elif correlation_function_type == 'kg': if data2 is None: raise ValueError('Must include data2 for this correlation function type') if random is not None or random2 is not None: - print "Warning: randoms ignored for this correlation function type" + print("Warning: randoms ignored for this correlation function type") data = self.makeCatalog(data, config=treecorr_kwargs, use_as_k=use_as_k, use_chip_coords=use_chip_coords) @@ -1104,9 +1104,9 @@ def __call__(self, array, percentiles=None, field=None, verbose=False, ignore_ba # It's a catalog, not a simple array if use_field is None: raise RuntimeError('StatSysTest called on a catalog without specifying a field!') - if use_field not in use_array.dtype.fields.keys(): + if use_field not in list(use_array.dtype.fields.keys()): raise RuntimeError('Field %s is not in this catalog, which contains %s!'% - (use_field, use_array.dtype.fields.keys())) + (use_field, list(use_array.dtype.fields.keys()))) # Select the appropriate field for this catalog. use_array = use_array[use_field] # Now take care of case (b): @@ -1171,7 +1171,7 @@ def __call__(self, array, percentiles=None, field=None, verbose=False, ignore_ba # Print, if verbose=True. if verbose: - print result.__str__() + print(result.__str__()) # Return. return result @@ -1685,7 +1685,7 @@ def HistoPlot(self, data_list, field=None, binning_style=None, nbins=None, elif style_use is 'manual': bins = nbins else: - print "Unrecognized code for binning style, use default instead!" + print("Unrecognized code for binning style, use default instead!") bins = nbins if weights is True: diff --git a/stile/treecorr_utils.py b/stile/treecorr_utils.py index 4fc33ba..a1aa9d8 100644 --- a/stile/treecorr_utils.py +++ b/stile/treecorr_utils.py @@ -3,7 +3,7 @@ program. """ import numpy -import file_io +from . import file_io import treecorr from treecorr.corr2 import corr2_valid_params @@ -90,7 +90,7 @@ def ReadTreeCorrResultsFile(file_name): :param file_name: The location of an output file from TreeCorr. :returns: A numpy array corresponding to the data in ``file_name``. """ - import stile_utils + from . import stile_utils output = file_io.ReadASCIITable(file_name, comments='#') if not len(output): From 5daecf6c725124663d6084361d23aa03d1c772b3 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 18:30:53 -0400 Subject: [PATCH 02/12] changelog update --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70c2db8..d07c664 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ +7/11/18: Update code to python3 3/17/16: Update documentation to Sphinx standard and add documentation build files (issue #17) -2/17/16: Changes to correlation function plots & documentation (issue #77) +2/17/16: Changes to correlation function plots & documentation (issue #77) 2/10/15: Add a setup.py installer (issue #57) 8/26/14: Change from relying on compiled C-code corr2 to Python package TreeCorr (issue #33) From 909ed89784ddd86bc0e363a281dd16aa337366ad Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 18:32:38 -0400 Subject: [PATCH 03/12] Fixes readme import --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9357c3c..34a0c4f 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python from setuptools import setup +from io import open # read the contents of the README file with open('README.md', encoding="utf-8") as f: From 6ce4f4e665707a1eff8d2ac698e88f743e676f5a Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 18:38:18 -0400 Subject: [PATCH 04/12] Switching to pyfits in the dependencies --- .travis.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f57130..f5babfb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ matrix: env: TOXENV=py36 script: -- python setup.py install +- python setup.py install --user - nosetests tests/test_binning.py tests/test_stile_utils.py after_success: diff --git a/setup.py b/setup.py index 34a0c4f..3ab08d6 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ version='0.1', description='Stile: Systematics Tests in Lensing pipeline', author='The Stile team', - install_requires=['numpy', 'treecorr', 'astropy', 'matplotlib'], + install_requires=['numpy', 'treecorr', 'pyfits', 'matplotlib'], author_email='melanie.simet@gmail.com', url='https://github.com/msimet/Stile', packages=['stile', 'stile.hsc'], From 88dccb3a1e7d8f720467326eca8613194a7b88d5 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 18:40:26 -0400 Subject: [PATCH 05/12] change install dir --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f5babfb..0f57130 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ matrix: env: TOXENV=py36 script: -- python setup.py install --user +- python setup.py install - nosetests tests/test_binning.py tests/test_stile_utils.py after_success: From 3b0e56ee142aa4f42960e4060503d539a3a5fdea Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 18:47:33 -0400 Subject: [PATCH 06/12] Simplified compile script --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f57130..f1ea26e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,6 @@ matrix: sudo: required python: 3.6 env: TOXENV=py36 - - os: osx - language: generic - env: TOXENV=py27 - - os: osx - language: generic - env: TOXENV=py36 script: - python setup.py install From 99d313b6132d9125ac137f8a14662c3a3221235f Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 18:50:46 -0400 Subject: [PATCH 07/12] Simplified build matrix --- .travis.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index f1ea26e..af20708 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,14 @@ language: python -matrix: - include: - - os: linux - python: 2.7 - sudo: required - env: TOXENV=py27 - - os: linux - sudo: required - python: 3.6 - env: TOXENV=py36 +python: + - "2.7" + - "3.6" + +install: + - pip install . script: -- python setup.py install -- nosetests tests/test_binning.py tests/test_stile_utils.py + - nosetests tests/test_binning.py tests/test_stile_utils.py after_success: - coveralls From f3bbb2389041da0211f0650578c34cde64a54369 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 18:56:39 -0400 Subject: [PATCH 08/12] switch to astropy with version 2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3ab08d6..04f96bf 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ version='0.1', description='Stile: Systematics Tests in Lensing pipeline', author='The Stile team', - install_requires=['numpy', 'treecorr', 'pyfits', 'matplotlib'], + install_requires=['numpy', 'treecorr', 'pyfits', 'matplotlib', 'astropy<3'], author_email='melanie.simet@gmail.com', url='https://github.com/msimet/Stile', packages=['stile', 'stile.hsc'], From e1a9abbe6a6f684dd5db088bd6255d6d9653bafd Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 19:01:43 -0400 Subject: [PATCH 09/12] Updates Readme with cool badge --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6edf3f9..ecc3fb7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Stile +Stile [![Build Status](https://travis-ci.org/EiffL/Stile.svg?branch=master)](https://travis-ci.org/EiffL/Stile) ===== Stile: the Systematics Tests In LEnsing pipeline @@ -7,7 +7,7 @@ Stile: the Systematics Tests In LEnsing pipeline #### Installation instructions #### Stile is a pure python package, no compilation needed. You can install it using: -> python setup.py install +> pip install stile #### Dependencies #### To run Stile, you must have: @@ -17,17 +17,17 @@ To run Stile, you must have: We also recommend: - - [TreeCorr](http://github.com/rmjarvis/TreeCorr), Mike Jarvis's 2-point correlation function code. All of our correlation function tests involve calls to this package. + - [TreeCorr](http://github.com/rmjarvis/TreeCorr), Mike Jarvis's 2-point correlation function code. All of our correlation function tests involve calls to this package. - PyFITS/Astropy to handle FITS tables and images. Stile can run on ASCII tables, but is much slower. - matplotlib to generate plots. -More dependencies may be added in the future. - +Note that these dependencies will be automatically installed by pip. More dependencies may be added in the future. + ------------------------------------- #### Documentation #### The documentation is available online at http://stile.readthedocs.io/. You can also build it using Sphinx in the `doc/` directory. - + ------------------------------------- #### Current functionality #### @@ -41,7 +41,7 @@ Right now, Stile can: - Generate histograms. - Perform any of the above tests for data with different binning schemes applied, using a simple method of specifying the bins. - Interface with sufficiently recent versions of the HSC pipeline. - + ------------------------------------- #### Wishlist #### @@ -52,4 +52,3 @@ Over the upcoming months, we plan to add: - Automatic drivers to run as many tests as your data set allows. - A larger suite of example code. - Tests on images, and the utilities to make those tests easier. - From 9f805bf8adb9d633bb1064555cbcb8a1bebfbee4 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 19:02:40 -0400 Subject: [PATCH 10/12] forgot to remove pyfits --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 04f96bf..8f48618 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ version='0.1', description='Stile: Systematics Tests in Lensing pipeline', author='The Stile team', - install_requires=['numpy', 'treecorr', 'pyfits', 'matplotlib', 'astropy<3'], + install_requires=['numpy', 'treecorr', 'matplotlib', 'astropy<3'], author_email='melanie.simet@gmail.com', url='https://github.com/msimet/Stile', packages=['stile', 'stile.hsc'], From a941957b73605012da08e6bf14cad5557964bdff Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 19:32:53 -0400 Subject: [PATCH 11/12] switch to codecov --- .travis.yml | 6 ++++-- README.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index af20708..6a57e1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,12 @@ install: - pip install . script: - - nosetests tests/test_binning.py tests/test_stile_utils.py + - cd tests + - nosetests --with-coverage --cover-package=stile --with-doctest --cover-erase tests/test_binning.py tests/test_stile_utils.py after_success: -- coveralls + - coverage xml + - codecov deploy: provider: pypi diff --git a/README.md b/README.md index ecc3fb7..9dda06d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Stile [![Build Status](https://travis-ci.org/EiffL/Stile.svg?branch=master)](https://travis-ci.org/EiffL/Stile) +Stile [![Build Status](https://travis-ci.org/msimet/Stile.svg?branch=master)](https://travis-ci.org/msimet/Stile) ===== Stile: the Systematics Tests In LEnsing pipeline From 9d5bab3ccb47d3fb1b8fc98db3c0e03605ff0169 Mon Sep 17 00:00:00 2001 From: Francois Date: Wed, 11 Jul 2018 19:36:53 -0400 Subject: [PATCH 12/12] oups --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6a57e1c..4cd91b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,12 @@ python: - "3.6" install: + - pip install coverage - pip install . script: - cd tests - - nosetests --with-coverage --cover-package=stile --with-doctest --cover-erase tests/test_binning.py tests/test_stile_utils.py + - nosetests --with-coverage --cover-package=stile --with-doctest --cover-erase test_binning.py test_stile_utils.py after_success: - coverage xml