Merge branch 'master' of github.com:astropy/astroquery into vamdc

.travis.yml

@@ -1,4 +1,10 @@
-language: python
+# We set the language to c because python isn't supported on the MacOS X nodes
+# on Travis. However, the language ends up being irrelevant anyway, since we
+# install Python ourselves using conda.
+language: c
+
+os:
+    - linux
 
 # Setting sudo to false opts in to Travis-CI container-based builds.
 sudo: false
@@ -12,98 +18,98 @@ addons:
             - texlive-latex-extra
             - dvipng
 
-python:
-    - 2.7
-    - 3.4
-
 env:
     global:
         # The following versions are the 'default' for tests, unless
         # overridden underneath. They are defined here in order to save having
         # to repeat them for all configurations.
+        - PYTHON_VERSION=3.5
         - MAIN_CMD='python setup.py'
-        - NUMPY_VERSION=1.10
+        - NUMPY_VERSION=stable
         - ASTROPY_VERSION=stable
-        - CONDA_DEPENDENCIES='requests beautiful-soup matplotlib html5lib'
-        - PIP_DEPENDENCIES='keyring aplpy pyregion'
+        - CONDA_DEPENDENCIES='requests beautifulsoup4 matplotlib html5lib keyring aplpy pyregion'
+        - PIP_DEPENDENCIES=''
+        - SETUP_CMD='test'
+        - CONDA_CHANNELS='astropy-ci-extras astropy'
+        - CONDA_DEPENDENCIES_OLD='requests beautiful-soup matplotlib html5lib'
+        - PIP_DEPENDENCIES_OLD='pyregion aplpy keyring'
 
     matrix:
-        - SETUP_CMD='egg_info'
-
-        - SETUP_CMD='test'
+        - PYTHON_VERSION=2.7 SETUP_CMD='egg_info'
+        - PYTHON_VERSION=3.3 SETUP_CMD='egg_info'
+        - PYTHON_VERSION=3.4 SETUP_CMD='egg_info'
+        - PYTHON_VERSION=3.5 SETUP_CMD='egg_info'
 
 matrix:
-    include:
 
-        # Test for py 3.5 (move this up to the main matrix once beautiful-soup is in conda)
-        - python: 3.5
-          env: SETUP_CMD='egg_info'
-        - python: 3.5
+    # Don't wait for allowed failures
+    fast_finish: true
+
+    include:
+        # Try MacOS X
+        - os: osx
           env: SETUP_CMD='test'
-               CONDA_DEPENDENCIES='requests matplotlib html5lib'
-               PIP_DEPENDENCIES='keyring aplpy pyregion beautifulsoup4'
 
-        # Do a coverage test in Python 2.
-        - python: 2.7
-          env: SETUP_CMD='test --coverage'
+        # Do a coverage test in Python 2. Move coverage to 3.x once speed
+        # issues have been solved; astropy/astropy#4826
+        - os: linux
+          env: PYTHON_VERSION=2.7 SETUP_CMD='test --coverage'
 
         # Check for sphinx doc build warnings - we do this first because it
         # may run for a long time
-        - python: 2.7
-          env: SETUP_CMD='build_sphinx -w'
-
-        # Python 3.3 doesn't have numpy 1.10 in conda, revoke this commit once
-        # it's available in the astropy-ci-extras channel
-        - python: 3.3
-          env: SETUP_CMD='egg_info'
-        - python: 3.3
-          env: SETUP_CMD='test' NUMPY_VERSION=1.9
-
-        # Try Astropy development and LTS version
-        - python: 2.7
-          env: ASTROPY_VERSION=development SETUP_CMD='test'
-        - python: 3.5
-          env: ASTROPY_VERSION=development SETUP_CMD='test'
-               CONDA_DEPENDENCIES='requests matplotlib html5lib'
-               PIP_DEPENDENCIES='keyring aplpy pyregion beautifulsoup4'
-        - python: 2.7
-          env: ASTROPY_VERSION=lts SETUP_CMD='test'
-        - python: 3.5
-          env: ASTROPY_VERSION=lts SETUP_CMD='test'
-               CONDA_DEPENDENCIES='requests matplotlib html5lib'
-               PIP_DEPENDENCIES='keyring aplpy pyregion beautifulsoup4'
+        - os: linux
+          env: PYTHON_VERSION=2.7 SETUP_CMD='build_sphinx -w'
+
+        # Try all python versions and Numpy versions. Since we can assume that
+        # the Numpy developers have taken care of testing Numpy with different
+        # versions of Python, we can vary Python and Numpy versions at the same
+        # time.
+        - os: linux
+          env: PYTHON_VERSION=2.7 NUMPY_VERSION=1.7
+               CONDA_DEPENDENCIES=$CONDA_DEPENDENCIES_OLD
+               PIP_DEPENDENCIES=$PIP_DEPENDENCIES_OLD
+        - os: linux
+          env: PYTHON_VERSION=3.3 NUMPY_VERSION=1.8
+               CONDA_DEPENDENCIES=$CONDA_DEPENDENCIES_OLD
+               PIP_DEPENDENCIES=$PIP_DEPENDENCIES_OLD
+        - os: linux
+          env: PYTHON_VERSION=3.4 NUMPY_VERSION=1.9
+               CONDA_DEPENDENCIES=$CONDA_DEPENDENCIES_OLD
+               PIP_DEPENDENCIES=$PIP_DEPENDENCIES_OLD
+        - os: linux
+          env: PYTHON_VERSION=3.5 NUMPY_VERSION=1.10
+
+        # Now try Astropy dev and LTS vesions with the latest 3.x and 2.7.
+        - os: linux
+          env: PYTHON_VERSION=2.7 ASTROPY_VERSION=development
+        - os: linux
+          env: ASTROPY_VERSION=development
+        - os: linux
+          env: PYTHON_VERSION=2.7 ASTROPY_VERSION=lts
+        - os: linux
+          env: ASTROPY_VERSION=lts
 
         # Try with optional dependencies disabled
-        - python: 2.7
-          env: SETUP_CMD='test'
-               CONDA_DEPENDENCIES='requests beautiful-soup html5lib'
-               PIP_DEPENDENCIES='keyring'
-        - python: 3.5
-          env: SETUP_CMD='test'
-               CONDA_DEPENDENCIES='requests html5lib'
-               PIP_DEPENDENCIES='keyring beautifulsoup4'
-
-        # Try older numpy versions
-        - python: 2.7
-          env: NUMPY_VERSION=1.9 SETUP_CMD='test'
-        - python: 2.7
-          env: NUMPY_VERSION=1.8 SETUP_CMD='test'
-        - python: 2.7
-          env: NUMPY_VERSION=1.7 SETUP_CMD='test'
-        - python: 2.7
-          env: NUMPY_VERSION=1.6 SETUP_CMD='test'
+        - os: linux
+          env: PYTHON_VERSION=2.7
+               CONDA_DEPENDENCIES='requests beautifulsoup4 html5lib keyring'
+        - os: linux
+          env: CONDA_DEPENDENCIES='requests beautifulsoup4 html5lib keyring'
 
         # Try numpy pre-release version, this runs only when a pre-release
         # is available on pypi.
-        - python: 3.4
-          env: NUMPY_VERSION=prerelease SETUP_CMD='test'
+        - os: linux
+          env: NUMPY_VERSION=prerelease DEBUG=True
 
         # Do a PEP8 test with pycodestyle
-        - python: 2.7
-          env: MAIN_CMD='pycodestyle astroquery --count' SETUP_CMD=''
+        - os: linux
+          env: PYTHON_VERSION=2.7 MAIN_CMD='pycodestyle astroquery --count'
+               SETUP_CMD=''
 
     allow_failures:
-        - env: MAIN_CMD='pycodestyle astroquery --count' SETUP_CMD=''
+        - os: linux
+          env: PYTHON_VERSION=2.7 MAIN_CMD='pycodestyle astroquery --count'
+               SETUP_CMD=''
 
 install:
     - git clone git://github.com/astropy/ci-helpers.git

appveyor.yml

@@ -0,0 +1,37 @@
+# AppVeyor.com is a Continuous Integration service to build and run tests under
+# Windows
+
+environment:
+
+  global:
+      PYTHON: "C:\\conda"
+      MINICONDA_VERSION: "latest"
+      CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci-helpers\\appveyor\\windows_sdk.cmd"
+      PYTHON_ARCH: "64" # needs to be set for CMD_IN_ENV to succeed. If a mix
+                        # of 32 bit and 64 bit builds are needed, move this
+                        # to the matrix section.
+      CONDA_DEPENDENCIES: "requests beautifulsoup4 matplotlib html5lib keyring aplpy pyregion"
+
+  matrix:
+      - PYTHON_VERSION: "2.7"
+        ASTROPY_VERSION: "stable"
+        NUMPY_VERSION: "stable"
+
+      - PYTHON_VERSION: "3.5"
+        ASTROPY_VERSION: "stable"
+        NUMPY_VERSION: "stable"
+
+platform:
+    -x64
+
+install:
+    - "git clone git://github.com/astropy/ci-helpers.git"
+    - "powershell ci-helpers/appveyor/install-miniconda.ps1"
+    - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+    - "activate test"
+
+# Not a .NET project, we build the package in the install step instead
+build: false
+
+test_script:
+    - "%CMD_IN_ENV% python setup.py test"

astroquery/alma/core.py

@@ -25,7 +25,7 @@
 
 from ..exceptions import (RemoteServiceError, TableParseError,
                           InvalidQueryError, LoginError)
-from ..utils import commons, system_tools
+from ..utils import commons, system_tools, url_helpers
 from ..utils.process_asyncs import async_to_sync
 from ..query import QueryWithLogin
 from . import conf
@@ -262,7 +262,7 @@ def stage_data(self, uids):
 
         # Submit a request for the specific request ID identified above
         submission_url = urljoin(self.dataarchive_url,
-                                 os.path.join('rh/submission', request_id))
+                                 url_helpers.join('rh/submission', request_id))
         log.debug("Submission URL: {0}".format(submission_url))
         self._staging_log['submission_url'] = submission_url
         staging_submission = self._request('GET', submission_url, cache=True)
@@ -282,8 +282,8 @@ def stage_data(self, uids):
         has_completed = False
         while not has_completed:
             time.sleep(1)
-            summary = self._request('GET', os.path.join(data_page_url,
-                                                        'summary'),
+            summary = self._request('GET', url_helpers.join(data_page_url,
+                                                            'summary'),
                                     cache=False)
             summary.raise_for_status()
             print(".", end='')

astroquery/gama/tests/test_gama.py

@@ -19,8 +19,8 @@ def test_GAMA_find_result():
     with open(data_path(DATA_FILES["html_page"])) as f_html:
         result_page = f_html.read()
     url = gama.core.find_data_url(result_page)
-    expected_url = os.path.join("../tmp", DATA_FILES["fits_data"])
-    assert url == expected_url
+    expected_url = os.path.join("..", "tmp", DATA_FILES["fits_data"])
+    assert os.path.abspath(url) == os.path.abspath(expected_url)
 
 
 def test_GAMA_read_data():

astroquery/ibe/core.py

@@ -15,6 +15,7 @@
 
 import astropy.coordinates as coord
 from astropy.table import Table
+from astropy.extern import six
 
 from ..exceptions import InvalidQueryError
 from ..query import BaseQuery
@@ -240,15 +241,16 @@ def query_region_async(
             args['where'] = where
 
         if columns:
-            if isinstance(columns, basestring):
+            if isinstance(columns, six.string_types):
                 columns = columns.split()
             args['columns'] = ','.join(columns)
 
-        url = os.path.join(
-            self.URL, action,
-            mission or self.MISSION,
-            dataset or self.DATASET,
-            table or self.TABLE)
+        url = "{URL}{action}/{mission}/{dataset}/{table}".format(
+                URL=self.URL,
+                action=action,
+                mission=mission or self.MISSION,
+                dataset=dataset or self.DATASET,
+                table=table or self.TABLE)
 
         return self._request('GET', url, args, timeout=self.TIMEOUT)
 
@@ -378,11 +380,11 @@ def show_docs(self, mission=None, dataset=None, table=None):
             The table to be queried (if not the default table).
         """
 
-        url = os.path.join(
-            self.URL, 'docs',
-            mission or self.MISSION,
-            dataset or self.DATASET,
-            table or self.TABLE)
+        url = "{URL}docs/{mission}/{dataset}/{table}".format(
+                URL=self.URL,
+                mission=mission or self.MISSION,
+                dataset=dataset or self.DATASET,
+                table=table or self.TABLE)
 
         return webbrowser.open(url)
 
@@ -405,11 +407,11 @@ def get_columns(self, mission=None, dataset=None, table=None):
             A table containing a description of the columns
         """
 
-        url = os.path.join(
-            self.URL, 'search',
-            mission or self.MISSION,
-            dataset or self.DATASET,
-            table or self.TABLE)
+        url = "{URL}search/{mission}/{dataset}/{table}".format(
+                URL=self.URL,
+                mission=mission or self.MISSION,
+                dataset=dataset or self.DATASET,
+                table=table or self.TABLE)
 
         response = self._request(
             'GET', url, {'FORMAT': 'METADATA'}, timeout=self.TIMEOUT)

astroquery/sdss/tests/test_sdss.py

@@ -278,7 +278,7 @@ def test_query_crossid(patch_post, dr):
     xid = sdss.SDSS.query_crossid(coords_column, data_release=dr)
     data = Table.read(data_path(DATA_FILES['images_id']),
                       format='ascii.csv', comment='#')
-    assert all(xid == data)
+    compare_xid_data(xid, data)
     url_tester_crossid(dr)
 
 

astroquery/utils/commons.py

@@ -417,10 +417,13 @@ def save_fits(self, savepath, link_cache='hard'):
         if link_cache == 'hard':
             try:
                 os.link(target, savepath)
-            except (IOError, OSError) as e:
+            except (IOError, OSError, AttributeError) as e:
                 shutil.copy(target, savepath)
         elif link_cache == 'sym':
-            os.symlink(target, savepath)
+            try:
+                os.symlink(target, savepath)
+            except AttributeError:
+                raise OSError('Creating symlinks is not possible on this OS.')
         else:
             shutil.copy(target, savepath)
 

astroquery/utils/tests/test_utils.py

@@ -17,7 +17,6 @@
 import os
 from astropy.io import fits
 import astropy.utils.data as aud
-import astropy.version
 import tempfile
 
 
@@ -414,12 +413,8 @@ def test_filecontainer_save(patch_getreadablefileobj):
     ffile = commons.FileContainer(fitsfilepath, encoding='binary')
     temp_dir = tempfile.mkdtemp()
     empty_temp_file = temp_dir + os.sep + 'test_emptyfile.fits'
-    try:
-        ffile.save_fits(empty_temp_file)
-        assert os.path.exists(empty_temp_file)
-    finally:
-        os.remove(empty_temp_file)
-        os.rmdir(temp_dir)
+    ffile.save_fits(empty_temp_file)
+    assert os.path.exists(empty_temp_file)
 
 
 def test_filecontainer_get(patch_getreadablefileobj):

astroquery/utils/url_helpers.py

@@ -27,6 +27,22 @@ def urljoin_keep_path(url, path):
     splitted_url = urlsplit(url)
     return SplitResult(splitted_url.scheme,
                        splitted_url.netloc,
-                       os.path.join(splitted_url.path, path),
+                       join(splitted_url.path, path),
                        splitted_url.query,
                        splitted_url.fragment).geturl()
+
+
+def join(a, *p):
+    """Taken from python posixpath."""
+    sep = '/'
+    path = a
+    if not p:
+        path[:0] + sep
+    for b in p:
+        if b.startswith(sep):
+            path = b
+        elif not path or path.endswith(sep):
+            path += b
+        else:
+            path += sep + b
+    return path