Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add tests for HiPS tile <-> Numpy conversions #140

Merged
merged 3 commits into from
Jan 8, 2019
Merged

Add tests for HiPS tile <-> Numpy conversions #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7404beb
Add tests for HiPS tile <-> Numpy conversions
cdeil Jan 8, 2019
83d3174
Improve hips <-> numpy code and tests
cdeil Jan 8, 2019
baff232
Formatting of hips/tiles/tile.py
cdeil Jan 8, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 44 additions & 1 deletion hips/tiles/tests/test_tile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from pathlib import Path
import pytest
from astropy.tests.helper import remote_data
import numpy as np
from numpy.testing import assert_allclose, assert_equal
from ...utils.testing import get_hips_extra_file, requires_hips_extra
from ..tile import HipsTileMeta, HipsTile
Expand Down Expand Up @@ -116,7 +117,7 @@ def test_tile_path(self):
child_shape=(256, 256, 4),
child_ipix=[24448, 24449, 24450, 24451],
child_pix_idx=([255], [255]),
child_pix_val=[[[15, 15, 15, 255]], [[17, 17, 17, 255]], [[20, 20, 20, 255]], [[13, 13, 13, 255]]],
child_pix_val=[[[15, 15, 15, 255]], [[17, 17, 17, 255]], [[20, 20, 20, 255]], [[13, 13, 13, 255]]],
),
]

Expand Down Expand Up @@ -196,3 +197,45 @@ def test_children(self, pars):
assert tile.children[0].data.shape == pars['child_shape']
assert_equal(child_ipix, pars['child_ipix'])
assert_equal(child_data, pars['child_pix_val'])


class TestFromNumpyRoundTrip:
"""Check if HipsTile to / from Numpy roundtrips for a simple test case.

Exactly for FITS and PNG, and up to encoding noise for JPEG.
"""
def test_fits(self):
data = np.array([[0, 1], [100, 200]], dtype='uint8')
meta = HipsTileMeta(order=1, ipix=0, file_format='fits', width=2)

tile = HipsTile.from_numpy(meta, data)

assert tile.data.dtype == np.uint8
assert tile.data.shape == (2, 2)
assert_equal(tile.data, data)

def test_jpg(self):
data = np.array([[0, 1], [100, 200]], dtype='uint8')
data = np.moveaxis([data, data + 1, data + 2], 0, -1)
meta = HipsTileMeta(order=1, ipix=0, file_format='jpg', width=2)

tile = HipsTile.from_numpy(meta, data)

assert tile.data.dtype == np.uint8
assert tile.data.shape == (2, 2, 3)

# JPEG encoding noise is large. On my machine `diff = 23`
# So here we only do an approximate assert
diff = np.max(np.abs(tile.data.astype('float') - data))
assert diff < 30

def test_png(self):
data = np.array([[0, 1], [100, 200]], dtype='uint8')
data = np.moveaxis([data, data + 1, data + 2, data + 3], 0, -1)
meta = HipsTileMeta(order=1, ipix=0, file_format='png', width=2)

tile = HipsTile.from_numpy(meta, data)

assert tile.data.dtype == np.uint8
assert tile.data.shape == (2, 2, 4)
assert_equal(tile.data, data)
110 changes: 54 additions & 56 deletions hips/tiles/tile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,9 @@
from ..utils.healpix import healpix_pixel_corners
from .io import tile_default_url, tile_default_path

__all__ = [
'HipsTileMeta',
'HipsTile',
]
__all__ = ["HipsTileMeta", "HipsTile"]

__doctest_skip__ = [
'HipsTile',
'HipsTileMeta',
]
__doctest_skip__ = ["HipsTile", "HipsTileMeta"]


# TODO: this could be a dict. Would that be better?
Expand All @@ -47,14 +41,14 @@ def compute_image_shape(width: int, height: int, fmt: str) -> tuple:
shape : tuple
Numpy array shape
"""
if fmt == 'fits':
if fmt == "fits":
return height, width
elif fmt == 'jpg':
elif fmt == "jpg":
return height, width, 3
elif fmt == 'png':
elif fmt == "png":
return height, width, 4
else:
raise ValueError(f'Invalid format: {fmt}')
raise ValueError(f"Invalid format: {fmt}")


class HipsTileMeta:
Expand Down Expand Up @@ -89,8 +83,14 @@ class HipsTileMeta:
'Norder3/Dir0/Npix450.fits'
"""

def __init__(self, order: int, ipix: int, file_format: str,
frame: str = 'icrs', width: int = 512) -> None:
def __init__(
self,
order: int,
ipix: int,
file_format: str,
frame: str = "icrs",
width: int = 512,
) -> None:
self.order = order
self.ipix = ipix
self.file_format = file_format
Expand All @@ -99,20 +99,20 @@ def __init__(self, order: int, ipix: int, file_format: str,

def __repr__(self):
return (
'HipsTileMeta('
f'order={self.order}, ipix={self.ipix}, '
f'file_format={self.file_format!r}, frame={self.frame!r}, '
f'width={self.width}'
')'
"HipsTileMeta("
f"order={self.order}, ipix={self.ipix}, "
f"file_format={self.file_format!r}, frame={self.frame!r}, "
f"width={self.width}"
")"
)

def __eq__(self, other: 'HipsTileMeta') -> bool:
def __eq__(self, other: "HipsTileMeta") -> bool:
return (
self.order == other.order and
self.ipix == other.ipix and
self.file_format == other.file_format and
self.frame == other.frame and
self.width == other.width
self.order == other.order
and self.ipix == other.ipix
and self.file_format == other.file_format
and self.frame == other.frame
and self.width == other.width
)

def copy(self):
Expand Down Expand Up @@ -180,14 +180,11 @@ def __init__(self, meta: HipsTileMeta, raw_data: bytes) -> None:
self.raw_data = raw_data
self._data = None

def __eq__(self, other: 'HipsTile') -> bool:
return (
self.meta == other.meta and
self.raw_data == other.raw_data
)
def __eq__(self, other: "HipsTile") -> bool:
return self.meta == other.meta and self.raw_data == other.raw_data

@classmethod
def from_numpy(cls, meta: HipsTileMeta, data: np.ndarray) -> 'HipsTile':
def from_numpy(cls, meta: HipsTileMeta, data: np.ndarray) -> "HipsTile":
"""Create a tile from given pixel data.

Parameters
Expand All @@ -205,37 +202,39 @@ def from_numpy(cls, meta: HipsTileMeta, data: np.ndarray) -> 'HipsTile':
fmt = meta.file_format
bio = BytesIO()

if fmt == 'fits':
if fmt == "fits":
hdu = fits.PrimaryHDU(data)
hdu.writeto(bio)
elif fmt == 'jpg':
elif fmt == "jpg":
# Flip tile to be consistent with FITS orientation
data = np.flipud(data)
image = Image.fromarray(data)
image.save(bio, format='jpeg')
elif fmt == 'png':
image.save(bio, format="jpeg")
elif fmt == "png":
# Flip tile to be consistent with FITS orientation
data = np.flipud(data)
image = Image.fromarray(data)
image.save(bio, format='png')
image.save(bio, format="png")
else:
raise ValueError(f'Tile file format not supported: {fmt}. '
'Supported formats: fits, jpg, png')
raise ValueError(
f"Tile file format not supported: {fmt}. "
"Supported formats: fits, jpg, png"
)

bio.seek(0)
raw_data = bio.read()

return cls(meta, raw_data)

@property
def children(self) -> List['HipsTile']:
def children(self) -> List["HipsTile"]:
"""Create four children tiles from parent tile."""
w = self.data.shape[0] // 2
data = [
self.data[w: w * 2, 0: w],
self.data[0: w, 0: w],
self.data[w: w * 2, w: w * 2],
self.data[0: w, w: w * 2]
self.data[w : w * 2, 0:w],
self.data[0:w, 0:w],
self.data[w : w * 2, w : w * 2],
self.data[0:w, w : w * 2],
]

tiles = []
Expand All @@ -245,7 +244,7 @@ def children(self) -> List['HipsTile']:
self.meta.ipix * 4 + idx,
self.meta.file_format,
self.meta.frame,
len(data[0])
len(data[0]),
)
tile = self.from_numpy(meta, data[idx])
tiles.append(tile)
Expand All @@ -261,10 +260,7 @@ def data(self) -> np.ndarray:
See the `to_numpy` function.
"""
if self._data is None:
self._data = self.to_numpy(
self.raw_data,
self.meta.file_format,
)
self._data = self.to_numpy(self.raw_data, self.meta.file_format)

return self._data

Expand All @@ -286,29 +282,31 @@ def to_numpy(raw_data: bytes, fmt: str) -> np.ndarray:
"""
bio = BytesIO(raw_data)

if fmt == 'fits':
if fmt == "fits":
# At the moment CDS is serving FITS tiles in non-standard FITS files
# https://github.com/hipspy/hips/issues/42
# The following warnings handling is supposed to suppress these warnings
# (but hopefully still surface other issues in a useful way).
with warnings.catch_warnings():
warnings.simplefilter('ignore', AstropyWarning)
warnings.simplefilter('ignore', VerifyWarning)
warnings.simplefilter("ignore", AstropyWarning)
warnings.simplefilter("ignore", VerifyWarning)
with fits.open(bio) as hdu_list:
data = hdu_list[0].data
elif fmt in {'jpg', 'png'}:
elif fmt in {"jpg", "png"}:
with Image.open(bio) as image:
data = np.array(image)
# Flip tile to be consistent with FITS orientation
data = np.flipud(data)
else:
raise ValueError(f'Tile file format not supported: {fmt}. '
'Supported formats: fits, jpg, png')
raise ValueError(
f"Tile file format not supported: {fmt}. "
"Supported formats: fits, jpg, png"
)

return data

@classmethod
def read(cls, meta: HipsTileMeta, filename: str = None) -> 'HipsTile':
def read(cls, meta: HipsTileMeta, filename: str = None) -> "HipsTile":
"""Read HiPS tile data from a directory and load into memory (`~hips.HipsTile`).

Parameters
Expand All @@ -322,7 +320,7 @@ def read(cls, meta: HipsTileMeta, filename: str = None) -> 'HipsTile':
return cls(meta, raw_data)

@classmethod
def fetch(cls, meta: HipsTileMeta, url: str) -> 'HipsTile':
def fetch(cls, meta: HipsTileMeta, url: str) -> "HipsTile":
"""Fetch HiPS tile and load into memory (`~hips.HipsTile`).

Parameters
Expand Down