From 7404beb69e7d656aeca0eda361c28dca19e5a445 Mon Sep 17 00:00:00 2001 From: Christoph Deil Date: Tue, 8 Jan 2019 16:00:37 +0100 Subject: [PATCH 1/3] Add tests for HiPS tile <-> Numpy conversions --- hips/tiles/tests/test_tile.py | 46 ++++++++++++++++++++++++++++++++++- hips/tiles/tile.py | 16 ++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/hips/tiles/tests/test_tile.py b/hips/tiles/tests/test_tile.py index 7b09509..1410626 100644 --- a/hips/tiles/tests/test_tile.py +++ b/hips/tiles/tests/test_tile.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest from astropy.tests.helper import remote_data +import numpy as np from numpy.testing import assert_allclose, assert_equal from ...utils.testing import get_hips_extra_file, requires_hips_extra from ..tile import HipsTileMeta, HipsTile @@ -116,7 +117,7 @@ def test_tile_path(self): child_shape=(256, 256, 4), child_ipix=[24448, 24449, 24450, 24451], child_pix_idx=([255], [255]), - child_pix_val=[[[15, 15, 15, 255]], [[17, 17, 17, 255]], [[20, 20, 20, 255]], [[13, 13, 13, 255]]], + child_pix_val=[[[15, 15, 15, 255]], [[17, 17, 17, 255]], [[20, 20, 20, 255]], [[13, 13, 13, 255]]], ), ] @@ -196,3 +197,46 @@ def test_children(self, pars): assert tile.children[0].data.shape == pars['child_shape'] assert_equal(child_ipix, pars['child_ipix']) assert_equal(child_data, pars['child_pix_val']) + + +class TestFromNumpyRoundTrip: + """Check if HipsTile.from_numpy().data == data. + """ + def test_fits(self): + data = np.array([[0, 1], [2, 3]], dtype='uint8') + meta = HipsTileMeta(order=1, ipix=0, file_format='fits', width=2) + + tile = HipsTile.from_numpy(meta, data) + + assert tile.data.dtype == np.uint8 + assert tile.data.shape == (2, 2) + assert_allclose(tile.data, data) + + # FIXME: This one is currently failing. + # Why is the pixel order not correct? Why is it OK for PNG, but not for JPG? + # E x: array([1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4], dtype=uint8) + # E y: array([0, 1, 2, 1, 2, 3, 2, 3, 4, 3, 4, 5], dtype=uint8) + # Looks like we never tested this, the TestHipsTile.test_write above + # only asserts that tile.raw_data round-trips, but I see no assert on + # the data in tile.data Numpy array. + def test_jpg(self): + data = np.array([[0, 1], [2, 3]], dtype='uint8') + data = np.moveaxis([data, data + 1, data + 2], 0, -1) + meta = HipsTileMeta(order=1, ipix=0, file_format='jpg', width=2) + + tile = HipsTile.from_numpy(meta, data) + + assert tile.data.dtype == np.uint8 + assert tile.data.shape == (2, 2, 3) + assert_allclose(tile.data, data) + + def test_png(self): + data = np.array([[0, 1], [2, 3]], dtype='uint8') + data = np.moveaxis([data, data + 1, data + 2, data + 3], 0, -1) + meta = HipsTileMeta(order=1, ipix=0, file_format='png', width=2) + + tile = HipsTile.from_numpy(meta, data) + + assert tile.data.dtype == np.uint8 + assert tile.data.shape == (2, 2, 4) + assert_allclose(tile.data, data) diff --git a/hips/tiles/tile.py b/hips/tiles/tile.py index 177144d..dfd3086 100644 --- a/hips/tiles/tile.py +++ b/hips/tiles/tile.py @@ -202,18 +202,34 @@ def from_numpy(cls, meta: HipsTileMeta, data: np.ndarray) -> 'HipsTile': tile : `~hips.HipsTile` HiPS tile object in the format requested in ``meta``. """ + data = np.asarray(data) fmt = meta.file_format bio = BytesIO() if fmt == 'fits': + if data.ndim != 2 or data.shape != (meta.width, meta.width): + raise ValueError( + f"Invalid data.shape: {data.shape}." + " Must be (meta.width, meta.width)." + ) hdu = fits.PrimaryHDU(data) hdu.writeto(bio) elif fmt == 'jpg': + if data.ndim != 3 or data.shape != (meta.width, meta.width, 3): + raise ValueError( + f"Invalid data.shape: {data.shape}." + " Must be (meta.width, meta.width, 3)." + ) # Flip tile to be consistent with FITS orientation data = np.flipud(data) image = Image.fromarray(data) image.save(bio, format='jpeg') elif fmt == 'png': + if data.ndim != 3 or data.shape != (meta.width, meta.width, 4): + raise ValueError( + f"Invalid data.shape: {data.shape}." + " Must be (meta.width, meta.width, 4)." + ) # Flip tile to be consistent with FITS orientation data = np.flipud(data) image = Image.fromarray(data) From 83d317444218ffbadf9c6b403b4a27d5f8a0723c Mon Sep 17 00:00:00 2001 From: Christoph Deil Date: Tue, 8 Jan 2019 17:55:18 +0100 Subject: [PATCH 2/3] Improve hips <-> numpy code and tests --- hips/tiles/tests/test_tile.py | 29 ++++++++++++++--------------- hips/tiles/tile.py | 15 --------------- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/hips/tiles/tests/test_tile.py b/hips/tiles/tests/test_tile.py index 1410626..2af0472 100644 --- a/hips/tiles/tests/test_tile.py +++ b/hips/tiles/tests/test_tile.py @@ -200,27 +200,22 @@ def test_children(self, pars): class TestFromNumpyRoundTrip: - """Check if HipsTile.from_numpy().data == data. + """Check if HipsTile to / from Numpy roundtrips for a simple test case. + + Exactly for FITS and PNG, and up to encoding noise for JPEG. """ def test_fits(self): - data = np.array([[0, 1], [2, 3]], dtype='uint8') + data = np.array([[0, 1], [100, 200]], dtype='uint8') meta = HipsTileMeta(order=1, ipix=0, file_format='fits', width=2) tile = HipsTile.from_numpy(meta, data) assert tile.data.dtype == np.uint8 assert tile.data.shape == (2, 2) - assert_allclose(tile.data, data) - - # FIXME: This one is currently failing. - # Why is the pixel order not correct? Why is it OK for PNG, but not for JPG? - # E x: array([1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4], dtype=uint8) - # E y: array([0, 1, 2, 1, 2, 3, 2, 3, 4, 3, 4, 5], dtype=uint8) - # Looks like we never tested this, the TestHipsTile.test_write above - # only asserts that tile.raw_data round-trips, but I see no assert on - # the data in tile.data Numpy array. + assert_equal(tile.data, data) + def test_jpg(self): - data = np.array([[0, 1], [2, 3]], dtype='uint8') + data = np.array([[0, 1], [100, 200]], dtype='uint8') data = np.moveaxis([data, data + 1, data + 2], 0, -1) meta = HipsTileMeta(order=1, ipix=0, file_format='jpg', width=2) @@ -228,10 +223,14 @@ def test_jpg(self): assert tile.data.dtype == np.uint8 assert tile.data.shape == (2, 2, 3) - assert_allclose(tile.data, data) + + # JPEG encoding noise is large. On my machine `diff = 23` + # So here we only do an approximate assert + diff = np.max(np.abs(tile.data.astype('float') - data)) + assert diff < 30 def test_png(self): - data = np.array([[0, 1], [2, 3]], dtype='uint8') + data = np.array([[0, 1], [100, 200]], dtype='uint8') data = np.moveaxis([data, data + 1, data + 2, data + 3], 0, -1) meta = HipsTileMeta(order=1, ipix=0, file_format='png', width=2) @@ -239,4 +238,4 @@ def test_png(self): assert tile.data.dtype == np.uint8 assert tile.data.shape == (2, 2, 4) - assert_allclose(tile.data, data) + assert_equal(tile.data, data) diff --git a/hips/tiles/tile.py b/hips/tiles/tile.py index dfd3086..f4d44b5 100644 --- a/hips/tiles/tile.py +++ b/hips/tiles/tile.py @@ -207,29 +207,14 @@ def from_numpy(cls, meta: HipsTileMeta, data: np.ndarray) -> 'HipsTile': bio = BytesIO() if fmt == 'fits': - if data.ndim != 2 or data.shape != (meta.width, meta.width): - raise ValueError( - f"Invalid data.shape: {data.shape}." - " Must be (meta.width, meta.width)." - ) hdu = fits.PrimaryHDU(data) hdu.writeto(bio) elif fmt == 'jpg': - if data.ndim != 3 or data.shape != (meta.width, meta.width, 3): - raise ValueError( - f"Invalid data.shape: {data.shape}." - " Must be (meta.width, meta.width, 3)." - ) # Flip tile to be consistent with FITS orientation data = np.flipud(data) image = Image.fromarray(data) image.save(bio, format='jpeg') elif fmt == 'png': - if data.ndim != 3 or data.shape != (meta.width, meta.width, 4): - raise ValueError( - f"Invalid data.shape: {data.shape}." - " Must be (meta.width, meta.width, 4)." - ) # Flip tile to be consistent with FITS orientation data = np.flipud(data) image = Image.fromarray(data) From baff232971e617d4f343fa4b17568ada14dfda51 Mon Sep 17 00:00:00 2001 From: Christoph Deil Date: Tue, 8 Jan 2019 18:31:34 +0100 Subject: [PATCH 3/3] Formatting of hips/tiles/tile.py --- hips/tiles/tile.py | 111 ++++++++++++++++++++++----------------------- 1 file changed, 54 insertions(+), 57 deletions(-) diff --git a/hips/tiles/tile.py b/hips/tiles/tile.py index f4d44b5..1fb362c 100644 --- a/hips/tiles/tile.py +++ b/hips/tiles/tile.py @@ -14,15 +14,9 @@ from ..utils.healpix import healpix_pixel_corners from .io import tile_default_url, tile_default_path -__all__ = [ - 'HipsTileMeta', - 'HipsTile', -] +__all__ = ["HipsTileMeta", "HipsTile"] -__doctest_skip__ = [ - 'HipsTile', - 'HipsTileMeta', -] +__doctest_skip__ = ["HipsTile", "HipsTileMeta"] # TODO: this could be a dict. Would that be better? @@ -47,14 +41,14 @@ def compute_image_shape(width: int, height: int, fmt: str) -> tuple: shape : tuple Numpy array shape """ - if fmt == 'fits': + if fmt == "fits": return height, width - elif fmt == 'jpg': + elif fmt == "jpg": return height, width, 3 - elif fmt == 'png': + elif fmt == "png": return height, width, 4 else: - raise ValueError(f'Invalid format: {fmt}') + raise ValueError(f"Invalid format: {fmt}") class HipsTileMeta: @@ -89,8 +83,14 @@ class HipsTileMeta: 'Norder3/Dir0/Npix450.fits' """ - def __init__(self, order: int, ipix: int, file_format: str, - frame: str = 'icrs', width: int = 512) -> None: + def __init__( + self, + order: int, + ipix: int, + file_format: str, + frame: str = "icrs", + width: int = 512, + ) -> None: self.order = order self.ipix = ipix self.file_format = file_format @@ -99,20 +99,20 @@ def __init__(self, order: int, ipix: int, file_format: str, def __repr__(self): return ( - 'HipsTileMeta(' - f'order={self.order}, ipix={self.ipix}, ' - f'file_format={self.file_format!r}, frame={self.frame!r}, ' - f'width={self.width}' - ')' + "HipsTileMeta(" + f"order={self.order}, ipix={self.ipix}, " + f"file_format={self.file_format!r}, frame={self.frame!r}, " + f"width={self.width}" + ")" ) - def __eq__(self, other: 'HipsTileMeta') -> bool: + def __eq__(self, other: "HipsTileMeta") -> bool: return ( - self.order == other.order and - self.ipix == other.ipix and - self.file_format == other.file_format and - self.frame == other.frame and - self.width == other.width + self.order == other.order + and self.ipix == other.ipix + and self.file_format == other.file_format + and self.frame == other.frame + and self.width == other.width ) def copy(self): @@ -180,14 +180,11 @@ def __init__(self, meta: HipsTileMeta, raw_data: bytes) -> None: self.raw_data = raw_data self._data = None - def __eq__(self, other: 'HipsTile') -> bool: - return ( - self.meta == other.meta and - self.raw_data == other.raw_data - ) + def __eq__(self, other: "HipsTile") -> bool: + return self.meta == other.meta and self.raw_data == other.raw_data @classmethod - def from_numpy(cls, meta: HipsTileMeta, data: np.ndarray) -> 'HipsTile': + def from_numpy(cls, meta: HipsTileMeta, data: np.ndarray) -> "HipsTile": """Create a tile from given pixel data. Parameters @@ -202,26 +199,27 @@ def from_numpy(cls, meta: HipsTileMeta, data: np.ndarray) -> 'HipsTile': tile : `~hips.HipsTile` HiPS tile object in the format requested in ``meta``. """ - data = np.asarray(data) fmt = meta.file_format bio = BytesIO() - if fmt == 'fits': + if fmt == "fits": hdu = fits.PrimaryHDU(data) hdu.writeto(bio) - elif fmt == 'jpg': + elif fmt == "jpg": # Flip tile to be consistent with FITS orientation data = np.flipud(data) image = Image.fromarray(data) - image.save(bio, format='jpeg') - elif fmt == 'png': + image.save(bio, format="jpeg") + elif fmt == "png": # Flip tile to be consistent with FITS orientation data = np.flipud(data) image = Image.fromarray(data) - image.save(bio, format='png') + image.save(bio, format="png") else: - raise ValueError(f'Tile file format not supported: {fmt}. ' - 'Supported formats: fits, jpg, png') + raise ValueError( + f"Tile file format not supported: {fmt}. " + "Supported formats: fits, jpg, png" + ) bio.seek(0) raw_data = bio.read() @@ -229,14 +227,14 @@ def from_numpy(cls, meta: HipsTileMeta, data: np.ndarray) -> 'HipsTile': return cls(meta, raw_data) @property - def children(self) -> List['HipsTile']: + def children(self) -> List["HipsTile"]: """Create four children tiles from parent tile.""" w = self.data.shape[0] // 2 data = [ - self.data[w: w * 2, 0: w], - self.data[0: w, 0: w], - self.data[w: w * 2, w: w * 2], - self.data[0: w, w: w * 2] + self.data[w : w * 2, 0:w], + self.data[0:w, 0:w], + self.data[w : w * 2, w : w * 2], + self.data[0:w, w : w * 2], ] tiles = [] @@ -246,7 +244,7 @@ def children(self) -> List['HipsTile']: self.meta.ipix * 4 + idx, self.meta.file_format, self.meta.frame, - len(data[0]) + len(data[0]), ) tile = self.from_numpy(meta, data[idx]) tiles.append(tile) @@ -262,10 +260,7 @@ def data(self) -> np.ndarray: See the `to_numpy` function. """ if self._data is None: - self._data = self.to_numpy( - self.raw_data, - self.meta.file_format, - ) + self._data = self.to_numpy(self.raw_data, self.meta.file_format) return self._data @@ -287,29 +282,31 @@ def to_numpy(raw_data: bytes, fmt: str) -> np.ndarray: """ bio = BytesIO(raw_data) - if fmt == 'fits': + if fmt == "fits": # At the moment CDS is serving FITS tiles in non-standard FITS files # https://github.com/hipspy/hips/issues/42 # The following warnings handling is supposed to suppress these warnings # (but hopefully still surface other issues in a useful way). with warnings.catch_warnings(): - warnings.simplefilter('ignore', AstropyWarning) - warnings.simplefilter('ignore', VerifyWarning) + warnings.simplefilter("ignore", AstropyWarning) + warnings.simplefilter("ignore", VerifyWarning) with fits.open(bio) as hdu_list: data = hdu_list[0].data - elif fmt in {'jpg', 'png'}: + elif fmt in {"jpg", "png"}: with Image.open(bio) as image: data = np.array(image) # Flip tile to be consistent with FITS orientation data = np.flipud(data) else: - raise ValueError(f'Tile file format not supported: {fmt}. ' - 'Supported formats: fits, jpg, png') + raise ValueError( + f"Tile file format not supported: {fmt}. " + "Supported formats: fits, jpg, png" + ) return data @classmethod - def read(cls, meta: HipsTileMeta, filename: str = None) -> 'HipsTile': + def read(cls, meta: HipsTileMeta, filename: str = None) -> "HipsTile": """Read HiPS tile data from a directory and load into memory (`~hips.HipsTile`). Parameters @@ -323,7 +320,7 @@ def read(cls, meta: HipsTileMeta, filename: str = None) -> 'HipsTile': return cls(meta, raw_data) @classmethod - def fetch(cls, meta: HipsTileMeta, url: str) -> 'HipsTile': + def fetch(cls, meta: HipsTileMeta, url: str) -> "HipsTile": """Fetch HiPS tile and load into memory (`~hips.HipsTile`). Parameters