From 68f756f31d51bdee6c6de5356785437ef3077ae1 Mon Sep 17 00:00:00 2001 From: Casper van der Wel Date: Thu, 20 Jul 2017 14:34:32 +0200 Subject: [PATCH 01/14] WIP: Rebase tiffstack_tifffile on new PimsFormat baseclass --- pims/api.py | 25 ++- pims/base_frames.py | 271 +++++++++++++++++++++++++ pims/tests/data/stuck_metadata_py2.pkl | Bin 144 -> 0 bytes pims/tests/data/stuck_metadata_py3.pkl | Bin 144 -> 0 bytes pims/tests/test_common.py | 14 +- pims/tiff_stack.py | 123 +++++------ 6 files changed, 353 insertions(+), 80 deletions(-) delete mode 100644 pims/tests/data/stuck_metadata_py2.pkl delete mode 100644 pims/tests/data/stuck_metadata_py3.pkl diff --git a/pims/api.py b/pims/api.py index c6c02dab..549f5050 100644 --- a/pims/api.py +++ b/pims/api.py @@ -7,6 +7,7 @@ from pims.display import (export, play, scrollable_stack, to_rgb, normalize, plot_to_frame, plots_to_frame) from itertools import chain +from functools import wraps import six import glob @@ -18,8 +19,8 @@ from pims.image_reader import ImageReader, ImageReaderND # noqa from .cine import Cine # noqa from .norpix_reader import NorpixSeq # noqa -from pims.tiff_stack import TiffStack_tifffile # noqa from .spe_stack import SpeStack +from imageio import formats, get_reader def not_available(requirement): @@ -28,6 +29,15 @@ def raiser(*args, **kwargs): "This reader requires {0}.".format(requirement)) return raiser + +def wrap_fmt(reader, name, description, extensions=None, modes=None): + formats.add_format(reader(name, description, extensions, modes)) + @wraps(reader) + def wrapper(filename, **kwargs): + return get_reader(filename, name, **kwargs) + return wrapper + + if export is None: export = not_available("PyAV or MoviePy") @@ -75,12 +85,18 @@ def raiser(*args, **kwargs): Video = not_available("PyAV, MoviePy, or ImageIO") import pims.tiff_stack -from pims.tiff_stack import (TiffStack_pil, TiffStack_libtiff, - TiffStack_tifffile) +from pims.tiff_stack import TiffStack_pil, TiffStack_libtiff, \ + FormatTiffStack_tifffile # First, check if each individual class is available # and drop in placeholders as needed. if not pims.tiff_stack.tifffile_available(): - TiffStack_tiffile = not_available("tifffile") + TiffStack_tifffile = not_available("tifffile") +else: + TiffStack_tifffile = wrap_fmt(FormatTiffStack_tifffile, + 'TIFF_tifffile', + 'Reads TIFF files through tifffile.py.', + 'tif tiff lsm stk', + 'iIvV') if not pims.tiff_stack.libtiff_available(): TiffStack_libtiff = not_available("libtiff") if not pims.tiff_stack.PIL_available(): @@ -97,6 +113,7 @@ def raiser(*args, **kwargs): TiffStack = not_available("tifffile, libtiff, or PIL/Pillow") + try: import pims.bioformats if pims.bioformats.available(): diff --git a/pims/base_frames.py b/pims/base_frames.py index 6962a078..2ee26f1a 100644 --- a/pims/base_frames.py +++ b/pims/base_frames.py @@ -8,6 +8,7 @@ from .frame import Frame from abc import ABCMeta, abstractmethod, abstractproperty from warnings import warn +from imageio.core import Format class FramesStream(with_metaclass(ABCMeta, object)): @@ -606,3 +607,273 @@ def __repr__(self): s += "Axis '{0}' size: {1}\n".format(dim, self._sizes[dim]) s += """Pixel Datatype: {dtype}""".format(dtype=self.pixel_type) return s + + +def guess_axes(image): + shape = image.shape + ndim = len(shape) + + if ndim == 2: + return 'yx' + elif ndim == 3 and shape[2] in [3, 4]: + return 'yxc' + elif ndim == 3: + return 'zyx' + elif ndim == 4 and shape[3] in [3, 4]: + return 'zyxc' + else: + raise ValueError("Cannot interpret dimensions for an image of " + "shape {0}".format(shape)) + + +def default_axes(sizes, mode): + if mode == 'i': # single image + if sizes.get('z', 1) == 1 and sizes.get('t', 1) == 1: + bundle_axes = 'yx' + iter_axes = '' + else: + raise ValueError("This file cannot be opened as single image.") + elif mode == 'I': # multiple images + if sizes.get('z', 1) == 1 and 't' in sizes: + bundle_axes = 'yx' + iter_axes = 't' + else: + raise ValueError("This file cannot be opened as multiple images.") + elif mode == 'v': # single volume + if 'z' in sizes and sizes.get('t', 1) == 1: + bundle_axes = 'zyx' + iter_axes = '' + else: + raise ValueError("This file cannot be opened as single volume.") + elif mode == 'v': # single volume + if 'z' in sizes and 't' in sizes: + bundle_axes = 'zyx' + iter_axes = 't' + else: + raise ValueError("This file cannot be opened as multiple volumes.") + else: + if 'z' in sizes: + bundle_axes = 'zyx' + else: + bundle_axes = 'yx' + if 't' in sizes: + iter_axes = 't' + else: + iter_axes = '' + + if 'c' in sizes: + bundle_axes += 'c' + + return bundle_axes, iter_axes + + +class PimsFormat(Format): + def _can_read(self, request): + if request.mode[1] in (self.modes + '?'): + if request.filename.lower().endswith(self.extensions): + return True + + def _can_write(self, request): + return False + + @Slicerator.from_class + class Reader(with_metaclass(ABCMeta, Format.Reader)): + def __init__(self, *args, **kwargs): + self._clear_axes() + self._get_frame_dict = dict() + Format.Reader.__init__(self, *args, **kwargs) + + def __getitem__(self, key): + return self.get_data(key) + + def _register_get_frame(self, method, axes): + axes = tuple([a for a in axes]) + if not hasattr(self, '_get_frame_dict'): + warn( + "Please call FramesSequenceND.__init__() at the start of the" + "the reader initialization.") + self._get_frame_dict = dict() + self._get_frame_dict[axes] = method + + def _clear_axes(self): + self._sizes = {} + self._default_coords = DefaultCoordsDict() + self._iter_axes = [] + self._bundle_axes = ['y', 'x'] + self._get_frame_wrapped = None + + def _init_axis(self, name, size, default=0): + # check if the axes have been initialized, if not, do it here + if not hasattr(self, '_sizes'): + warn( + "Please call FramesSequenceND.__init__() at the start of the" + "the reader initialization.") + self._clear_axes() + self._get_frame_dict = dict() + if name in self._sizes: + raise ValueError("axis '{}' already exists".format(name)) + self._sizes[name] = int(size) + self.default_coords.axes = self.axes + self.default_coords[name] = int(default) + + def _get_length(self): + return int(np.prod([self._sizes[d] for d in self._iter_axes])) + + @property + def dtype(self): + return self._get_dtype() + + @property + def shape(self): + iter_shape = tuple([self._sizes[d] for d in self._iter_axes]) + return iter_shape + self.frame_shape + + @property + def frame_shape(self): + """ Returns the shape of the frame as returned by get_frame. """ + return tuple([self._sizes[d] for d in self._bundle_axes]) + + @property + def axes(self): + """ Returns a list of all axes. """ + return [k for k in self._sizes] + + @property + def ndim(self): + """ Returns the number of axes. """ + return len(self._sizes) + + @property + def sizes(self): + """ Returns a dict of all axis sizes. """ + return self._sizes + + @property + def bundle_axes(self): + """ This determines which axes will be bundled into one Frame. + The ndarray that is returned by get_frame has the same axis order + as the order of `bundle_axes`. + """ + return self._bundle_axes[:] # return a copy + + @bundle_axes.setter + def bundle_axes(self, value): + value = list(value) + invalid = [k for k in value if k not in self._sizes] + if invalid: + raise ValueError("axes %r do not exist" % invalid) + + for k in value: + if k in self._iter_axes: + del self._iter_axes[self._iter_axes.index(k)] + + self._bundle_axes = value + if not hasattr(self, '_get_frame_dict'): + warn( + "Please call FramesSequenceND.__init__() at the start of the" + "the reader initialization.") + self._get_frame_dict = dict() + if len(self._get_frame_dict) == 0: + if hasattr(self, 'get_frame_2D'): + # include get_frame_2D for backwards compatibility + self._register_get_frame(self.get_frame_2D, 'yx') + else: + raise RuntimeError( + 'No reader methods found. Register a reader ' + 'method with _register_get_frame') + + # update the get_frame method + get_frame = _make_get_frame(self._bundle_axes, + self._get_frame_dict, + self.sizes, self.dtype) + self._get_frame_wrapped = get_frame + + @property + def iter_axes(self): + """ This determines which axes will be iterated over by the + FramesSequence. The last element will iterate fastest. """ + return self._iter_axes[:] # return a copy + + @iter_axes.setter + def iter_axes(self, value): + value = list(value) + invalid = [k for k in value if k not in self._sizes] + if invalid: + raise ValueError("axes %r do not exist" % invalid) + + for k in value: + if k in self._bundle_axes: + del self._bundle_axes[self._bundle_axes.index(k)] + + self._iter_axes = value + + @property + def default_coords(self): + """ When a axis is not present in both iter_axes and bundle_axes, the + coordinate contained in this dictionary will be used. """ + return self._default_coords # this is a custom dict (DefaultCoordsDict) + + @default_coords.setter + def default_coords(self, value): + self._default_coords.update(**value) + + def get_data(self, index, **kwargs): + """ get_data(index, **kwargs) + + Read image data from the file, using the image index. The + returned image has a 'meta' attribute with the meta data. + + Some formats may support additional keyword arguments. These are + listed in the documentation of those formats. + """ + self._checkClosed() + self._BaseReaderWriter_last_index = index + return self._get_data(index, **kwargs) + + def iter_data(self): + return iter(self[:]) + + def _get_data(self, i): + """ Returns a Frame of shape determined by bundle_axes. The index value + is interpreted according to the iter_axes property. Coordinates not + present in both iter_axes and bundle_axes will be set to their default + value (see default_coords). """ + if i > len(self): + raise IndexError('index out of range') + if self._get_frame_wrapped is None: + self.bundle_axes = tuple(self.bundle_axes) # kick bundle_axes + + # start with the default coordinates + coords = self.default_coords.copy() + + # list sizes of iteration axes + iter_sizes = [self._sizes[k] for k in self.iter_axes] + # list how much i has to increase to get an increase of coordinate n + iter_cumsizes = np.append(np.cumprod(iter_sizes[::-1])[-2::-1], 1) + # calculate the coordinates and update the coords dictionary + iter_coords = (i // iter_cumsizes) % iter_sizes + coords.update( + **{k: v for k, v in zip(self.iter_axes, iter_coords)}) + + result = self._get_frame_wrapped(**coords) + if hasattr(result, 'metadata'): + metadata = result.metadata + else: + metadata = dict() + + metadata_axes = set(self.axes) - set(self.bundle_axes) + metadata_coords = {ax: coords[ax] for ax in metadata_axes} + metadata.update( + dict(axes=self.bundle_axes, coords=metadata_coords)) + return Frame(result, frame_no=i, metadata=metadata) + + def _get_meta_data(self, i): + return self.get_data(i).metadata + + @abstractmethod + def _open(self, **kwargs): + pass + + @abstractmethod + def _get_dtype(self): + pass diff --git a/pims/tests/data/stuck_metadata_py2.pkl b/pims/tests/data/stuck_metadata_py2.pkl deleted file mode 100644 index b170f9eea2ed2717cf732be282bf855e0ea9e6cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 144 zcmZo*sx4&Dh~QvgU~ow+Ne#)&O)X?hP60AYfJ`npqmU_-i~T;M5Wk`!P-!7^YavTe zA!~#HkmH$~n4apATAW;zSx}OhpI68hAqW&M&PXgswbd}yF*P&-Va-BzurL>^ux24s z1P4$&I6tkVJh3RXkU4@I$SuiCOH0elN!2T;EMx&Ia7ipl4av++Eo4nj0WwQ~OfER1 bkjfb0K?cAxBUlXQ>_lD|aVh diff --git a/pims/tests/test_common.py b/pims/tests/test_common.py index 44612dc7..c8c3e8f3 100644 --- a/pims/tests/test_common.py +++ b/pims/tests/test_common.py @@ -14,6 +14,8 @@ from numpy.testing import (assert_equal, assert_allclose) from nose.tools import assert_true import pims +import imageio +from datetime import datetime path, _ = os.path.split(os.path.abspath(__file__)) path = os.path.join(path, 'data') @@ -476,13 +478,11 @@ def tearDown(self): class _tiff_image_series(_image_series): def test_metadata(self): m = self.v[0].metadata - if sys.version_info.major < 3: - pkl_path = os.path.join(path, 'stuck_metadata_py2.pkl') - else: - pkl_path = os.path.join(path, 'stuck_metadata_py3.pkl') - with open(pkl_path, 'rb') as p: - d = pickle.load(p) - assert_equal(m, d) + expected = {'Software': 'tifffile.py', + 'DateTime': datetime(2015, 1, 18, 15, 33, 49), + 'axes': ['y', 'x'], 'coords': {'t': 0}, + 'ImageDescription': 'shape=(5,512,512)'} + assert_equal(m, expected) class TestTiffStack_libtiff(_tiff_image_series, unittest.TestCase): diff --git a/pims/tiff_stack.py b/pims/tiff_stack.py index abbded15..2ad0939b 100644 --- a/pims/tiff_stack.py +++ b/pims/tiff_stack.py @@ -40,7 +40,7 @@ def tifffile_available(): return tifffile is not None -from pims.base_frames import FramesSequence +from pims.base_frames import FramesSequence, PimsFormat, guess_axes, default_axes _dtype_map = {4: np.uint8, 8: np.uint8, @@ -53,7 +53,7 @@ def _tiff_datetime(dt_str): minute=int(dt_str[14:16]), second=int(dt_str[17:19])) -class TiffStack_tifffile(FramesSequence): +class FormatTiffStack_tifffile(PimsFormat): """Read TIFF stacks (single files containing many images) into an iterable object that returns images as numpy arrays. @@ -97,73 +97,58 @@ class TiffStack_tifffile(FramesSequence): -------- TiffStack_pil, TiffStack_libtiff, ImageSequence """ - @classmethod - def class_exts(cls): - # TODO extend this set to match reality - return {'tif', 'tiff', 'lsm', - 'stk'} | super(TiffStack_tifffile, cls).class_exts() - - def __init__(self, filename): - self._filename = filename - record = tifffile.TiffFile(filename).series[0] - if hasattr(record, 'pages'): - self._tiff = record.pages - else: - self._tiff = record['pages'] - - tmp = self._tiff[0] - self._dtype = tmp.dtype - self._im_sz = tmp.shape - - def get_frame(self, j): - t = self._tiff[j] - data = t.asarray() - return Frame(data, frame_no=j, metadata=self._read_metadata(t)) - - def _read_metadata(self, tiff): - """Read metadata for current frame and return as dict""" - md = {} - try: - md["ImageDescription"] = ( - tiff.tags["image_description"].value.decode()) - except: - pass - try: - dt = tiff.tags["datetime"].value.decode() - md["DateTime"] = _tiff_datetime(dt) - except: - pass - try: - md["Software"] = tiff.tags["software"].value.decode() - except: - pass - try: - md["DocumentName"] = tiff.tags["document_name"].value.decode() - except: - pass - return md - - @property - def pixel_type(self): - return self._dtype - - @property - def frame_shape(self): - return self._im_sz - - def __len__(self): - return len(self._tiff) - - def __repr__(self): - # May be overwritten by subclasses - return """ -Source: {filename} -Length: {count} frames -Frame Shape: {frame_shape!r} -Pixel Datatype: {dtype}""".format(frame_shape=self.frame_shape, - count=len(self), - filename=self._filename, - dtype=self.pixel_type) + class Reader(PimsFormat.Reader): + def _open(self, **kwargs): + handle = self.request.get_file() + record = tifffile.TiffFile(handle, **kwargs).series[0] + if hasattr(record, 'pages'): + self._tiff = record.pages + else: + self._tiff = record['pages'] + + tmp = self._tiff[0] + self._dtype = tmp.dtype + + axes = guess_axes(tmp) + for name, size in zip(axes, tmp.shape): + self._init_axis(name, size) + self._init_axis('t', len(self._tiff)) + + self._register_get_frame(self._get_frame, axes) + + self.bundle_axes, self.iter_axes = default_axes(self.sizes, + self.request.mode) + + def _get_dtype(self): + return self._dtype + + def _get_frame(self, **inds): + t = self._tiff[inds['t']] + return Frame(t.asarray(), frame_no=0, + metadata=self._read_metadata(t)) + + def _read_metadata(self, tiff): + """Read metadata for current frame and return as dict""" + md = {} + try: + md["ImageDescription"] = ( + tiff.tags["image_description"].value.decode()) + except: + pass + try: + dt = tiff.tags["datetime"].value.decode() + md["DateTime"] = _tiff_datetime(dt) + except: + pass + try: + md["Software"] = tiff.tags["software"].value.decode() + except: + pass + try: + md["DocumentName"] = tiff.tags["document_name"].value.decode() + except: + pass + return md class TiffStack_libtiff(FramesSequence): From 434974d8b606efa788f5b388557fc6cacc13ad75 Mon Sep 17 00:00:00 2001 From: Casper van der Wel Date: Thu, 20 Jul 2017 14:46:45 +0200 Subject: [PATCH 02/14] WIP: Split ND and non-ND baseclasses --- pims/base_frames.py | 177 +++++++++++++++++++++++++++----------------- 1 file changed, 108 insertions(+), 69 deletions(-) diff --git a/pims/base_frames.py b/pims/base_frames.py index 2ee26f1a..1d8db94d 100644 --- a/pims/base_frames.py +++ b/pims/base_frames.py @@ -667,7 +667,7 @@ def default_axes(sizes, mode): return bundle_axes, iter_axes -class PimsFormat(Format): +class _PimsFormat(Format): def _can_read(self, request): if request.mode[1] in (self.modes + '?'): if request.filename.lower().endswith(self.extensions): @@ -678,14 +678,71 @@ def _can_write(self, request): @Slicerator.from_class class Reader(with_metaclass(ABCMeta, Format.Reader)): + def __getitem__(self, key): + return self.get_data(key) + + @abstractmethod + def _get_length(self): + pass + + @abstractmethod + def _open(self, **kwargs): + pass + + @abstractmethod + def _get_dtype(self): + pass + + @property + def dtype(self): + return self._get_dtype() + + @property + def shape(self): + return (len(self),) + self.frame_shape + + @abstractproperty + def frame_shape(self): + """ Returns the shape of the frame as returned by get_frame. """ + pass + + @property + def ndim(self): + """ Returns the number of axes. """ + return len(self.frame_shape) + 1 + + def get_data(self, index, **kwargs): + """ get_data(index, **kwargs) + + Read image data from the file, using the image index. The + returned image has a 'meta' attribute with the meta data. + + Some formats may support additional keyword arguments. These are + listed in the documentation of those formats. + """ + self._checkClosed() + self._BaseReaderWriter_last_index = index + im, meta = self._get_data(index, **kwargs) + return Frame(im, frame_no=index, metadata=meta) + + def iter_data(self): + return iter(self[:]) + + @abstractmethod + def _get_data(self, i): + pass + + def _get_meta_data(self, i): + return self.get_data(i).metadata + + +class PimsFormat(_PimsFormat): + class Reader(_PimsFormat.Reader): def __init__(self, *args, **kwargs): self._clear_axes() self._get_frame_dict = dict() Format.Reader.__init__(self, *args, **kwargs) - def __getitem__(self, key): - return self.get_data(key) - def _register_get_frame(self, method, axes): axes = tuple([a for a in axes]) if not hasattr(self, '_get_frame_dict'): @@ -716,13 +773,56 @@ def _init_axis(self, name, size, default=0): self.default_coords.axes = self.axes self.default_coords[name] = int(default) + def get_data(self, index, **kwargs): + """ get_data(index, **kwargs) + + Read image data from the file, using the image index. The + returned image has a 'meta' attribute with the meta data. + + Some formats may support additional keyword arguments. These are + listed in the documentation of those formats. + """ + self._checkClosed() + self._BaseReaderWriter_last_index = index + return self._get_data(index, **kwargs) + + def _get_data(self, i): + """ Returns a Frame of shape determined by bundle_axes. The index value + is interpreted according to the iter_axes property. Coordinates not + present in both iter_axes and bundle_axes will be set to their default + value (see default_coords). """ + if i > len(self): + raise IndexError('index out of range') + if self._get_frame_wrapped is None: + self.bundle_axes = tuple(self.bundle_axes) # kick bundle_axes + + # start with the default coordinates + coords = self.default_coords.copy() + + # list sizes of iteration axes + iter_sizes = [self._sizes[k] for k in self.iter_axes] + # list how much i has to increase to get an increase of coordinate n + iter_cumsizes = np.append(np.cumprod(iter_sizes[::-1])[-2::-1], 1) + # calculate the coordinates and update the coords dictionary + iter_coords = (i // iter_cumsizes) % iter_sizes + coords.update( + **{k: v for k, v in zip(self.iter_axes, iter_coords)}) + + result = self._get_frame_wrapped(**coords) + if hasattr(result, 'metadata'): + metadata = result.metadata + else: + metadata = dict() + + metadata_axes = set(self.axes) - set(self.bundle_axes) + metadata_coords = {ax: coords[ax] for ax in metadata_axes} + metadata.update( + dict(axes=self.bundle_axes, coords=metadata_coords)) + return Frame(result, frame_no=i, metadata=metadata) + def _get_length(self): return int(np.prod([self._sizes[d] for d in self._iter_axes])) - @property - def dtype(self): - return self._get_dtype() - @property def shape(self): iter_shape = tuple([self._sizes[d] for d in self._iter_axes]) @@ -816,64 +916,3 @@ def default_coords(self): @default_coords.setter def default_coords(self, value): self._default_coords.update(**value) - - def get_data(self, index, **kwargs): - """ get_data(index, **kwargs) - - Read image data from the file, using the image index. The - returned image has a 'meta' attribute with the meta data. - - Some formats may support additional keyword arguments. These are - listed in the documentation of those formats. - """ - self._checkClosed() - self._BaseReaderWriter_last_index = index - return self._get_data(index, **kwargs) - - def iter_data(self): - return iter(self[:]) - - def _get_data(self, i): - """ Returns a Frame of shape determined by bundle_axes. The index value - is interpreted according to the iter_axes property. Coordinates not - present in both iter_axes and bundle_axes will be set to their default - value (see default_coords). """ - if i > len(self): - raise IndexError('index out of range') - if self._get_frame_wrapped is None: - self.bundle_axes = tuple(self.bundle_axes) # kick bundle_axes - - # start with the default coordinates - coords = self.default_coords.copy() - - # list sizes of iteration axes - iter_sizes = [self._sizes[k] for k in self.iter_axes] - # list how much i has to increase to get an increase of coordinate n - iter_cumsizes = np.append(np.cumprod(iter_sizes[::-1])[-2::-1], 1) - # calculate the coordinates and update the coords dictionary - iter_coords = (i // iter_cumsizes) % iter_sizes - coords.update( - **{k: v for k, v in zip(self.iter_axes, iter_coords)}) - - result = self._get_frame_wrapped(**coords) - if hasattr(result, 'metadata'): - metadata = result.metadata - else: - metadata = dict() - - metadata_axes = set(self.axes) - set(self.bundle_axes) - metadata_coords = {ax: coords[ax] for ax in metadata_axes} - metadata.update( - dict(axes=self.bundle_axes, coords=metadata_coords)) - return Frame(result, frame_no=i, metadata=metadata) - - def _get_meta_data(self, i): - return self.get_data(i).metadata - - @abstractmethod - def _open(self, **kwargs): - pass - - @abstractmethod - def _get_dtype(self): - pass From f5053b07eab45b2b5145396a93ca68a5e6bd206e Mon Sep 17 00:00:00 2001 From: Casper van der Wel Date: Thu, 20 Jul 2017 17:40:28 +0200 Subject: [PATCH 03/14] Add imageio as required dep --- .travis.yml | 4 ++-- setup.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e0f289f7..0ff045c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,11 @@ sudo: false matrix: include: - python: "2.7" - env: DEPS="numpy=1.8* slicerator tifffile" BUILD_DOCS=false + env: DEPS="numpy=1.8* slicerator tifffile" DEPSPIP="imageio" BUILD_DOCS=false - python: "2.7" env: DEPS="numpy slicerator scipy pillow matplotlib scikit-image jinja2 av libtiff tifffile jpype1" DEPSPIP="moviepy imageio" BUILD_DOCS=false - python: "3.6" - env: DEPS="numpy slicerator tifffile" BUILD_DOCS=false + env: DEPS="numpy slicerator tifffile" DEPSPIP="imageio" BUILD_DOCS=false - python: "3.6" env: DEPS="numpy slicerator scipy pillow matplotlib scikit-image jinja2 av tifffile libtiff jpype1 ipython sphinx sphinx_rtd_theme numpydoc" DEPSPIP="moviepy imageio" BUILD_DOCS=true diff --git a/setup.py b/setup.py index 6402ab0d..ebe0f0ed 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,8 @@ cmdclass=versioneer.get_cmdclass(), description="Python Image Sequence", author="PIMS Contributors", - install_requires=['slicerator>=0.9.7', 'six>=1.8', 'numpy>=1.7'], + install_requires=['slicerator>=0.9.7', 'six>=1.8', 'numpy>=1.7', + 'imageio'], author_email="dallan@pha.jhu.edu", url="https://github.com/soft-matter/pims", packages=['pims', From 1ff0e5f92383af4852d4b4655fe6e8c29537a9d2 Mon Sep 17 00:00:00 2001 From: Casper van der Wel Date: Thu, 20 Jul 2017 18:05:19 +0200 Subject: [PATCH 04/14] DOC: Dcstrings --- pims/api.py | 13 ++++---- pims/base_frames.py | 79 ++++++++++++++++++--------------------------- pims/tiff_stack.py | 4 +-- 3 files changed, 40 insertions(+), 56 deletions(-) diff --git a/pims/api.py b/pims/api.py index 549f5050..01031009 100644 --- a/pims/api.py +++ b/pims/api.py @@ -30,7 +30,9 @@ def raiser(*args, **kwargs): return raiser -def wrap_fmt(reader, name, description, extensions=None, modes=None): +def register_fmt(reader, name, description, extensions=None, modes=None): + """Registers a Format with the format manager. Returns a reader + for backwards compatibility.""" formats.add_format(reader(name, description, extensions, modes)) @wraps(reader) def wrapper(filename, **kwargs): @@ -92,11 +94,10 @@ def wrapper(filename, **kwargs): if not pims.tiff_stack.tifffile_available(): TiffStack_tifffile = not_available("tifffile") else: - TiffStack_tifffile = wrap_fmt(FormatTiffStack_tifffile, - 'TIFF_tifffile', - 'Reads TIFF files through tifffile.py.', - 'tif tiff lsm stk', - 'iIvV') + TiffStack_tifffile = register_fmt(FormatTiffStack_tifffile, + 'TIFF_tifffile', + 'Reads TIFF files through tifffile.py.', + 'tif tiff lsm stk', 'iIvV') if not pims.tiff_stack.libtiff_available(): TiffStack_libtiff = not_available("libtiff") if not pims.tiff_stack.PIL_available(): diff --git a/pims/base_frames.py b/pims/base_frames.py index 1d8db94d..edad7bf7 100644 --- a/pims/base_frames.py +++ b/pims/base_frames.py @@ -669,11 +669,15 @@ def default_axes(sizes, mode): class _PimsFormat(Format): def _can_read(self, request): + """Determine whether `request.filename` can be read using this + Format.Reader, judging from the imageio.core.Request object.""" if request.mode[1] in (self.modes + '?'): if request.filename.lower().endswith(self.extensions): return True def _can_write(self, request): + """Determine whether file type `request.filename` can be written using + this Format.Writer, judging from the imageio.core.Request object.""" return False @Slicerator.from_class @@ -681,60 +685,47 @@ class Reader(with_metaclass(ABCMeta, Format.Reader)): def __getitem__(self, key): return self.get_data(key) - @abstractmethod - def _get_length(self): - pass + def get_data(self, index, **kwargs): + # This function is a copy of the imageio Format.Reader, + # replacing the returned Image object with a PIMS Frame. + self._checkClosed() + self._BaseReaderWriter_last_index = index + im, meta = self._get_data(index, **kwargs) + return Frame(im, frame_no=index, metadata=meta) + + def iter_data(self): + return iter(self[:]) + + def get_meta_data(self, i): + # can be overwritten by a reader for better performance + return self.get_data(i).metadata + + @property + def shape(self): + return (len(self),) + tuple(self.frame_shape) @abstractmethod def _open(self, **kwargs): pass @abstractmethod - def _get_dtype(self): + def _get_data(self, i): pass - @property - def dtype(self): - return self._get_dtype() - - @property - def shape(self): - return (len(self),) + self.frame_shape + @abstractmethod + def __len__(self): + pass @abstractproperty def frame_shape(self): """ Returns the shape of the frame as returned by get_frame. """ pass - @property - def ndim(self): - """ Returns the number of axes. """ - return len(self.frame_shape) + 1 - - def get_data(self, index, **kwargs): - """ get_data(index, **kwargs) - - Read image data from the file, using the image index. The - returned image has a 'meta' attribute with the meta data. - - Some formats may support additional keyword arguments. These are - listed in the documentation of those formats. - """ - self._checkClosed() - self._BaseReaderWriter_last_index = index - im, meta = self._get_data(index, **kwargs) - return Frame(im, frame_no=index, metadata=meta) - - def iter_data(self): - return iter(self[:]) - - @abstractmethod - def _get_data(self, i): + @abstractproperty + def pixel_type(self): + """Returns a numpy.dtype for the data type of the pixel values""" pass - def _get_meta_data(self, i): - return self.get_data(i).metadata - class PimsFormat(_PimsFormat): class Reader(_PimsFormat.Reader): @@ -774,14 +765,6 @@ def _init_axis(self, name, size, default=0): self.default_coords[name] = int(default) def get_data(self, index, **kwargs): - """ get_data(index, **kwargs) - - Read image data from the file, using the image index. The - returned image has a 'meta' attribute with the meta data. - - Some formats may support additional keyword arguments. These are - listed in the documentation of those formats. - """ self._checkClosed() self._BaseReaderWriter_last_index = index return self._get_data(index, **kwargs) @@ -820,7 +803,7 @@ def _get_data(self, i): dict(axes=self.bundle_axes, coords=metadata_coords)) return Frame(result, frame_no=i, metadata=metadata) - def _get_length(self): + def __len__(self): return int(np.prod([self._sizes[d] for d in self._iter_axes])) @property @@ -885,7 +868,7 @@ def bundle_axes(self, value): # update the get_frame method get_frame = _make_get_frame(self._bundle_axes, self._get_frame_dict, - self.sizes, self.dtype) + self.sizes, self.pixel_type) self._get_frame_wrapped = get_frame @property diff --git a/pims/tiff_stack.py b/pims/tiff_stack.py index 2ad0939b..d88b7af2 100644 --- a/pims/tiff_stack.py +++ b/pims/tiff_stack.py @@ -118,8 +118,8 @@ def _open(self, **kwargs): self.bundle_axes, self.iter_axes = default_axes(self.sizes, self.request.mode) - - def _get_dtype(self): + @property + def pixel_type(self): return self._dtype def _get_frame(self, **inds): From 031cdc600d5b8a1c1fe7e9c47d759b1d373e8836 Mon Sep 17 00:00:00 2001 From: Casper van der Wel Date: Thu, 20 Jul 2017 18:07:45 +0200 Subject: [PATCH 05/14] Rename pixel_type to dtype --- pims/base_frames.py | 4 ++-- pims/tiff_stack.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pims/base_frames.py b/pims/base_frames.py index edad7bf7..5b70f630 100644 --- a/pims/base_frames.py +++ b/pims/base_frames.py @@ -722,7 +722,7 @@ def frame_shape(self): pass @abstractproperty - def pixel_type(self): + def dtype(self): """Returns a numpy.dtype for the data type of the pixel values""" pass @@ -868,7 +868,7 @@ def bundle_axes(self, value): # update the get_frame method get_frame = _make_get_frame(self._bundle_axes, self._get_frame_dict, - self.sizes, self.pixel_type) + self.sizes, self.dtype) self._get_frame_wrapped = get_frame @property diff --git a/pims/tiff_stack.py b/pims/tiff_stack.py index d88b7af2..edfb1ae4 100644 --- a/pims/tiff_stack.py +++ b/pims/tiff_stack.py @@ -119,7 +119,7 @@ def _open(self, **kwargs): self.bundle_axes, self.iter_axes = default_axes(self.sizes, self.request.mode) @property - def pixel_type(self): + def dtype(self): return self._dtype def _get_frame(self, **inds): From 09b9838c2b21dfdf03a1989adf4bb32acfc94480 Mon Sep 17 00:00:00 2001 From: caspervdw Date: Fri, 21 Jul 2017 17:44:04 +0200 Subject: [PATCH 06/14] TST: Make TIFF metadata test work for ND and non-ND readers --- pims/tests/test_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pims/tests/test_common.py b/pims/tests/test_common.py index c8c3e8f3..032a992b 100644 --- a/pims/tests/test_common.py +++ b/pims/tests/test_common.py @@ -480,9 +480,9 @@ def test_metadata(self): m = self.v[0].metadata expected = {'Software': 'tifffile.py', 'DateTime': datetime(2015, 1, 18, 15, 33, 49), - 'axes': ['y', 'x'], 'coords': {'t': 0}, 'ImageDescription': 'shape=(5,512,512)'} - assert_equal(m, expected) + for key in expected: + assert_equal(m[key], expected[key]) class TestTiffStack_libtiff(_tiff_image_series, unittest.TestCase): From 20a4c5a7754d1f4637f715ff4aaab0090048dc8e Mon Sep 17 00:00:00 2001 From: caspervdw Date: Fri, 21 Jul 2017 17:58:26 +0200 Subject: [PATCH 07/14] Add Pillow dependency via imageio, fix tests --- .travis.yml | 4 ++-- pims/tests/test_common.py | 8 +++++++- setup.py | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0ff045c8..d51ad3e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,11 @@ sudo: false matrix: include: - python: "2.7" - env: DEPS="numpy=1.8* slicerator tifffile" DEPSPIP="imageio" BUILD_DOCS=false + env: DEPS="numpy=1.8* slicerator pillow tifffile jinja2" DEPSPIP="imageio" BUILD_DOCS=false - python: "2.7" env: DEPS="numpy slicerator scipy pillow matplotlib scikit-image jinja2 av libtiff tifffile jpype1" DEPSPIP="moviepy imageio" BUILD_DOCS=false - python: "3.6" - env: DEPS="numpy slicerator tifffile" DEPSPIP="imageio" BUILD_DOCS=false + env: DEPS="numpy slicerator pillow tifffile jinja2" DEPSPIP="imageio" BUILD_DOCS=false - python: "3.6" env: DEPS="numpy slicerator scipy pillow matplotlib scikit-image jinja2 av tifffile libtiff jpype1 ipython sphinx sphinx_rtd_theme numpydoc" DEPSPIP="moviepy imageio" BUILD_DOCS=true diff --git a/pims/tests/test_common.py b/pims/tests/test_common.py index 032a992b..ee111010 100644 --- a/pims/tests/test_common.py +++ b/pims/tests/test_common.py @@ -448,7 +448,11 @@ def setUp(self): self.frame1 = np.load(os.path.join(path, 'bulk-water_frame1.npy')) self.klass = pims.ImageIOReader self.kwargs = dict() - self.v = self.klass(self.filename, **self.kwargs) + try: + self.v = self.klass(self.filename, **self.kwargs) + except imageio.core.fetching.NeedDownloadError: + imageio.plugins.ffmpeg.download() + self.v = self.klass(self.filename, **self.kwargs) self.expected_shape = (424, 640, 3) self.expected_len = 480 @@ -566,6 +570,7 @@ def setUp(self): _skip_if_no_PIL() def test_open_png(self): + _skip_if_no_imread() self.filenames = ['dummy_png.png'] shape = (10, 11) save_dummy_png(path, self.filenames, shape) @@ -573,6 +578,7 @@ def test_open_png(self): clean_dummy_png(path, self.filenames) def test_open_pngs(self): + _skip_if_no_imread() self.filepath = os.path.join(path, 'image_sequence') self.filenames = ['T76S3F00001.png', 'T76S3F00002.png', 'T76S3F00003.png', 'T76S3F00004.png', diff --git a/setup.py b/setup.py index ebe0f0ed..34006d43 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ description="Python Image Sequence", author="PIMS Contributors", install_requires=['slicerator>=0.9.7', 'six>=1.8', 'numpy>=1.7', - 'imageio'], + 'pillow', 'imageio'], author_email="dallan@pha.jhu.edu", url="https://github.com/soft-matter/pims", packages=['pims', From 90525435e115248b3a033f79b874509e5a5cc947 Mon Sep 17 00:00:00 2001 From: caspervdw Date: Fri, 22 Sep 2017 14:51:11 +0200 Subject: [PATCH 08/14] WIP: TiffStack as pure imageio plugin --- pims/tiff_stack.py | 70 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/pims/tiff_stack.py b/pims/tiff_stack.py index edfb1ae4..5fb17d8b 100644 --- a/pims/tiff_stack.py +++ b/pims/tiff_stack.py @@ -8,6 +8,7 @@ import itertools import numpy as np from pims.frame import Frame +from imageio.core import Format try: from PIL import Image # should work with PIL or PILLOW @@ -53,7 +54,7 @@ def _tiff_datetime(dt_str): minute=int(dt_str[14:16]), second=int(dt_str[17:19])) -class FormatTiffStack_tifffile(PimsFormat): +class FormatTiffStack_tifffile(Format): """Read TIFF stacks (single files containing many images) into an iterable object that returns images as numpy arrays. @@ -97,8 +98,27 @@ class FormatTiffStack_tifffile(PimsFormat): -------- TiffStack_pil, TiffStack_libtiff, ImageSequence """ - class Reader(PimsFormat.Reader): + def _can_read(self, request): + """Determine whether `request.filename` can be read using this + Format.Reader, judging from the imageio.core.Request object.""" + if request.mode[1] in (self.modes + '?'): + if request.filename.lower().endswith(self.extensions): + return True + + def _can_write(self, request): + """Determine whether file type `request.filename` can be written using + this Format.Writer, judging from the imageio.core.Request object.""" + return False + + class Reader(Format.Reader): def _open(self, **kwargs): + # Specify kwargs here. Optionally, the user-specified kwargs + # can also be accessed via the request.kwargs object. + # + # The request object provides two ways to get access to the + # data. Use just one: + # - Use request.get_file() for a file object (preferred) + # - Use request.get_local_filename() for a file on the system handle = self.request.get_file() record = tifffile.TiffFile(handle, **kwargs).series[0] if hasattr(record, 'pages'): @@ -106,26 +126,40 @@ def _open(self, **kwargs): else: self._tiff = record['pages'] - tmp = self._tiff[0] - self._dtype = tmp.dtype + # tmp = self._tiff[0] + # self._dtype = tmp.dtype + # + # axes = guess_axes(tmp) + # for name, size in zip(axes, tmp.shape): + # self._init_axis(name, size) + # self._init_axis('t', len(self._tiff)) + # + # self._register_get_frame(self._get_frame, axes) + # + # self.bundle_axes, self.iter_axes = default_axes(self.sizes, + # self.request.mode) + + def _close(self): + # Close the reader. + # Note that the request object will close self._fp + pass - axes = guess_axes(tmp) - for name, size in zip(axes, tmp.shape): - self._init_axis(name, size) - self._init_axis('t', len(self._tiff)) + def _get_pims_info(self): + return dict(dtype=self._dtype) - self._register_get_frame(self._get_frame, axes) + def _get_length(self): + # Return the number of images. Can be np.inf + return len(self._tiff) - self.bundle_axes, self.iter_axes = default_axes(self.sizes, - self.request.mode) - @property - def dtype(self): - return self._dtype + def _get_data(self, index): + tiff = self._tiff[index] + return tiff.asarray(), self._read_metadata(tiff) - def _get_frame(self, **inds): - t = self._tiff[inds['t']] - return Frame(t.asarray(), frame_no=0, - metadata=self._read_metadata(t)) + def _get_meta_data(self, index): + # Get the meta data for the given index. If index is None, it + # should return the global meta data. + tiff = self._tiff[index] + return self._read_metadata(tiff) def _read_metadata(self, tiff): """Read metadata for current frame and return as dict""" From 65db37a54ff4a0d4be8390f4c62b9f31b635d9aa Mon Sep 17 00:00:00 2001 From: caspervdw Date: Fri, 22 Sep 2017 15:16:43 +0200 Subject: [PATCH 09/14] Working example for imageio plugin-based architecture --- pims/api.py | 3 +- pims/base_frames.py | 277 +++++++------------------------------------- pims/tiff_stack.py | 17 +-- 3 files changed, 49 insertions(+), 248 deletions(-) diff --git a/pims/api.py b/pims/api.py index 01031009..ecee1745 100644 --- a/pims/api.py +++ b/pims/api.py @@ -15,6 +15,7 @@ from warnings import warn # has to be here for API stuff +from pims.base_frames import WrapImageIOReader from pims.image_sequence import ImageSequence, ImageSequenceND, ReaderSequence # noqa from pims.image_reader import ImageReader, ImageReaderND # noqa from .cine import Cine # noqa @@ -36,7 +37,7 @@ def register_fmt(reader, name, description, extensions=None, modes=None): formats.add_format(reader(name, description, extensions, modes)) @wraps(reader) def wrapper(filename, **kwargs): - return get_reader(filename, name, **kwargs) + return WrapImageIOReader(get_reader(filename, name, **kwargs)) return wrapper diff --git a/pims/base_frames.py b/pims/base_frames.py index 5b70f630..8f22830b 100644 --- a/pims/base_frames.py +++ b/pims/base_frames.py @@ -667,235 +667,48 @@ def default_axes(sizes, mode): return bundle_axes, iter_axes -class _PimsFormat(Format): - def _can_read(self, request): - """Determine whether `request.filename` can be read using this - Format.Reader, judging from the imageio.core.Request object.""" - if request.mode[1] in (self.modes + '?'): - if request.filename.lower().endswith(self.extensions): - return True - - def _can_write(self, request): - """Determine whether file type `request.filename` can be written using - this Format.Writer, judging from the imageio.core.Request object.""" - return False - - @Slicerator.from_class - class Reader(with_metaclass(ABCMeta, Format.Reader)): - def __getitem__(self, key): - return self.get_data(key) - - def get_data(self, index, **kwargs): - # This function is a copy of the imageio Format.Reader, - # replacing the returned Image object with a PIMS Frame. - self._checkClosed() - self._BaseReaderWriter_last_index = index - im, meta = self._get_data(index, **kwargs) - return Frame(im, frame_no=index, metadata=meta) - - def iter_data(self): - return iter(self[:]) - - def get_meta_data(self, i): - # can be overwritten by a reader for better performance - return self.get_data(i).metadata - - @property - def shape(self): - return (len(self),) + tuple(self.frame_shape) - - @abstractmethod - def _open(self, **kwargs): - pass - - @abstractmethod - def _get_data(self, i): - pass - - @abstractmethod - def __len__(self): - pass - - @abstractproperty - def frame_shape(self): - """ Returns the shape of the frame as returned by get_frame. """ - pass - - @abstractproperty - def dtype(self): - """Returns a numpy.dtype for the data type of the pixel values""" - pass - - -class PimsFormat(_PimsFormat): - class Reader(_PimsFormat.Reader): - def __init__(self, *args, **kwargs): - self._clear_axes() - self._get_frame_dict = dict() - Format.Reader.__init__(self, *args, **kwargs) - - def _register_get_frame(self, method, axes): - axes = tuple([a for a in axes]) - if not hasattr(self, '_get_frame_dict'): - warn( - "Please call FramesSequenceND.__init__() at the start of the" - "the reader initialization.") - self._get_frame_dict = dict() - self._get_frame_dict[axes] = method - - def _clear_axes(self): - self._sizes = {} - self._default_coords = DefaultCoordsDict() - self._iter_axes = [] - self._bundle_axes = ['y', 'x'] - self._get_frame_wrapped = None - - def _init_axis(self, name, size, default=0): - # check if the axes have been initialized, if not, do it here - if not hasattr(self, '_sizes'): - warn( - "Please call FramesSequenceND.__init__() at the start of the" - "the reader initialization.") - self._clear_axes() - self._get_frame_dict = dict() - if name in self._sizes: - raise ValueError("axis '{}' already exists".format(name)) - self._sizes[name] = int(size) - self.default_coords.axes = self.axes - self.default_coords[name] = int(default) - - def get_data(self, index, **kwargs): - self._checkClosed() - self._BaseReaderWriter_last_index = index - return self._get_data(index, **kwargs) - - def _get_data(self, i): - """ Returns a Frame of shape determined by bundle_axes. The index value - is interpreted according to the iter_axes property. Coordinates not - present in both iter_axes and bundle_axes will be set to their default - value (see default_coords). """ - if i > len(self): - raise IndexError('index out of range') - if self._get_frame_wrapped is None: - self.bundle_axes = tuple(self.bundle_axes) # kick bundle_axes - - # start with the default coordinates - coords = self.default_coords.copy() - - # list sizes of iteration axes - iter_sizes = [self._sizes[k] for k in self.iter_axes] - # list how much i has to increase to get an increase of coordinate n - iter_cumsizes = np.append(np.cumprod(iter_sizes[::-1])[-2::-1], 1) - # calculate the coordinates and update the coords dictionary - iter_coords = (i // iter_cumsizes) % iter_sizes - coords.update( - **{k: v for k, v in zip(self.iter_axes, iter_coords)}) - - result = self._get_frame_wrapped(**coords) - if hasattr(result, 'metadata'): - metadata = result.metadata - else: - metadata = dict() - - metadata_axes = set(self.axes) - set(self.bundle_axes) - metadata_coords = {ax: coords[ax] for ax in metadata_axes} - metadata.update( - dict(axes=self.bundle_axes, coords=metadata_coords)) - return Frame(result, frame_no=i, metadata=metadata) - - def __len__(self): - return int(np.prod([self._sizes[d] for d in self._iter_axes])) - - @property - def shape(self): - iter_shape = tuple([self._sizes[d] for d in self._iter_axes]) - return iter_shape + self.frame_shape - - @property - def frame_shape(self): - """ Returns the shape of the frame as returned by get_frame. """ - return tuple([self._sizes[d] for d in self._bundle_axes]) - - @property - def axes(self): - """ Returns a list of all axes. """ - return [k for k in self._sizes] - - @property - def ndim(self): - """ Returns the number of axes. """ - return len(self._sizes) - - @property - def sizes(self): - """ Returns a dict of all axis sizes. """ - return self._sizes - - @property - def bundle_axes(self): - """ This determines which axes will be bundled into one Frame. - The ndarray that is returned by get_frame has the same axis order - as the order of `bundle_axes`. - """ - return self._bundle_axes[:] # return a copy - - @bundle_axes.setter - def bundle_axes(self, value): - value = list(value) - invalid = [k for k in value if k not in self._sizes] - if invalid: - raise ValueError("axes %r do not exist" % invalid) - - for k in value: - if k in self._iter_axes: - del self._iter_axes[self._iter_axes.index(k)] - - self._bundle_axes = value - if not hasattr(self, '_get_frame_dict'): - warn( - "Please call FramesSequenceND.__init__() at the start of the" - "the reader initialization.") - self._get_frame_dict = dict() - if len(self._get_frame_dict) == 0: - if hasattr(self, 'get_frame_2D'): - # include get_frame_2D for backwards compatibility - self._register_get_frame(self.get_frame_2D, 'yx') - else: - raise RuntimeError( - 'No reader methods found. Register a reader ' - 'method with _register_get_frame') - - # update the get_frame method - get_frame = _make_get_frame(self._bundle_axes, - self._get_frame_dict, - self.sizes, self.dtype) - self._get_frame_wrapped = get_frame - - @property - def iter_axes(self): - """ This determines which axes will be iterated over by the - FramesSequence. The last element will iterate fastest. """ - return self._iter_axes[:] # return a copy - - @iter_axes.setter - def iter_axes(self, value): - value = list(value) - invalid = [k for k in value if k not in self._sizes] - if invalid: - raise ValueError("axes %r do not exist" % invalid) - - for k in value: - if k in self._bundle_axes: - del self._bundle_axes[self._bundle_axes.index(k)] - - self._iter_axes = value - - @property - def default_coords(self): - """ When a axis is not present in both iter_axes and bundle_axes, the - coordinate contained in this dictionary will be used. """ - return self._default_coords # this is a custom dict (DefaultCoordsDict) - - @default_coords.setter - def default_coords(self, value): - self._default_coords.update(**value) +def wrap_get_data(get_data_func, axis='t'): + # takes only kwargs (one index per named axis) + def get_frame(**kwargs): + index = kwargs[axis] + im, md = get_data_func(index) + return Frame(im, frame_no=index, metadata=md) + return get_frame + + +class WrapImageIOReader(FramesSequenceND): + def __init__(self, imageio_reader): + if not isinstance(imageio_reader, Format.Reader): + raise ValueError("Can only wrap ImageIO readers") + + super(WrapImageIOReader, self).__init__() + self.rdr = imageio_reader + + # TODO pass the dimension-awareness through this field + try: + info = self.rdr._get_pims_info() + if not isinstance(info, dict): + info = dict() + except AttributeError: + info = dict() + + # interpret first frame (TODO skip if reader is already dimension-aware) + tmp, _ = self.rdr._get_data(0) + self._dtype = tmp.dtype + + axes = guess_axes(tmp) + for name, size in zip(axes, tmp.shape): + self._init_axis(name, size) + self._init_axis('t', self.rdr._get_length()) + self._register_get_frame(wrap_get_data(self.rdr._get_data, 't'), axes) + + self.bundle_axes, self.iter_axes = default_axes(self.sizes, + self.rdr.request.mode) + + @property + def pixel_type(self): + return self._dtype + + @property + def dtype(self): + return self._dtype diff --git a/pims/tiff_stack.py b/pims/tiff_stack.py index 5fb17d8b..0bfa89ba 100644 --- a/pims/tiff_stack.py +++ b/pims/tiff_stack.py @@ -41,7 +41,7 @@ def tifffile_available(): return tifffile is not None -from pims.base_frames import FramesSequence, PimsFormat, guess_axes, default_axes +from pims.base_frames import FramesSequence _dtype_map = {4: np.uint8, 8: np.uint8, @@ -126,26 +126,13 @@ def _open(self, **kwargs): else: self._tiff = record['pages'] - # tmp = self._tiff[0] - # self._dtype = tmp.dtype - # - # axes = guess_axes(tmp) - # for name, size in zip(axes, tmp.shape): - # self._init_axis(name, size) - # self._init_axis('t', len(self._tiff)) - # - # self._register_get_frame(self._get_frame, axes) - # - # self.bundle_axes, self.iter_axes = default_axes(self.sizes, - # self.request.mode) - def _close(self): # Close the reader. # Note that the request object will close self._fp pass def _get_pims_info(self): - return dict(dtype=self._dtype) + return dict() def _get_length(self): # Return the number of images. Can be np.inf From 5bde2265d72d79d860db2613b71ef537f90d26cc Mon Sep 17 00:00:00 2001 From: caspervdw Date: Wed, 11 Oct 2017 10:29:26 +0200 Subject: [PATCH 10/14] WIP ImageIO wrapping of Bioformats --- pims/api.py | 15 +- pims/base_frames.py | 33 +-- pims/bioformats.py | 525 ++++++++++++++++++++++---------------------- 3 files changed, 298 insertions(+), 275 deletions(-) diff --git a/pims/api.py b/pims/api.py index ecee1745..504ca294 100644 --- a/pims/api.py +++ b/pims/api.py @@ -119,14 +119,25 @@ def wrapper(filename, **kwargs): try: import pims.bioformats if pims.bioformats.available(): - Bioformats = pims.bioformats.BioformatsReader + Bioformats = register_fmt(pims.bioformats.BioformatsFormat, + 'Bioformats', + 'Reads multidimensional images from a file supported by bioformats.', + 'lsm ipl dm3 seq nd2 ics ids ipw tif tiff' + ' jpg bmp lif lei', 'iIvV') else: raise ImportError() except (ImportError, IOError): - BioformatsRaw = not_available("JPype") Bioformats = not_available("JPype") +if not pims.tiff_stack.tifffile_available(): + TiffStack_tifffile = not_available("tifffile") +else: + TiffStack_tifffile = register_fmt(FormatTiffStack_tifffile, + 'TIFF_tifffile', + 'Reads TIFF files through tifffile.py.', + 'tif tiff lsm stk', 'iIvV') + try: from pims_nd2 import ND2_Reader as ND2Reader_SDK diff --git a/pims/base_frames.py b/pims/base_frames.py index 8f22830b..669f1cab 100644 --- a/pims/base_frames.py +++ b/pims/base_frames.py @@ -686,21 +686,28 @@ def __init__(self, imageio_reader): # TODO pass the dimension-awareness through this field try: - info = self.rdr._get_pims_info() - if not isinstance(info, dict): - info = dict() + info = self.rdr.pims_info except AttributeError: - info = dict() - - # interpret first frame (TODO skip if reader is already dimension-aware) - tmp, _ = self.rdr._get_data(0) - self._dtype = tmp.dtype + info = None + + if info is None: + # guess everything from the first frame + tmp, _ = self.rdr._get_data(0) + self._dtype = tmp.dtype + + axes = guess_axes(tmp) + for name, size in zip(axes, tmp.shape): + self._init_axis(name, size) + self._init_axis('t', self.rdr._get_length()) + self._register_get_frame(wrap_get_data(self.rdr._get_data, 't'), axes) + else: + self._dtype = info['dtype'] + for axis, size in info['sizes'].items(): + self._init_axis(axis, size) - axes = guess_axes(tmp) - for name, size in zip(axes, tmp.shape): - self._init_axis(name, size) - self._init_axis('t', self.rdr._get_length()) - self._register_get_frame(wrap_get_data(self.rdr._get_data, 't'), axes) + for name, axes in info['read_methods']: + method = getattr(self.rdr, name) + self._register_get_frame(method, axes) self.bundle_axes, self.iter_axes = default_axes(self.sizes, self.rdr.request.mode) diff --git a/pims/bioformats.py b/pims/bioformats.py index 85868bf2..4623d8dd 100644 --- a/pims/bioformats.py +++ b/pims/bioformats.py @@ -3,7 +3,7 @@ import numpy as np -from pims.base_frames import FramesSequence, FramesSequenceND +from imageio.core import Format from pims.frame import Frame from warnings import warn import os @@ -197,7 +197,7 @@ def __repr__(self): 'MetadataRetrieve functions: ' + ', '.join(self.fields) -class BioformatsReader(FramesSequenceND): +class BioformatsFormat(Format): """Reads multidimensional images from the frames of a file supported by bioformats into an iterable object that returns images as numpy arrays. The axes inside the numpy array (czyx, zyx, cyx or yx) depend on the @@ -315,264 +315,269 @@ class BioformatsReader(FramesSequenceND): x_um, y_um, z_um: physical location of the image in microns t_s: timestamp of the image in seconds """ - @classmethod - def class_exts(cls): - return {'lsm', 'ipl', 'dm3', 'seq', 'nd2', 'ics', 'ids', - 'ipw', 'tif', 'tiff', 'jpg', 'bmp', 'lif', 'lei'} - - class_priority = 2 - propagate_attrs = ['frame_shape', 'pixel_type', 'metadata', - 'get_metadata_raw', 'reader_class_name'] - - @property - def pixel_type(self): - return self._pixel_type - - def __init__(self, filename, meta=True, java_memory='512m', - read_mode='auto', series=0): - global loci - super(BioformatsReader, self).__init__() - - if read_mode not in ['auto', 'jpype', 'stringbuffer', 'javacasting']: - raise ValueError('Invalid read_mode value.') - - # Make sure that file exists before starting java - if not os.path.isfile(filename): - raise IOError('The file "{}" does not exist.'.format(filename)) - - # Start java VM and initialize logger (globally) - if not jpype.isJVMStarted(): - loci_path = _find_jar() - jpype.startJVM(jpype.getDefaultJVMPath(), '-ea', - '-Djava.class.path=' + loci_path, - '-Xmx' + java_memory) - log4j = jpype.JPackage('org.apache.log4j') - log4j.BasicConfigurator.configure() - log4j_logger = log4j.Logger.getRootLogger() - log4j_logger.setLevel(log4j.Level.ERROR) - - if not jpype.isThreadAttachedToJVM(): - jpype.attachThreadToJVM() - - loci = jpype.JPackage('loci') - - # Initialize reader and metadata - self.filename = str(filename) - self.rdr = loci.formats.ChannelSeparator(loci.formats.ChannelFiller()) - - # patch for issue with ND2 files and the Chunkmap implemented in 5.4.0 - # See https://github.com/openmicroscopy/bioformats/issues/2955 - # circumventing the reserved keyword 'in' - mo = getattr(loci.formats, 'in').DynamicMetadataOptions() - mo.set('nativend2.chunkmap', 'False') # Format Bool as String - self.rdr.setMetadataOptions(mo) - - if meta: - self._metadata = loci.formats.MetadataTools.createOMEXMLMetadata() - self.rdr.setMetadataStore(self._metadata) - self.rdr.setId(self.filename) - if meta: - self.metadata = MetadataRetrieve(self._metadata) - - # Checkout reader dtype and define read mode - isLittleEndian = self.rdr.isLittleEndian() - LE_prefix = ['>', '<'][isLittleEndian] - FormatTools = loci.formats.FormatTools - self._dtype_dict = {FormatTools.INT8: 'i1', - FormatTools.UINT8: 'u1', - FormatTools.INT16: LE_prefix + 'i2', - FormatTools.UINT16: LE_prefix + 'u2', - FormatTools.INT32: LE_prefix + 'i4', - FormatTools.UINT32: LE_prefix + 'u4', - FormatTools.FLOAT: LE_prefix + 'f4', - FormatTools.DOUBLE: LE_prefix + 'f8'} - self._dtype_dict_java = {} - for loci_format in self._dtype_dict.keys(): - self._dtype_dict_java[loci_format] = \ - (FormatTools.getBytesPerPixel(loci_format), - FormatTools.isFloatingPoint(loci_format), - isLittleEndian) - - # Set the correct series and initialize the sizes - self.size_series = self.rdr.getSeriesCount() - if series >= self.size_series or series < 0: - self.rdr.close() - raise IndexError('Series index out of bounds.') - self._series = series - self._change_series() - - # Set read mode. When auto, tryout fast and check the image size. - if read_mode == 'auto': - Jarr = self.rdr.openBytes(0) - if isinstance(Jarr[:], np.ndarray): - read_mode = 'jpype' - else: - warn('Due to an issue with JPype 0.6.0, reading is slower. ' - 'Please consider upgrading JPype to 0.6.1 or later.') - try: - im = self._jbytearr_stringbuffer(Jarr) - im.reshape(self._sizeRGB, self._sizeX, self._sizeY) - except (AttributeError, ValueError): - read_mode = 'javacasting' + def _can_read(self, request): + """Determine whether `request.filename` can be read using this + Format.Reader, judging from the imageio.core.Request object.""" + if request.mode[1] in (self.modes + '?'): + if request.filename.lower().endswith(self.extensions): + return True + + def _can_write(self, request): + """Determine whether file type `request.filename` can be written using + this Format.Writer, judging from the imageio.core.Request object.""" + return False + + # TODO What to do with these?? + # class_priority = 2 + # propagate_attrs = ['frame_shape', 'pixel_type', 'metadata', + # 'get_metadata_raw', 'reader_class_name'] + + class BioformatsReader(Format.Reader): + @property + def pixel_type(self): + return self._pixel_type + + def _open(self, series=0, meta=True, java_memory='512m', + read_mode='auto'): + global loci + self.pims_info = None + + if read_mode not in ['auto', 'jpype', 'stringbuffer', 'javacasting']: + raise ValueError('Invalid read_mode value.') + + filename = str(self.request.get_local_filename()) + + # Make sure that file exists before starting java + if not os.path.isfile(filename): + raise IOError('The file "{}" does not exist.'.format(filename)) + + # Start java VM and initialize logger (globally) + if not jpype.isJVMStarted(): + loci_path = _find_jar() + jpype.startJVM(jpype.getDefaultJVMPath(), '-ea', + '-Djava.class.path=' + loci_path, + '-Xmx' + java_memory) + log4j = jpype.JPackage('org.apache.log4j') + log4j.BasicConfigurator.configure() + log4j_logger = log4j.Logger.getRootLogger() + log4j_logger.setLevel(log4j.Level.ERROR) + + if not jpype.isThreadAttachedToJVM(): + jpype.attachThreadToJVM() + + loci = jpype.JPackage('loci') + + # Initialize reader and metadata + self.filename = str(filename) + self.rdr = loci.formats.ChannelSeparator(loci.formats.ChannelFiller()) + + # patch for issue with ND2 files and the Chunkmap implemented in 5.4.0 + # See https://github.com/openmicroscopy/bioformats/issues/2955 + # circumventing the reserved keyword 'in' + mo = getattr(loci.formats, 'in').DynamicMetadataOptions() + mo.set('nativend2.chunkmap', 'False') # Format Bool as String + self.rdr.setMetadataOptions(mo) + + if meta: + self._metadata = loci.formats.MetadataTools.createOMEXMLMetadata() + self.rdr.setMetadataStore(self._metadata) + self.rdr.setId(self.filename) + if meta: + self.metadata = MetadataRetrieve(self._metadata) + + # Checkout reader dtype and define read mode + isLittleEndian = self.rdr.isLittleEndian() + LE_prefix = ['>', '<'][isLittleEndian] + FormatTools = loci.formats.FormatTools + self._dtype_dict = {FormatTools.INT8: 'i1', + FormatTools.UINT8: 'u1', + FormatTools.INT16: LE_prefix + 'i2', + FormatTools.UINT16: LE_prefix + 'u2', + FormatTools.INT32: LE_prefix + 'i4', + FormatTools.UINT32: LE_prefix + 'u4', + FormatTools.FLOAT: LE_prefix + 'f4', + FormatTools.DOUBLE: LE_prefix + 'f8'} + self._dtype_dict_java = {} + for loci_format in self._dtype_dict.keys(): + self._dtype_dict_java[loci_format] = \ + (FormatTools.getBytesPerPixel(loci_format), + FormatTools.isFloatingPoint(loci_format), + isLittleEndian) + + # Set the correct series and initialize the sizes + self.size_series = self.rdr.getSeriesCount() + if series >= self.size_series or series < 0: + self.rdr.close() + raise IndexError('Series index out of bounds.') + self._series = series + self._change_series() + + # Set read mode. When auto, tryout fast and check the image size. + if read_mode == 'auto': + Jarr = self.rdr.openBytes(0) + if isinstance(Jarr[:], np.ndarray): + read_mode = 'jpype' + else: + warn('Due to an issue with JPype 0.6.0, reading is slower. ' + 'Please consider upgrading JPype to 0.6.1 or later.') + try: + im = self._jbytearr_stringbuffer(Jarr) + im.reshape(self._sizeRGB, self._sizeX, self._sizeY) + except (AttributeError, ValueError): + read_mode = 'javacasting' + else: + read_mode = 'stringbuffer' + self.read_mode = read_mode + + # Define the names of the standard per frame metadata. + self.frame_metadata = {} + if meta: + if hasattr(self.metadata, 'PlaneDeltaT'): + self.frame_metadata['t_s'] = 'PlaneDeltaT' + if hasattr(self.metadata, 'PlanePositionX'): + self.frame_metadata['x_um'] = 'PlanePositionX' + if hasattr(self.metadata, 'PlanePositionY'): + self.frame_metadata['y_um'] = 'PlanePositionY' + if hasattr(self.metadata, 'PlanePositionZ'): + self.frame_metadata['z_um'] = 'PlanePositionZ' + + def _change_series(self): + """Changes series and rereads axes, sizes and metadata. + """ + series = self._series + self.rdr.setSeries(series) + sizeX = self.rdr.getSizeX() + sizeY = self.rdr.getSizeY() + sizeT = self.rdr.getSizeT() + sizeZ = self.rdr.getSizeZ() + self.isRGB = self.rdr.isRGB() + self.isInterleaved = self.rdr.isInterleaved() + if self.isRGB: + sizeC = self.rdr.getRGBChannelCount() + if self.isInterleaved: + self._frame_shape_2D = (sizeY, sizeX, sizeC) + self.pims_info['read_methods'] = {'get_frame_2D': 'yxc'} else: - read_mode = 'stringbuffer' - self.read_mode = read_mode - - # Define the names of the standard per frame metadata. - self.frame_metadata = {} - if meta: - if hasattr(self.metadata, 'PlaneDeltaT'): - self.frame_metadata['t_s'] = 'PlaneDeltaT' - if hasattr(self.metadata, 'PlanePositionX'): - self.frame_metadata['x_um'] = 'PlanePositionX' - if hasattr(self.metadata, 'PlanePositionY'): - self.frame_metadata['y_um'] = 'PlanePositionY' - if hasattr(self.metadata, 'PlanePositionZ'): - self.frame_metadata['z_um'] = 'PlanePositionZ' - - def _change_series(self): - """Changes series and rereads axes, sizes and metadata. - """ - series = self._series - self._clear_axes() - self.rdr.setSeries(series) - sizeX = self.rdr.getSizeX() - sizeY = self.rdr.getSizeY() - sizeT = self.rdr.getSizeT() - sizeZ = self.rdr.getSizeZ() - self.isRGB = self.rdr.isRGB() - self.isInterleaved = self.rdr.isInterleaved() - if self.isRGB: - sizeC = self.rdr.getRGBChannelCount() - if self.isInterleaved: - self._frame_shape_2D = (sizeY, sizeX, sizeC) - self._register_get_frame(self.get_frame_2D, 'yxc') + self._frame_shape_2D = (sizeC, sizeY, sizeX) + self.pims_info['read_methods'] = {'get_frame_2D': 'cyx'} else: - self._frame_shape_2D = (sizeC, sizeY, sizeX) - self._register_get_frame(self.get_frame_2D, 'cyx') - else: - sizeC = self.rdr.getSizeC() - self._frame_shape_2D = (sizeY, sizeX) - self._register_get_frame(self.get_frame_2D, 'yx') - - self._init_axis('x', sizeX) - self._init_axis('y', sizeY) - if sizeC > 1: - self._init_axis('c', sizeC) - if sizeT > 1: - self._init_axis('t', sizeT) - if sizeZ > 1: - self._init_axis('z', sizeZ) - - # determine pixel type - pixel_type = self.rdr.getPixelType() - dtype = self._dtype_dict[pixel_type] - java_dtype = self._dtype_dict_java[pixel_type] - - self._jbytearr_stringbuffer = \ - lambda arr: _jbytearr_stringbuffer(arr, dtype) - self._jbytearr_javacasting = \ - lambda arr: _jbytearr_javacasting(arr, dtype, *java_dtype) - self._pixel_type = dtype - - if 'z' in self.axes: - self.bundle_axes = 'zyx' - if 't' in self.axes: - self.iter_axes = 't' - - # get some metadata fields - try: - self.colors = [_jrgba_to_rgb(self.metadata.ChannelColor(series, c)) - for c in range(sizeC)] - except AttributeError: - self.colors = None - try: - self.calibration = self.metadata.PixelsPhysicalSizeX(series) - except AttributeError: + sizeC = self.rdr.getSizeC() + self._frame_shape_2D = (sizeY, sizeX) + self.pims_info['read_methods'] = {'get_frame_2D': 'yx'} + + self.pims_info['sizes'] = {'x': sizeX, 'y': sizeY} + if sizeC > 1: + self.pims_info['sizes']['c'] = sizeC + if sizeT > 1: + self.pims_info['sizes']['t'] = sizeT + if sizeZ > 1: + self.pims_info['sizes']['z'] = sizeZ + + # determine pixel type + pixel_type = self.rdr.getPixelType() + dtype = self._dtype_dict[pixel_type] + java_dtype = self._dtype_dict_java[pixel_type] + + self._jbytearr_stringbuffer = \ + lambda arr: _jbytearr_stringbuffer(arr, dtype) + self._jbytearr_javacasting = \ + lambda arr: _jbytearr_javacasting(arr, dtype, *java_dtype) + + self.pims_info['dtype'] = dtype + + # get some metadata fields try: - self.calibration = self.metadata.PixelsPhysicalSizeY(series) - except: - self.calibration = None - try: - self.calibrationZ = self.metadata.PixelsPhysicalSizeZ(series) - except AttributeError: - self.calibrationZ = None - - def close(self): - self.rdr.close() - - @property - def series(self): - return self._series - - @series.setter - def series(self, value): - if value >= self.size_series or value < 0: - raise IndexError('Series index out of bounds.') - else: - if value != self._series: - self._series = value - self._change_series() - - def get_frame_2D(self, **coords): - """Actual reader, returns image as 2D numpy array and metadata as - dict. - """ - _coords = {'t': 0, 'c': 0, 'z': 0} - _coords.update(coords) - if self.isRGB: - _coords['c'] = 0 - j = self.rdr.getIndex(int(_coords['z']), int(_coords['c']), - int(_coords['t'])) - if self.read_mode == 'jpype': - im = np.frombuffer(self.rdr.openBytes(j)[:], - dtype=self._pixel_type) - elif self.read_mode == 'stringbuffer': - im = self._jbytearr_stringbuffer(self.rdr.openBytes(j)) - elif self.read_mode == 'javacasting': - im = self._jbytearr_javacasting(self.rdr.openBytes(j)) - - im.shape = self._frame_shape_2D - im = im.astype(self._pixel_type, copy=False) - - metadata = {'frame': j, - 'series': self._series} - if self.colors is not None: - metadata['colors'] = self.colors - if self.calibration is not None: - metadata['mpp'] = self.calibration - if self.calibrationZ is not None: - metadata['mppZ'] = self.calibrationZ - metadata.update(coords) - for key, method in self.frame_metadata.items(): - metadata[key] = getattr(self.metadata, method)(self._series, j) - - return Frame(im, metadata=metadata) - - def get_metadata_raw(self, form='dict'): - hashtable = self.rdr.getGlobalMetadata() - keys = hashtable.keys() - if form == 'dict': - result = {} - while keys.hasMoreElements(): - key = keys.nextElement() - result[key] = _maybe_tostring(hashtable.get(key)) - elif form == 'list': - result = [] - while keys.hasMoreElements(): - key = keys.nextElement() - result.append(key + ': ' + _maybe_tostring(hashtable.get(key))) - elif form == 'string': - result = u'' - while keys.hasMoreElements(): - key = keys.nextElement() - result += key + ': ' + _maybe_tostring(hashtable.get(key)) + '\n' - return result - - @property - def reader_class_name(self): - return self.rdr.getFormat() - - @property - def version(self): - return loci.formats.FormatTools.VERSION + self.colors = [_jrgba_to_rgb(self.metadata.ChannelColor(series, c)) + for c in range(sizeC)] + except AttributeError: + self.colors = None + try: + self.calibration = self.metadata.PixelsPhysicalSizeX(series) + except AttributeError: + try: + self.calibration = self.metadata.PixelsPhysicalSizeY(series) + except: + self.calibration = None + try: + self.calibrationZ = self.metadata.PixelsPhysicalSizeZ(series) + except AttributeError: + self.calibrationZ = None + + def close(self): + self.rdr.close() + + @property + def series(self): + return self._series + + @series.setter + def series(self, value): + if value >= self.size_series or value < 0: + raise IndexError('Series index out of bounds.') + else: + if value != self._series: + self._series = value + self._change_series() + + def get_frame_2D(self, **coords): + """Actual reader, returns image as 2D numpy array and metadata as + dict. + """ + _coords = {'t': 0, 'c': 0, 'z': 0} + _coords.update(coords) + if self.isRGB: + _coords['c'] = 0 + j = self.rdr.getIndex(int(_coords['z']), int(_coords['c']), + int(_coords['t'])) + if self.read_mode == 'jpype': + im = np.frombuffer(self.rdr.openBytes(j)[:], + dtype=self._pixel_type) + elif self.read_mode == 'stringbuffer': + im = self._jbytearr_stringbuffer(self.rdr.openBytes(j)) + elif self.read_mode == 'javacasting': + im = self._jbytearr_javacasting(self.rdr.openBytes(j)) + + im.shape = self._frame_shape_2D + im = im.astype(self._pixel_type, copy=False) + + metadata = {'frame': j, + 'series': self._series} + if self.colors is not None: + metadata['colors'] = self.colors + if self.calibration is not None: + metadata['mpp'] = self.calibration + if self.calibrationZ is not None: + metadata['mppZ'] = self.calibrationZ + metadata.update(coords) + for key, method in self.frame_metadata.items(): + metadata[key] = getattr(self.metadata, method)(self._series, j) + + return Frame(im, metadata=metadata) + + def get_metadata_raw(self, form='dict'): + hashtable = self.rdr.getGlobalMetadata() + keys = hashtable.keys() + if form == 'dict': + result = {} + while keys.hasMoreElements(): + key = keys.nextElement() + result[key] = _maybe_tostring(hashtable.get(key)) + elif form == 'list': + result = [] + while keys.hasMoreElements(): + key = keys.nextElement() + result.append(key + ': ' + _maybe_tostring(hashtable.get(key))) + elif form == 'string': + result = u'' + while keys.hasMoreElements(): + key = keys.nextElement() + result += key + ': ' + _maybe_tostring(hashtable.get(key)) + '\n' + return result + + @property + def reader_class_name(self): + return self.rdr.getFormat() + + @property + def version(self): + return loci.formats.FormatTools.VERSION From 6d340c542f8800816f3d1c80b1a7849dd28775f5 Mon Sep 17 00:00:00 2001 From: caspervdw Date: Wed, 11 Oct 2017 12:29:53 +0200 Subject: [PATCH 11/14] API: Rebased Bioformats on imageio, changed default bundle_axes --- pims/api.py | 9 ----- pims/base_frames.py | 11 +++--- pims/bioformats.py | 68 ++++++++++++++++++++--------------- pims/tests/test_bioformats.py | 18 +++++----- 4 files changed, 53 insertions(+), 53 deletions(-) diff --git a/pims/api.py b/pims/api.py index 504ca294..96d828d8 100644 --- a/pims/api.py +++ b/pims/api.py @@ -129,15 +129,6 @@ def wrapper(filename, **kwargs): except (ImportError, IOError): Bioformats = not_available("JPype") - -if not pims.tiff_stack.tifffile_available(): - TiffStack_tifffile = not_available("tifffile") -else: - TiffStack_tifffile = register_fmt(FormatTiffStack_tifffile, - 'TIFF_tifffile', - 'Reads TIFF files through tifffile.py.', - 'tif tiff lsm stk', 'iIvV') - try: from pims_nd2 import ND2_Reader as ND2Reader_SDK diff --git a/pims/base_frames.py b/pims/base_frames.py index 669f1cab..f9fc2663 100644 --- a/pims/base_frames.py +++ b/pims/base_frames.py @@ -627,25 +627,25 @@ def guess_axes(image): def default_axes(sizes, mode): - if mode == 'i': # single image + if 'i' in mode: # single image if sizes.get('z', 1) == 1 and sizes.get('t', 1) == 1: bundle_axes = 'yx' iter_axes = '' else: raise ValueError("This file cannot be opened as single image.") - elif mode == 'I': # multiple images + elif 'I' in mode: # multiple images if sizes.get('z', 1) == 1 and 't' in sizes: bundle_axes = 'yx' iter_axes = 't' else: raise ValueError("This file cannot be opened as multiple images.") - elif mode == 'v': # single volume + elif 'v' in mode: # single volume if 'z' in sizes and sizes.get('t', 1) == 1: bundle_axes = 'zyx' iter_axes = '' else: raise ValueError("This file cannot be opened as single volume.") - elif mode == 'v': # single volume + elif 'V' in mode: # multiple volumes if 'z' in sizes and 't' in sizes: bundle_axes = 'zyx' iter_axes = 't' @@ -684,7 +684,6 @@ def __init__(self, imageio_reader): super(WrapImageIOReader, self).__init__() self.rdr = imageio_reader - # TODO pass the dimension-awareness through this field try: info = self.rdr.pims_info except AttributeError: @@ -705,7 +704,7 @@ def __init__(self, imageio_reader): for axis, size in info['sizes'].items(): self._init_axis(axis, size) - for name, axes in info['read_methods']: + for name, axes in info['read_methods'].items(): method = getattr(self.rdr, name) self._register_get_frame(method, axes) diff --git a/pims/bioformats.py b/pims/bioformats.py index 4623d8dd..29c802a5 100644 --- a/pims/bioformats.py +++ b/pims/bioformats.py @@ -332,11 +332,7 @@ def _can_write(self, request): # propagate_attrs = ['frame_shape', 'pixel_type', 'metadata', # 'get_metadata_raw', 'reader_class_name'] - class BioformatsReader(Format.Reader): - @property - def pixel_type(self): - return self._pixel_type - + class Reader(Format.Reader): def _open(self, series=0, meta=True, java_memory='512m', read_mode='auto'): global loci @@ -444,6 +440,7 @@ def _open(self, series=0, meta=True, java_memory='512m', def _change_series(self): """Changes series and rereads axes, sizes and metadata. """ + self.pims_info = dict(read_methods=dict(), sizes=dict(), dtype=None) series = self._series self.rdr.setSeries(series) sizeX = self.rdr.getSizeX() @@ -466,12 +463,16 @@ def _change_series(self): self.pims_info['read_methods'] = {'get_frame_2D': 'yx'} self.pims_info['sizes'] = {'x': sizeX, 'y': sizeY} + self._len = 1 if sizeC > 1: self.pims_info['sizes']['c'] = sizeC + self._len *= sizeC if sizeT > 1: self.pims_info['sizes']['t'] = sizeT + self._len *= sizeT if sizeZ > 1: self.pims_info['sizes']['z'] = sizeZ + self._len *= sizeZ # determine pixel type pixel_type = self.rdr.getPixelType() @@ -503,9 +504,40 @@ def _change_series(self): except AttributeError: self.calibrationZ = None - def close(self): + def _close(self): self.rdr.close() + def _get_data(self, j): + if self.read_mode == 'jpype': + im = np.frombuffer(self.rdr.openBytes(j)[:], + dtype=self.pims_info['dtype']) + elif self.read_mode == 'stringbuffer': + im = self._jbytearr_stringbuffer(self.rdr.openBytes(j)) + elif self.read_mode == 'javacasting': + im = self._jbytearr_javacasting(self.rdr.openBytes(j)) + + im.shape = self._frame_shape_2D + return im, self._get_meta_data(j) + + def _get_meta_data(self, j): + if j is None: + return self.get_metadata_raw() + + z, c, t = self.rdr.getZCTCoords(j) + metadata = dict(frame=j, z=z, c=c, t=t, series=self._series) + if self.colors is not None: + metadata['colors'] = self.colors + if self.calibration is not None: + metadata['mpp'] = self.calibration + if self.calibrationZ is not None: + metadata['mppZ'] = self.calibrationZ + for key, method in self.frame_metadata.items(): + metadata[key] = getattr(self.metadata, method)(self._series, j) + return metadata + + def _get_length(self): + return self._len + @property def series(self): return self._series @@ -529,29 +561,7 @@ def get_frame_2D(self, **coords): _coords['c'] = 0 j = self.rdr.getIndex(int(_coords['z']), int(_coords['c']), int(_coords['t'])) - if self.read_mode == 'jpype': - im = np.frombuffer(self.rdr.openBytes(j)[:], - dtype=self._pixel_type) - elif self.read_mode == 'stringbuffer': - im = self._jbytearr_stringbuffer(self.rdr.openBytes(j)) - elif self.read_mode == 'javacasting': - im = self._jbytearr_javacasting(self.rdr.openBytes(j)) - - im.shape = self._frame_shape_2D - im = im.astype(self._pixel_type, copy=False) - - metadata = {'frame': j, - 'series': self._series} - if self.colors is not None: - metadata['colors'] = self.colors - if self.calibration is not None: - metadata['mpp'] = self.calibration - if self.calibrationZ is not None: - metadata['mppZ'] = self.calibrationZ - metadata.update(coords) - for key, method in self.frame_metadata.items(): - metadata[key] = getattr(self.metadata, method)(self._series, j) - + im, metadata = self._get_data(j) return Frame(im, metadata=metadata) def get_metadata_raw(self, form='dict'): diff --git a/pims/tests/test_bioformats.py b/pims/tests/test_bioformats.py index 68f651f0..b99677f9 100644 --- a/pims/tests/test_bioformats.py +++ b/pims/tests/test_bioformats.py @@ -109,7 +109,7 @@ def check_skip(self): def test_getting_stack(self): self.check_skip() - assert_equal(self.v[0].shape[-3], self.expected_Z) + assert_equal(self.v[0].shape[0], self.expected_Z) def test_sizeZ(self): self.check_skip() @@ -171,7 +171,7 @@ def setUp(self): self.klass = pims.Bioformats self.kwargs = {'meta': False} self.v = self.klass(self.filename, **self.kwargs) - self.expected_shape = (10, 31, 38) + self.expected_shape = (10, 31, 38, 2) self.expected_len = 3 self.expected_Z = 10 self.expected_C = 2 @@ -195,7 +195,7 @@ def setUp(self): self.klass = pims.Bioformats self.kwargs = {'meta': False} self.v = self.klass(self.filename, **self.kwargs) - self.expected_shape = (240, 320) + self.expected_shape = (240, 320, 3) self.expected_len = 108 self.expected_C = 3 @@ -220,7 +220,7 @@ def setUp(self): self.klass = pims.Bioformats self.kwargs = {'meta': False} self.v = self.klass(self.filename, **self.kwargs) - self.expected_shape = (24, 256, 256) + self.expected_shape = (24, 256, 256, 2) self.expected_len = 7 self.expected_C = 2 self.expected_Z = 24 @@ -267,7 +267,7 @@ def setUp(self): self.klass = pims.Bioformats self.kwargs = {'meta': False} self.v = self.klass(self.filename, **self.kwargs) - self.expected_shape = (21, 300, 400) + self.expected_shape = (21, 300, 400, 2) self.expected_len = 19 self.expected_C = 2 self.expected_Z = 21 @@ -292,7 +292,7 @@ def setUp(self): self.klass = pims.Bioformats self.kwargs = {'meta': False} self.v = self.klass(self.filename, **self.kwargs) - self.expected_shape = (4, 256, 256) + self.expected_shape = (4, 256, 256, 2) self.expected_len = 5 self.expected_C = 2 self.expected_Z = 4 @@ -345,7 +345,7 @@ def setUp(self): self.klass = pims.Bioformats self.kwargs = {'meta': False, 'series': 0} self.v = self.klass(self.filename, **self.kwargs) - self.expected_shape = (25, 512, 512) + self.expected_shape = (25, 512, 512, 4) self.expected_len = 1 self.expected_C = 4 self.expected_Z = 25 @@ -375,7 +375,7 @@ def setUp(self): self.klass = pims.Bioformats self.kwargs = {'meta': False, 'series': 1} self.v = self.klass(self.filename, **self.kwargs) - self.expected_shape = (46, 512, 512) + self.expected_shape = (46, 512, 512, 4) self.expected_len = 1 self.expected_C = 4 self.expected_Z = 46 @@ -491,7 +491,7 @@ def setUp(self): self.klass = pims.Bioformats self.kwargs = {'meta': False} self.v = self.klass(self.filename, **self.kwargs) - self.expected_shape = (29, 672, 512) + self.expected_shape = (29, 672, 512, 3) self.expected_C = 3 self.expected_Z = 29 From 70d3e405c5d1140790e29bc4943b26c889c6e8c2 Mon Sep 17 00:00:00 2001 From: caspervdw Date: Wed, 11 Oct 2017 12:35:50 +0200 Subject: [PATCH 12/14] REF Rebase pims.open on imageio --- pims/api.py | 83 +++++++++++++++++++---------------- pims/tests/test_bioformats.py | 2 +- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/pims/api.py b/pims/api.py index 96d828d8..087db0cd 100644 --- a/pims/api.py +++ b/pims/api.py @@ -182,51 +182,56 @@ def open(sequence, **kwargs): >>> frame_count = len(video) # Number of frames in video >>> frame_shape = video.frame_shape # Pixel dimensions of video """ + # check if it is an ImageSequence files = glob.glob(sequence) if len(files) > 1: # todo: test if ImageSequence can read the image type, # delegate to subclasses as needed return ImageSequence(sequence, **kwargs) - _, ext = os.path.splitext(sequence) - if ext is None or len(ext) < 2: - raise UnknownFormatError( - "Could not detect your file type because it did not have an " - "extension. Try specifying a loader class, e.g. " - "Video({0})".format(sequence)) - ext = ext.lower()[1:] - - # list all readers derived from the pims baseclasses - all_handlers = chain(_recursive_subclasses(FramesSequence), - _recursive_subclasses(FramesSequenceND)) - # keep handlers that support the file ext. use set to avoid duplicates. - eligible_handlers = set(h for h in all_handlers - if ext and ext in map(_drop_dot, h.class_exts())) - if len(eligible_handlers) < 1: - raise UnknownFormatError( - "Could not autodetect how to load a file of type {0}. " - "Try manually " - "specifying a loader class, e.g. Video({1})".format(ext, sequence)) - - def sort_on_priority(handlers): - # This uses optional priority information from subclasses - # > 10 means that it will be used instead of than built-in subclasses - def priority(cls): - try: - return cls.class_priority - except AttributeError: - return 10 - return sorted(handlers, key=priority, reverse=True) - - exceptions = '' - for handler in sort_on_priority(eligible_handlers): - try: - return handler(sequence, **kwargs) - except Exception as e: - message = '{0} errored: {1}'.format(str(handler), str(e)) - warn(message) - exceptions += message + '\n' - raise UnknownFormatError("All handlers returned exceptions:\n" + exceptions) + # the rest of the Reader choosing logic is handled by imageio + return WrapImageIOReader(get_reader(sequence, **kwargs)) + # + # + # _, ext = os.path.splitext(sequence) + # if ext is None or len(ext) < 2: + # raise UnknownFormatError( + # "Could not detect your file type because it did not have an " + # "extension. Try specifying a loader class, e.g. " + # "Video({0})".format(sequence)) + # ext = ext.lower()[1:] + # + # # list all readers derived from the pims baseclasses + # all_handlers = chain(_recursive_subclasses(FramesSequence), + # _recursive_subclasses(FramesSequenceND)) + # # keep handlers that support the file ext. use set to avoid duplicates. + # eligible_handlers = set(h for h in all_handlers + # if ext and ext in map(_drop_dot, h.class_exts())) + # if len(eligible_handlers) < 1: + # raise UnknownFormatError( + # "Could not autodetect how to load a file of type {0}. " + # "Try manually " + # "specifying a loader class, e.g. Video({1})".format(ext, sequence)) + # + # def sort_on_priority(handlers): + # # This uses optional priority information from subclasses + # # > 10 means that it will be used instead of than built-in subclasses + # def priority(cls): + # try: + # return cls.class_priority + # except AttributeError: + # return 10 + # return sorted(handlers, key=priority, reverse=True) + # + # exceptions = '' + # for handler in sort_on_priority(eligible_handlers): + # try: + # return handler(sequence, **kwargs) + # except Exception as e: + # message = '{0} errored: {1}'.format(str(handler), str(e)) + # warn(message) + # exceptions += message + '\n' + # raise UnknownFormatError("All handlers returned exceptions:\n" + exceptions) class UnknownFormatError(Exception): diff --git a/pims/tests/test_bioformats.py b/pims/tests/test_bioformats.py index b99677f9..28fd61e9 100644 --- a/pims/tests/test_bioformats.py +++ b/pims/tests/test_bioformats.py @@ -42,7 +42,7 @@ def test_bool(self): def test_open(self): self.v.close() - self.v = pims.open(self.filename) + self.v = pims.open(self.filename, format='Bioformats') def test_integer_attributes(self): self.check_skip() From ab4684910fed83d55094070dd9f594f14127d2ce Mon Sep 17 00:00:00 2001 From: caspervdw Date: Wed, 11 Oct 2017 13:19:15 +0200 Subject: [PATCH 13/14] CLN --- pims/tiff_stack.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pims/tiff_stack.py b/pims/tiff_stack.py index 0bfa89ba..afb39a98 100644 --- a/pims/tiff_stack.py +++ b/pims/tiff_stack.py @@ -131,9 +131,6 @@ def _close(self): # Note that the request object will close self._fp pass - def _get_pims_info(self): - return dict() - def _get_length(self): # Return the number of images. Can be np.inf return len(self._tiff) From 03d773fdf178912562f820962037423665e758c0 Mon Sep 17 00:00:00 2001 From: caspervdw Date: Wed, 11 Oct 2017 14:17:49 +0200 Subject: [PATCH 14/14] FIX Methods that change reader shape --- pims/base_frames.py | 8 +++++++ pims/bioformats.py | 42 +++++++++++++---------------------- pims/tests/test_bioformats.py | 8 ++++--- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/pims/base_frames.py b/pims/base_frames.py index f9fc2663..6ebe960e 100644 --- a/pims/base_frames.py +++ b/pims/base_frames.py @@ -683,6 +683,11 @@ def __init__(self, imageio_reader): super(WrapImageIOReader, self).__init__() self.rdr = imageio_reader + self.update_nd() + + def update_nd(self): + self._clear_axes() + self._get_frame_dict = dict() try: info = self.rdr.pims_info @@ -711,6 +716,9 @@ def __init__(self, imageio_reader): self.bundle_axes, self.iter_axes = default_axes(self.sizes, self.rdr.request.mode) + def __getattr__(self, key): + return getattr(self.rdr, key) + @property def pixel_type(self): return self._dtype diff --git a/pims/bioformats.py b/pims/bioformats.py index 29c802a5..20754110 100644 --- a/pims/bioformats.py +++ b/pims/bioformats.py @@ -327,10 +327,9 @@ def _can_write(self, request): this Format.Writer, judging from the imageio.core.Request object.""" return False - # TODO What to do with these?? - # class_priority = 2 - # propagate_attrs = ['frame_shape', 'pixel_type', 'metadata', - # 'get_metadata_raw', 'reader_class_name'] + # class_priority = 2 # Unused, ImageIO handles the reader choosing logic + propagate_attrs = ['frame_shape', 'pixel_type', 'metadata', + 'get_metadata_raw', 'reader_class_name'] class Reader(Format.Reader): def _open(self, series=0, meta=True, java_memory='512m', @@ -402,11 +401,11 @@ def _open(self, series=0, meta=True, java_memory='512m', # Set the correct series and initialize the sizes self.size_series = self.rdr.getSeriesCount() - if series >= self.size_series or series < 0: + self.series = None + try: + self.change_series(series) + except IndexError: self.rdr.close() - raise IndexError('Series index out of bounds.') - self._series = series - self._change_series() # Set read mode. When auto, tryout fast and check the image size. if read_mode == 'auto': @@ -437,11 +436,13 @@ def _open(self, series=0, meta=True, java_memory='512m', if hasattr(self.metadata, 'PlanePositionZ'): self.frame_metadata['z_um'] = 'PlanePositionZ' - def _change_series(self): - """Changes series and rereads axes, sizes and metadata. - """ + def change_series(self, series): + """Changes series and rereads axes, sizes and metadata.""" + if series >= self.size_series or series < 0: + raise IndexError('Series index out of bounds.') + self.series = series self.pims_info = dict(read_methods=dict(), sizes=dict(), dtype=None) - series = self._series + self.rdr.setSeries(series) sizeX = self.rdr.getSizeX() sizeY = self.rdr.getSizeY() @@ -524,7 +525,7 @@ def _get_meta_data(self, j): return self.get_metadata_raw() z, c, t = self.rdr.getZCTCoords(j) - metadata = dict(frame=j, z=z, c=c, t=t, series=self._series) + metadata = dict(frame=j, z=z, c=c, t=t, series=self.series) if self.colors is not None: metadata['colors'] = self.colors if self.calibration is not None: @@ -532,25 +533,12 @@ def _get_meta_data(self, j): if self.calibrationZ is not None: metadata['mppZ'] = self.calibrationZ for key, method in self.frame_metadata.items(): - metadata[key] = getattr(self.metadata, method)(self._series, j) + metadata[key] = getattr(self.metadata, method)(self.series, j) return metadata def _get_length(self): return self._len - @property - def series(self): - return self._series - - @series.setter - def series(self, value): - if value >= self.size_series or value < 0: - raise IndexError('Series index out of bounds.') - else: - if value != self._series: - self._series = value - self._change_series() - def get_frame_2D(self, **coords): """Actual reader, returns image as 2D numpy array and metadata as dict. diff --git a/pims/tests/test_bioformats.py b/pims/tests/test_bioformats.py index 28fd61e9..2bf1061f 100644 --- a/pims/tests/test_bioformats.py +++ b/pims/tests/test_bioformats.py @@ -354,7 +354,8 @@ def test_count_series(self): assert_equal(self.v.size_series, 2) def test_switch_series(self): - self.v.series = 1 + self.v.change_series(1) + self.v.update_nd() assert_equal(self.v.sizes['z'], 46) def tearDown(self): @@ -384,7 +385,7 @@ def tearDown(self): self.v.close() -class TestBioformatsIPL(_image_single, unittest.TestCase): +class TestBioformatsIPL(_image_single,_image_multichannel, unittest.TestCase): # IPLab format, 650 x 515 pixels, 8 bits per sample, 3 channels # Scanalytics has provided a sample multi-channel image in IPLab format. def check_skip(self): @@ -398,7 +399,8 @@ def setUp(self): self.klass = pims.Bioformats self.kwargs = {'meta': False} self.v = self.klass(self.filename, **self.kwargs) - self.expected_shape = (515, 650) + self.expected_shape = (515, 650, 3) + self.expected_C = 3 self.expected_len = 1 def tearDown(self):