001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.annotation.Annotation;
017import java.lang.reflect.Array;
018import java.lang.reflect.Field;
019import java.util.ArrayList;
020import java.util.Arrays;
021import java.util.Collection;
022import java.util.concurrent.ConcurrentMap;
023import java.util.concurrent.ConcurrentHashMap;
024import java.util.List;
025import java.util.Map;
026
027import org.eclipse.january.DatasetException;
028import org.eclipse.january.MetadataException;
029import org.eclipse.january.metadata.Dirtiable;
030import org.eclipse.january.metadata.ErrorMetadata;
031import org.eclipse.january.metadata.IMetadata;
032import org.eclipse.january.metadata.MetadataFactory;
033import org.eclipse.january.metadata.MetadataType;
034import org.eclipse.january.metadata.Reshapeable;
035import org.eclipse.january.metadata.Sliceable;
036import org.eclipse.january.metadata.Transposable;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 * Common base for both lazy and normal dataset implementations
042 */
043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
044
045        private static final long serialVersionUID = 767926846438976050L;
046
047        private static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
048
049        transient private boolean dirty = true; // indicate dirty state of metadata
050        protected String name = "";
051
052        /**
053         * The shape or dimensions of the dataset
054         */
055        protected int[] shape;
056
057        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
058
059        /**
060         * @return type of dataset item
061         */
062        abstract public int getDType();
063
064        @Override
065        public LazyDatasetBase clone() {
066                return null;
067        }
068
069        @Override
070        public boolean equals(Object obj) {
071                if (this == obj) {
072                        return true;
073                }
074                if (obj == null) {
075                        return false;
076                }
077                if (!getClass().equals(obj.getClass())) {
078                        return false;
079                }
080
081                LazyDatasetBase other = (LazyDatasetBase) obj;
082                if (getDType() != other.getDType()) {
083                        return false;
084                }
085                if (getElementsPerItem() != other.getElementsPerItem()) {
086                        return false;
087                }
088                if (!Arrays.equals(shape, other.shape)) {
089                        return false;
090                }
091                return true;
092        }
093
094        @Override
095        public int hashCode() {
096                int hash = getDType() * 17 + getElementsPerItem();
097                int rank = shape.length;
098                for (int i = 0; i < rank; i++) {
099                        hash = hash*17 + shape[i];
100                }
101                return hash;
102        }
103
104        @Override
105        public String getName() {
106                return name;
107        }
108
109        @Override
110        public void setName(String name) {
111                this.name = name;
112        }
113
114        @Override
115        public int[] getShape() {
116                return shape.clone();
117        }
118
119        @Override
120        public int getRank() {
121                return shape.length;
122        }
123
124        /**
125         * This method allows anything that dirties the dataset to clear various metadata values
126         * so that the other methods can work correctly.
127         * @since 2.1
128         */
129        public void setDirty() {
130                dirty = true;
131        }
132
133        protected void checkSliceND(SliceND slice) {
134                if (slice != null) {
135                        int[] source = slice.getSourceShape();
136                        boolean fail = false;
137                        if (slice.isExpanded()) {
138                                fail = shape.length != source.length;
139                                if (!fail) {
140                                        for (int i = 0; i < shape.length; i++) {
141                                                if (shape[i] > source[i]) {
142                                                        fail = true;
143                                                        break;
144                                                }
145                                        }
146                                }
147                        } else {
148                                fail = !Arrays.equals(shape, source);
149                        }
150                        if (fail) {
151                                throw new IllegalArgumentException("Slice's shape must match dataset's shape");
152                        }
153                }
154        }
155
156        /**
157         * Find first sub-interface of (or class that directly implements) MetadataType
158         * @param clazz metadata type
159         * @return sub-interface
160         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
161         */
162        @SuppressWarnings("unchecked")
163        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
164                if (clazz.equals(MetadataType.class)) {
165                        throw new IllegalArgumentException("Cannot accept MetadataType");
166                }
167
168                if (clazz.isInterface()) {
169                        return clazz;
170                }
171
172                if (clazz.isAnonymousClass()) { // special case
173                        Class<?> s = clazz.getSuperclass();
174                        if (!s.equals(Object.class)) {
175                                // only use super class if it is not an anonymous class of an interface
176                                clazz = (Class<? extends MetadataType>) s;
177                        }
178                }
179
180                for (Class<?> c : clazz.getInterfaces()) {
181                        if (c.equals(MetadataType.class)) {
182                                if (clazz.isAnonymousClass()) {
183                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
184                                }
185                                return clazz;
186                        }
187                        if (MetadataType.class.isAssignableFrom(c)) {
188                                return (Class<? extends MetadataType>) c;
189                        }
190                }
191
192                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
193                if (c != null) {
194                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
195                }
196
197                logger.error("Somehow the search for metadata type interface ended in a bad place");
198                assert false; // should not be able to get here!!!
199                return null;
200        }
201
202        @Override
203        public void setMetadata(MetadataType metadata) {
204                addMetadata(metadata, true);
205        }
206
207        @Override
208        public void addMetadata(MetadataType metadata) {
209                addMetadata(metadata, false);
210        }
211
212        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
213                if (metadata == null) {
214                        return;
215                }
216
217                if (this.metadata == null) {
218                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
219                }
220
221                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
222                if (!this.metadata.containsKey(clazz)) {
223                        this.metadata.put(clazz, new ArrayList<MetadataType>());
224                } else if (clear) {
225                        this.metadata.get(clazz).clear();
226                }
227                this.metadata.get(clazz).add(metadata);
228
229                // add for special case of sub-interfaces of IMetadata
230                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
231                        clazz = IMetadata.class;
232                        if (!this.metadata.containsKey(clazz)) {
233                                this.metadata.put(clazz, new ArrayList<MetadataType>());
234                        } else if (clear) {
235                                this.metadata.get(clazz).clear();
236                        }
237                        this.metadata.get(clazz).add(metadata);
238                }
239        }
240
241        @Override
242        @Deprecated
243        public synchronized IMetadata getMetadata() {
244                return getFirstMetadata(IMetadata.class);
245        }
246
247        @SuppressWarnings("unchecked")
248        @Override
249        public synchronized <T extends MetadataType> List<T> getMetadata(Class<T> clazz) throws MetadataException {
250                if (metadata == null) {
251                        dirty = false;
252                        return null;
253                }
254
255                if (dirty) {
256                        dirtyMetadata();
257                        dirty = false;
258                }
259
260                if (clazz == null) {
261                        List<T> all = new ArrayList<>();
262                        for (Class<? extends MetadataType> c : metadata.keySet()) {
263                                all.addAll((Collection<T>) metadata.get(c));
264                        }
265                        return all;
266                }
267
268                return (List<T>) metadata.get(findMetadataTypeSubInterfaces(clazz));
269        }
270
271        @Override
272        public synchronized <T extends MetadataType> T getFirstMetadata(Class<T> clazz) {
273                try {
274                        List<T> ml = getMetadata(clazz);
275                        if (ml == null) {
276                                return null;
277                        }
278                        for (T t : ml) {
279                                if (clazz.isInstance(t)) {
280                                        return t;
281                                }
282                        }
283                } catch (Exception e) {
284                        logger.error("Get metadata failed!",e);
285                }
286
287                return null;
288        }
289
290        @Override
291        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
292                if (metadata == null) {
293                        return;
294                }
295
296                if (clazz == null) {
297                        metadata.clear();
298                        return;
299                }
300
301                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
302                if( list != null) {
303                        list.clear();
304                }
305        }
306
307        /**
308         * @return copy of metadata
309         * @since 2.0
310         */
311        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
312                return copyMetadata(metadata);
313        }
314
315        /**
316         * @param metadata type
317         * @return copy of metadata of given type
318         * @since 2.0
319         */
320        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
321                if (metadata == null) {
322                        return null;
323                }
324
325                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
326                copyMetadata(metadata, map);
327                return map;
328        }
329
330        private static void copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> inMetadata,
331                        Map<Class<? extends MetadataType>, List<MetadataType>> outMetadata) {
332                for (Class<? extends MetadataType> c : inMetadata.keySet()) {
333                        List<MetadataType> l = inMetadata.get(c);
334                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
335                        outMetadata.put(c, nl);
336                        for (MetadataType m : l) {
337                                if (m == null || isMetadataDirty(m)) { // skip dirty metadata
338                                        continue;
339                                }
340                                nl.add(m.clone());
341                        }
342                }
343        }
344
345        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
346                copyMetadata(oldMetadata, metadata);
347        }
348
349        /**
350         * @param a dataset
351         * @param clone if true, copy metadata
352         * @return copy of metadata
353         * @since 2.2
354         */
355        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(ILazyDataset a, boolean clone) {
356                List<MetadataType> all = null;
357                try {
358                        all = a.getMetadata(null);
359                } catch (Exception e) {
360                }
361                if (all == null) {
362                        return null;
363                }
364
365                ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
366
367                for (MetadataType m : all) {
368                        if (m == null || isMetadataDirty(m)) { // skip dirty metadata
369                                continue;
370                        }
371                        Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass());
372                        List<MetadataType> l = map.get(c);
373                        if (l == null) {
374                                l = new ArrayList<MetadataType>();
375                                map.put(c, l);
376                        }
377                        if (clone) {
378                                m = m.clone();
379                        }
380                        l.add(m);
381                }
382                return map;
383        }
384
385        private static boolean isMetadataDirty(MetadataType m) {
386                Class<? extends MetadataType> c = m.getClass();
387                for (Field f : c.getDeclaredFields()) {
388                        if (f.isAnnotationPresent(Dirtiable.class)) {
389                                Class<?> t = f.getType();
390                                if (t.equals(boolean.class) || t.equals(Boolean.class)) {
391                                        try {
392                                                f.setAccessible(true);
393                                                Object o = f.get(m);
394                                                if (o.equals(true)) {
395                                                        return true;
396                                                }
397                                        } catch (Exception e) {
398                                                logger.debug("Could not retrieve value of dirty variable: {}", c.getCanonicalName(), e);
399                                        }
400                                }
401                        }
402                }
403
404                return false;
405        }
406
407        interface MetadatasetAnnotationOperation {
408                /**
409                 * Process value of given field
410                 * <p>
411                 * When the field is not a container then the returned value
412                 * may replace the old value
413                 * @param f given field
414                 * @param o value of field
415                 * @return transformed field
416                 */
417                Object processField(Field f, Object o);
418
419                /**
420                 * @return annotated class
421                 */
422                Class<? extends Annotation> getAnnClass();
423
424                /**
425                 * @param axis
426                 * @return number of dimensions to insert or remove
427                 */
428                int change(int axis);
429
430                /**
431                 * 
432                 * @return rank or -1 to match
433                 */
434                int getNewRank();
435
436                /**
437                 * Run on given lazy dataset
438                 * @param lz
439                 * @return 
440                 */
441                ILazyDataset run(ILazyDataset lz);
442        }
443
444        class MdsSlice implements MetadatasetAnnotationOperation {
445                private boolean asView;
446                private SliceND slice;
447                private int[] oShape;
448                private long oSize;
449
450                public MdsSlice(boolean asView, SliceND slice) {
451                        this.asView = asView;
452                        this.slice = slice;
453                        oShape = slice.getSourceShape();
454                        oSize = ShapeUtils.calcLongSize(oShape);
455                }
456
457                @Override
458                public Object processField(Field field, Object o) {
459                        return o;
460                }
461
462                @Override
463                public Class<? extends Annotation> getAnnClass() {
464                        return Sliceable.class;
465                }
466
467                @Override
468                public int change(int axis) {
469                        return 0;
470                }
471
472                @Override
473                public int getNewRank() {
474                        return -1;
475                }
476
477                @Override
478                public ILazyDataset run(ILazyDataset lz) {
479                        int rank = lz.getRank();
480                        if (slice.getStart().length != rank) {
481                                throw new IllegalArgumentException("Slice rank does not match dataset!");
482                        }
483
484                        int[] shape = lz.getShape();
485                        SliceND nslice;
486                        if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) {
487                                nslice = new SliceND(shape);
488                                for (int i = 0; i < rank; i++) {
489                                        int s = shape[i];
490                                        int os = oShape[i];
491                                        if (s >= os) {
492                                                nslice.setSlice(i, 0, os, 1);
493                                        } else if (s == 1) {
494                                                nslice.setSlice(i, 0, 1, 1);
495                                        } else {
496                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
497                                        }
498                                }
499                                lz = lz.getSliceView(nslice);
500                                shape = nslice.getShape();
501                        }
502                        if (lz.getSize() == oSize && Arrays.equals(shape, oShape)) {
503                                nslice = slice;
504                        } else {
505                                nslice = slice.clone();
506                                for (int i = 0; i < rank; i++) {
507                                        int s = shape[i];
508                                        if (s >= oShape[i]) {
509                                                continue;
510                                        } else if (s == 1) {
511                                                nslice.setSlice(i, 0, 1, 1);
512                                        } else {
513                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
514                                        }
515                                }
516                                nslice.updateSourceShape(shape);
517                        }
518
519                        if (asView || (lz instanceof IDataset)) {
520                                return lz.getSliceView(nslice);
521                        }
522                        try {
523                                return lz.getSlice(nslice);
524                        } catch (DatasetException e) {
525                                logger.error("Could not slice dataset in metadata", e);
526                                return null;
527                        }
528                }
529        }
530
531        class MdsReshape implements MetadatasetAnnotationOperation {
532                private boolean matchRank;
533                private int[] oldShape;
534                private int[] newShape;
535                boolean onesOnly;
536                int[] differences;
537
538                /*
539                 * if only ones then record differences (insertions and deletions)
540                 * 
541                 * if shape changing, find broadcasted dimensions and disallow
542                 * merging that include those dimensions
543                 */
544                public MdsReshape(final int[] oldShape, final int[] newShape) {
545                        this.oldShape = oldShape;
546                        this.newShape = newShape;
547                        differences = null;
548                }
549
550                @Override
551                public Object processField(Field field, Object o) {
552                        Annotation a = field.getAnnotation(Reshapeable.class);
553                        if (a != null) { // cannot be null
554                                matchRank = ((Reshapeable) a).matchRank();
555                        }
556                        return o;
557                }
558
559                @Override
560                public Class<? extends Annotation> getAnnClass() {
561                        return Reshapeable.class;
562                }
563
564                @Override
565                public int change(int axis) {
566                        if (matchRank) {
567                                if (differences == null) {
568                                        init();
569                                }
570
571                                if (onesOnly) {
572                                        return differences == null ? 0 : differences[axis];
573                                }
574                                throw new UnsupportedOperationException("TODO support other shape operations");
575                        }
576                        return 0;
577                }
578
579                @Override
580                public int getNewRank() {
581                        return matchRank ? newShape.length : -1;
582                }
583
584                private void init() {
585                        int or = oldShape.length - 1;
586                        int nr = newShape.length - 1;
587                        if (or < 0 || nr < 0) { // zero-rank shapes
588                                onesOnly = true;
589                                differences = new int[1];
590                                differences[0] = or < 0 ? nr + 1 : or + 1;
591                                return;
592                        }
593                        onesOnly = ShapeUtils.differsByOnes(oldShape, newShape);
594                        int ob = 0;
595                        int nb = 0;
596                        if (onesOnly) {
597                                differences = ShapeUtils.calcShapePadding(oldShape, newShape);
598                        } else {
599                                differences = new int[or + 2];
600                                if (matchRank) {
601                                        logger.error("Combining dimensions is currently not supported");
602                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
603                                }
604                                // work out mapping: contiguous dimensions can be grouped or split
605                                while (ob <= or && nb <= nr) {
606                                        int ol = oldShape[ob];
607                                        while (ol == 1 && ol <= or) {
608                                                ob++;
609                                                ol = oldShape[ob];
610                                        }
611                                        int oe = ob + 1;
612                                        int nl = newShape[nb];
613                                        while (nl == 1 && nl <= nr) {
614                                                nb++;
615                                                nl = newShape[nb];
616                                        }
617                                        int ne = nb + 1;
618                                        if (ol < nl) {
619                                                differences[ob] = 1;
620                                                do { // case where new shape combines several dimensions into one dimension
621                                                        if (oe == (or + 1)) {
622                                                                break;
623                                                        }
624                                                        differences[oe] = 1;
625                                                        ol *= oldShape[oe++];
626                                                } while (ol < nl);
627                                                differences[oe - 1] = oe - ob; // signal end with difference
628                                                if (nl != ol) {
629                                                        logger.error("Single dimension is incompatible with subshape");
630                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
631                                                }
632                                        } else if (ol > nl) {
633                                                do { // case where new shape spreads single dimension over several dimensions
634                                                        if (ne == (nr + 1)) {
635                                                                break;
636                                                        }
637                                                        nl *= newShape[ne++];
638                                                } while (nl < ol);
639                                                if (nl != ol) {
640                                                        logger.error("Subshape is incompatible with single dimension");
641                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
642                                                }
643                                        }
644
645                                        ob = oe;
646                                        nb = ne;
647                                }
648                        }
649                }
650
651                @Override
652                public ILazyDataset run(ILazyDataset lz) {
653                        if (differences == null) {
654                                init();
655                        }
656
657                        int[] lshape = lz.getShape();
658                        if (Arrays.equals(newShape, lshape)) {
659                                return lz;
660                        }
661                        int or = lshape.length;
662                        int nr = newShape.length;
663                        int[] nshape;
664                        if (onesOnly) {
665                                nshape = ShapeUtils.padShape(differences, nr, lshape);
666                        } else {
667                                nshape = new int[nr];
668                                boolean[] broadcast = new boolean[or];
669                                for (int ob = 0; ob < or; ob++) {
670                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
671                                }
672                                int osize = lz.getSize();
673
674                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
675                                int ob = 0;
676                                int nsize = 1;
677                                for (int i = 0; i < nr; i++) {
678                                        if (ob < or && broadcast[ob]) {
679                                                if (differences[ob] != 0) {
680                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
681                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
682                                                }
683                                                nshape[i] = 1;
684                                        } else {
685                                                nshape[i] = nsize < osize ? newShape[i] : 1;
686                                        }
687                                        nsize *= nshape[i];
688                                        ob++;
689                                }
690                        }
691
692                        ILazyDataset nlz;
693                        if (lz instanceof Dataset) {
694                                nlz = ((Dataset) lz).reshape(nshape);
695                        } else {
696                                nlz = lz.getSliceView();
697                                nlz.setShape(nshape);
698                        }
699                        return nlz;
700                }
701        }
702
703        class MdsTranspose implements MetadatasetAnnotationOperation {
704                int[] map;
705
706                public MdsTranspose(final int[] axesMap) {
707                        map = axesMap;
708                }
709
710                @SuppressWarnings({ "rawtypes", "unchecked" })
711                @Override
712                public Object processField(Field f, Object o) {
713                        // reorder arrays and lists according the axes map
714                        if (o.getClass().isArray()) {
715                                int l = Array.getLength(o);
716                                if (l == map.length) {
717                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
718                                        for (int i = 0; i < l; i++) {
719                                                Array.set(narray, i, Array.get(o, map[i]));
720                                        }
721                                        for (int i = 0; i < l; i++) {
722                                                Array.set(o, i, Array.get(narray, i));
723                                        }
724                                }
725                        } else if (o instanceof List<?>) {
726                                List list = (List) o;
727                                int l = list.size();
728                                if (l == map.length) {
729                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
730                                        for (int i = 0; i < l; i++) {
731                                                Array.set(narray, i, list.get(map[i]));
732                                        }
733                                        list.clear();
734                                        for (int i = 0; i < l; i++) {
735                                                list.add(Array.get(narray, i));
736                                        }
737                                }
738                        }
739                        return o;
740                }
741
742                @Override
743                public Class<? extends Annotation> getAnnClass() {
744                        return Transposable.class;
745                }
746
747                @Override
748                public int change(int axis) {
749                        return 0;
750                }
751
752                @Override
753                public int getNewRank() {
754                        return -1;
755                }
756
757                @Override
758                public ILazyDataset run(ILazyDataset lz) {
759                        return lz.getTransposedView(map);
760                }
761        }
762
763        class MdsDirty implements MetadatasetAnnotationOperation {
764
765                @Override
766                public Object processField(Field f, Object o) {
767                        // throw exception if not boolean???
768                        Class<?> t = f.getType();
769                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
770                                if (o.equals(false)) {
771                                        o = true;
772                                }
773                        }
774                        return o;
775                }
776
777                @Override
778                public Class<? extends Annotation> getAnnClass() {
779                        return Dirtiable.class;
780                }
781
782                @Override
783                public int change(int axis) {
784                        return 0;
785                }
786
787                @Override
788                public int getNewRank() {
789                        return -1;
790                }
791
792                @Override
793                public ILazyDataset run(ILazyDataset lz) {
794                        return lz;
795                }
796        }
797
798        /**
799         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
800         * dataset after cloning the metadata
801         * @param asView if true then just a view
802         * @param slice an n-D slice
803         */
804        protected void sliceMetadata(boolean asView, final SliceND slice) {
805                processAnnotatedMetadata(new MdsSlice(asView, slice));
806        }
807
808        /**
809         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
810         * or setting the shape
811         * @param oldShape old shape
812         * @param newShape new shape
813         */
814        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
815                processAnnotatedMetadata(new MdsReshape(oldShape, newShape));
816        }
817
818        /**
819         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
820         * dataset after cloning the metadata
821         * @param axesMap if zero length then axes order reversed
822         */
823        protected void transposeMetadata(final int[] axesMap) {
824                processAnnotatedMetadata(new MdsTranspose(axesMap));
825        }
826
827        /**
828         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
829         * @since 2.0
830         */
831        protected void dirtyMetadata() {
832                processAnnotatedMetadata(new MdsDirty());
833        }
834
835        @SuppressWarnings("unchecked")
836        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op) {
837                if (metadata == null)
838                        return;
839
840                for (List<MetadataType> l : metadata.values()) {
841                        for (MetadataType m : l) {
842                                if (m == null) {
843                                        continue;
844                                }
845
846                                Class<? extends MetadataType> mc = m.getClass();
847                                do { // iterate over super-classes
848                                        processClass(op, m, mc);
849                                        Class<?> sclazz = mc.getSuperclass();
850                                        if (!MetadataType.class.isAssignableFrom(sclazz)) {
851                                                break;
852                                        }
853                                        mc = (Class<? extends MetadataType>) sclazz;
854                                } while (true);
855                        }
856                }
857        }
858
859        @SuppressWarnings({ "unchecked", "rawtypes" })
860        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc) {
861                for (Field f : mc.getDeclaredFields()) {
862                        if (!f.isAnnotationPresent(op.getAnnClass()))
863                                continue;
864
865                        try {
866                                f.setAccessible(true);
867                                Object o = f.get(m);
868                                if (o == null) {
869                                        continue;
870                                }
871
872                                Object no = op.processField(f, o);
873                                if (no != o) {
874                                        f.set(m, no);
875                                        continue;
876                                }
877                                Object r = null;
878                                if (o instanceof ILazyDataset) {
879                                        try {
880                                                f.set(m, op.run((ILazyDataset) o));
881                                        } catch (Exception e) {
882                                                logger.error("Problem processing " + o, e);
883                                                throw e;
884                                        }
885                                } else if (o.getClass().isArray()) {
886                                        int l = Array.getLength(o);
887
888                                        for (int i = 0; r == null && i < l; i++) {
889                                                r = Array.get(o, i);
890                                        }
891                                        int n = op.getNewRank();
892                                        if (r == null) {
893                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
894                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
895                                                }
896                                                continue;
897                                        }
898                                        if (n < 0) {
899                                                n = l;
900                                        }
901                                        Object narray = Array.newInstance(r.getClass(), n);
902                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
903                                                int c = op.change(i);
904                                                if (c == 0) {
905                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
906                                                } else if (c > 0) {
907                                                        di += c; // add nulls by skipping forward in destination array
908                                                } else if (c < 0) {
909                                                        si -= c; // remove dimensions by skipping forward in source array
910                                                }
911                                        }
912                                        if (n == l) {
913                                                for (int i = 0; i < l; i++) {
914                                                        Array.set(o, i, Array.get(narray, i));
915                                                }
916                                        } else {
917                                                f.set(m, narray);
918                                        }
919                                } else if (o instanceof List<?>) {
920                                        List list = (List) o;
921                                        int l = list.size();
922
923                                        for (int i = 0; r == null && i < l; i++) {
924                                                r = list.get(i);
925                                        }
926                                        int n = op.getNewRank();
927                                        if (r == null) {
928                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
929                                                        list.clear();
930                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
931                                                                list.add(null);
932                                                        }
933                                                }
934                                                continue;
935                                        }
936
937                                        if (n < 0) {
938                                                n = l;
939                                        }
940                                        Object narray = Array.newInstance(r.getClass(), n);
941                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
942                                                int c = op.change(i);
943                                                if (c == 0) {
944                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
945                                                } else if (c > 0) {
946                                                        di += c; // add nulls by skipping forward in destination array
947                                                } else if (c < 0) {
948                                                        si -= c; // remove dimensions by skipping forward in source array
949                                                }
950                                        }
951                                        list.clear();
952                                        for (int i = 0; i < n; i++) {
953                                                list.add(Array.get(narray, i));
954                                        }
955                                } else if (o instanceof Map<?,?>) {
956                                        Map map = (Map) o;
957                                        for (Object k : map.keySet()) {
958                                                map.put(k, processObject(op, map.get(k)));
959                                        }
960                                }
961                        } catch (Exception e) {
962                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
963                                throw new RuntimeException(e);
964                        }
965                }
966        }
967
968        @SuppressWarnings({ "unchecked", "rawtypes" })
969        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
970                if (o == null) {
971                        return o;
972                }
973
974                if (o instanceof ILazyDataset) {
975                        try {
976                                return op.run((ILazyDataset) o);
977                        } catch (Exception e) {
978                                logger.error("Problem processing " + o, e);
979                                throw e;
980                        }
981                } else if (o.getClass().isArray()) {
982                        int l = Array.getLength(o);
983                        for (int i = 0; i < l; i++) {
984                                Array.set(o, i, processObject(op, Array.get(o, i)));
985                        }
986                } else if (o instanceof List<?>) {
987                        List list = (List) o;
988                        for (int i = 0, imax = list.size(); i < imax; i++) {
989                                list.set(i, processObject(op, list.get(i)));
990                        }
991                } else if (o instanceof Map<?,?>) {
992                        Map map = (Map) o;
993                        for (Object k : map.keySet()) {
994                                map.put(k, processObject(op, map.get(k)));
995                        }
996                }
997                return o;
998        }
999
1000        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
1001                ILazyDataset d = null;
1002                if (blob instanceof ILazyDataset) {
1003                        d = (ILazyDataset) blob;
1004                        if (d instanceof IDataset) {
1005                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
1006                                int is = ed.getElementsPerItem();
1007                                if (is != 1 && is != getElementsPerItem()) {
1008                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
1009                                }
1010                                d = ed.cast(is == 1 ? DoubleDataset.class: CompoundDoubleDataset.class);
1011                        } else if (!keepLazy) {
1012                                final int is = getElementsPerItem();
1013                                try {
1014                                        d = DatasetUtils.cast(is == 1 ? DoubleDataset.class: CompoundDoubleDataset.class, d.getSlice());
1015                                } catch (DatasetException e) {
1016                                        logger.error("Could not get data from lazy dataset", e);
1017                                        return null;
1018                                }
1019                        }
1020                } else {
1021                        final int is = getElementsPerItem();
1022                        if (is == 1) {
1023                                d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
1024                        } else {
1025                                try {
1026                                        d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob);
1027                                } catch (IllegalArgumentException e) { // if only single value supplied try again
1028                                        d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
1029                                }
1030                        }
1031                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
1032                                d.setShape(shape.clone());
1033                        }
1034                }
1035                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
1036                d.setShape(s.get(0));
1037
1038                return d;
1039        }
1040
1041        @Override
1042        public void setErrors(Serializable errors) {
1043                if (shape == null) {
1044                        throw new IllegalArgumentException("Cannot set errors for null dataset");
1045                }
1046                if (errors == null) {
1047                        clearMetadata(ErrorMetadata.class);
1048                        return;
1049                }
1050                if (errors == this) {
1051                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
1052                        return;
1053                }
1054
1055                ILazyDataset errorData = createFromSerializable(errors, true);
1056
1057                ErrorMetadata emd = getErrorMetadata();
1058                if (emd == null) {
1059                        try {
1060                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1061                                setMetadata(emd);
1062                        } catch (MetadataException me) {
1063                                logger.error("Could not create metadata", me);
1064                        }
1065                }
1066                emd.setError(errorData);
1067        }
1068
1069        protected ErrorMetadata getErrorMetadata() {
1070                try {
1071                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
1072                        if (el != null && !el.isEmpty()) {
1073                                 return el.get(0);
1074                        }
1075                } catch (Exception e) {
1076                }
1077                return null;
1078        }
1079
1080        @Override
1081        public ILazyDataset getErrors() {
1082                ErrorMetadata emd = getErrorMetadata();
1083                return emd == null ? null : emd.getError();
1084        }
1085
1086        @Override
1087        public boolean hasErrors() {
1088                return LazyDatasetBase.this.getErrors() != null;
1089        }
1090
1091        /**
1092         * Check permutation axes
1093         * @param shape to use
1094         * @param axes if zero length then axes order reversed
1095         * @return cleaned up copy of axes or null if trivial
1096         */
1097        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
1098                int rank = shape == null ? 0 : shape.length;
1099
1100                if (axes == null || axes.length == 0) {
1101                        axes = new int[rank];
1102                        for (int i = 0; i < rank; i++) {
1103                                axes[i] = rank - 1 - i;
1104                        }
1105                } else {
1106                        axes = axes.clone();
1107                }
1108
1109                if (axes.length != rank) {
1110                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
1111                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
1112                }
1113        
1114                // check all permutation values are within bounds
1115                for (int i = 0; i < rank; i++) {
1116                        axes[i] = ShapeUtils.checkAxis(rank, axes[i]);
1117                }
1118        
1119                // check for a valid permutation (is this an unnecessary restriction?)
1120                int[] perm = axes.clone();
1121                Arrays.sort(perm);
1122
1123                for (int i = 0; i < rank; i++) {
1124                        if (perm[i] != i) {
1125                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
1126                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");
1127                        }
1128                }
1129
1130                if (Arrays.equals(axes, perm)) {
1131                        return null; // signal identity or trivial permutation
1132                }
1133
1134                return axes;
1135        }
1136}