001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.annotation.Annotation; 017import java.lang.reflect.Array; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.Collection; 022import java.util.concurrent.ConcurrentMap; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.List; 025import java.util.Map; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.MetadataException; 029import org.eclipse.january.metadata.Dirtiable; 030import org.eclipse.january.metadata.ErrorMetadata; 031import org.eclipse.january.metadata.IMetadata; 032import org.eclipse.january.metadata.MetadataFactory; 033import org.eclipse.january.metadata.MetadataType; 034import org.eclipse.january.metadata.Reshapeable; 035import org.eclipse.january.metadata.Sliceable; 036import org.eclipse.january.metadata.Transposable; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * Common base for both lazy and normal dataset implementations 042 */ 043public abstract class LazyDatasetBase implements ILazyDataset, Serializable { 044 045 private static final long serialVersionUID = 767926846438976050L; 046 047 protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class); 048 049 protected static boolean catchExceptions; 050 051 static { 052 /** 053 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE 054 */ 055 try { 056 catchExceptions = Boolean.getBoolean("run.in.eclipse"); 057 } catch (SecurityException e) { 058 // set a default for when the security manager does not allow access to the requested key 059 catchExceptions = false; 060 } 061 } 062 063 transient private boolean dirty = true; // indicate dirty state of metadata 064 protected String name = ""; 065 066 /** 067 * The shape or dimensions of the dataset 068 */ 069 protected int[] shape; 070 071 protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null; 072 073 /** 074 * @return type of dataset item 075 */ 076 abstract public int getDType(); 077 078 @Override 079 public Class<?> getElementClass() { 080 return DTypeUtils.getElementClass(getDType()); 081 } 082 083 @Override 084 public LazyDatasetBase clone() { 085 return null; 086 } 087 088 @Override 089 public boolean equals(Object obj) { 090 if (this == obj) { 091 return true; 092 } 093 if (obj == null) { 094 return false; 095 } 096 if (!getClass().equals(obj.getClass())) { 097 return false; 098 } 099 100 LazyDatasetBase other = (LazyDatasetBase) obj; 101 if (getDType() != other.getDType()) { 102 return false; 103 } 104 if (getElementsPerItem() != other.getElementsPerItem()) { 105 return false; 106 } 107 if (!Arrays.equals(shape, other.shape)) { 108 return false; 109 } 110 return true; 111 } 112 113 @Override 114 public int hashCode() { 115 int hash = getDType() * 17 + getElementsPerItem(); 116 int rank = shape.length; 117 for (int i = 0; i < rank; i++) { 118 hash = hash*17 + shape[i]; 119 } 120 return hash; 121 } 122 123 @Override 124 public String getName() { 125 return name; 126 } 127 128 @Override 129 public void setName(String name) { 130 this.name = name; 131 } 132 133 @Override 134 public int[] getShape() { 135 return shape.clone(); 136 } 137 138 @Override 139 public int getRank() { 140 return shape.length; 141 } 142 143 /** 144 * This method allows anything that dirties the dataset to clear various metadata values 145 * so that the other methods can work correctly. 146 * @since 2.1 147 */ 148 public void setDirty() { 149 dirty = true; 150 } 151 152 /** 153 * Find first sub-interface of (or class that directly implements) MetadataType 154 * @param clazz 155 * @return sub-interface 156 * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it 157 */ 158 @SuppressWarnings("unchecked") 159 public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) { 160 if (clazz.equals(MetadataType.class)) { 161 throw new IllegalArgumentException("Cannot accept MetadataType"); 162 } 163 164 if (clazz.isInterface()) { 165 return clazz; 166 } 167 168 if (clazz.isAnonymousClass()) { // special case 169 Class<?> s = clazz.getSuperclass(); 170 if (!s.equals(Object.class)) { 171 // only use super class if it is not an anonymous class of an interface 172 clazz = (Class<? extends MetadataType>) s; 173 } 174 } 175 176 for (Class<?> c : clazz.getInterfaces()) { 177 if (c.equals(MetadataType.class)) { 178 if (clazz.isAnonymousClass()) { 179 throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType"); 180 } 181 return clazz; 182 } 183 if (MetadataType.class.isAssignableFrom(c)) { 184 return (Class<? extends MetadataType>) c; 185 } 186 } 187 188 Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class 189 if (c != null) { 190 return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c); 191 } 192 193 logger.error("Somehow the search for metadata type interface ended in a bad place"); 194 assert false; // should not be able to get here!!! 195 return null; 196 } 197 198 @Override 199 public void setMetadata(MetadataType metadata) { 200 addMetadata(metadata, true); 201 } 202 203 @Override 204 public void addMetadata(MetadataType metadata) { 205 addMetadata(metadata, false); 206 } 207 208 private synchronized void addMetadata(MetadataType metadata, boolean clear) { 209 if (metadata == null) 210 return; 211 212 if (this.metadata == null) { 213 this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 214 } 215 216 Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass()); 217 if (!this.metadata.containsKey(clazz)) { 218 this.metadata.put(clazz, new ArrayList<MetadataType>()); 219 } else if (clear) { 220 this.metadata.get(clazz).clear(); 221 } 222 this.metadata.get(clazz).add(metadata); 223 224 // add for special case of sub-interfaces of IMetadata 225 if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) { 226 clazz = IMetadata.class; 227 if (!this.metadata.containsKey(clazz)) { 228 this.metadata.put(clazz, new ArrayList<MetadataType>()); 229 } else if (clear) { 230 this.metadata.get(clazz).clear(); 231 } 232 this.metadata.get(clazz).add(metadata); 233 } 234 } 235 236 @Override 237 @Deprecated 238 public synchronized IMetadata getMetadata() { 239 return getFirstMetadata(IMetadata.class); 240 } 241 242 @SuppressWarnings("unchecked") 243 @Override 244 public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException { 245 if (metadata == null) { 246 dirty = false; 247 return null; 248 } 249 250 if (dirty) { 251 dirtyMetadata(); 252 dirty = false; 253 } 254 255 if (clazz == null) { 256 List<S> all = new ArrayList<S>(); 257 for (Class<? extends MetadataType> c : metadata.keySet()) { 258 all.addAll((Collection<S>) metadata.get(c)); 259 } 260 return all; 261 } 262 263 return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz)); 264 } 265 266 @Override 267 public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) { 268 try { 269 List<S> ml = getMetadata(clazz); 270 if (ml == null) { 271 return null; 272 } 273 for (S t : ml) { 274 if (clazz.isInstance(t)) { 275 return t; 276 } 277 } 278 } catch (Exception e) { 279 logger.error("Get metadata failed!",e); 280 } 281 282 return null; 283 } 284 285 @Override 286 public synchronized void clearMetadata(Class<? extends MetadataType> clazz) { 287 if (metadata == null) 288 return; 289 290 if (clazz == null) { 291 metadata.clear(); 292 return; 293 } 294 295 List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz)); 296 if( list != null) { 297 list.clear(); 298 } 299 } 300 301 /** 302 * @since 2.0 303 */ 304 protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() { 305 return copyMetadata(metadata, null); 306 } 307 308 /** 309 * @since 2.0 310 */ 311 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) { 312 return copyMetadata(metadata, null); 313 } 314 315 /** 316 * Copy metadata. If oMetadata is not null, then copy from that when it has the corresponding items 317 * @since 2.1.2 318 * @param metadata 319 * @param oMetadata can be null 320 * @return 321 */ 322 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata, Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata) { 323 if (metadata == null) 324 return null; 325 326 ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 327 328 for (Class<? extends MetadataType> c : metadata.keySet()) { 329 List<MetadataType> l = metadata.get(c); 330 if (!l.isEmpty() && oMetadata != null && oMetadata.containsKey(c)) { // only override when not empty 331 l = oMetadata.get(c); 332 } 333 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 334 map.put(c, nl); 335 for (MetadataType m : l) { 336 nl.add(m.clone()); 337 } 338 } 339 return map; 340 } 341 342 interface MetadatasetAnnotationOperation { 343 /** 344 * Process value of given field 345 * <p> 346 * When the field is not a container then the returned value 347 * may replace the old value 348 * @param f given field 349 * @param o value of field 350 * @return transformed field 351 */ 352 Object processField(Field f, Object o); 353 354 /** 355 * @return annotated class 356 */ 357 Class<? extends Annotation> getAnnClass(); 358 359 /** 360 * @param axis 361 * @return number of dimensions to insert or remove 362 */ 363 int change(int axis); 364 365 /** 366 * 367 * @return rank or -1 to match 368 */ 369 int getNewRank(); 370 371 /** 372 * Run on given lazy dataset 373 * @param lz 374 * @return 375 */ 376 ILazyDataset run(ILazyDataset lz); 377 } 378 379 class MdsSlice implements MetadatasetAnnotationOperation { 380 private boolean asView; 381 private int[] start; 382 private int[] stop; 383 private int[] step; 384 private int[] oShape; 385 private long oSize; 386 387 public MdsSlice(boolean asView, final int[] start, final int[] stop, final int[] step, final int[] oShape) { 388 this.asView = asView; 389 this.start = start; 390 this.stop = stop; 391 this.step = step; 392 this.oShape = oShape; 393 oSize = ShapeUtils.calcLongSize(oShape); 394 } 395 396 @Override 397 public Object processField(Field field, Object o) { 398 return o; 399 } 400 401 @Override 402 public Class<? extends Annotation> getAnnClass() { 403 return Sliceable.class; 404 } 405 406 @Override 407 public int change(int axis) { 408 return 0; 409 } 410 411 @Override 412 public int getNewRank() { 413 return -1; 414 } 415 416 @Override 417 public ILazyDataset run(ILazyDataset lz) { 418 int rank = lz.getRank(); 419 if (start.length != rank) { 420 throw new IllegalArgumentException("Slice dimensions do not match dataset!"); 421 } 422 423 int[] shape = lz.getShape(); 424 int[] stt; 425 int[] stp; 426 int[] ste; 427 if (lz.getSize() == oSize) { 428 stt = start; 429 stp = stop; 430 ste = step; 431 } else { 432 stt = start.clone(); 433 stp = stop.clone(); 434 ste = step.clone(); 435 for (int i = 0; i < rank; i++) { 436 if (shape[i] >= oShape[i]) continue; 437 if (shape[i] == 1) { 438 stt[i] = 0; 439 stp[i] = 1; 440 ste[i] = 1; 441 } else { 442 throw new IllegalArgumentException("Sliceable dataset has invalid size!"); 443 } 444 } 445 } 446 447 if (asView || (lz instanceof IDataset)) 448 return lz.getSliceView(stt, stp, ste); 449 try { 450 return lz.getSlice(stt, stp, ste); 451 } catch (DatasetException e) { 452 logger.error("Could not slice dataset in metadata", e); 453 return null; 454 } 455 } 456 } 457 458 class MdsReshape implements MetadatasetAnnotationOperation { 459 private boolean matchRank; 460 private int[] oldShape; 461 private int[] newShape; 462 boolean onesOnly; 463 int[] differences; 464 465 /* 466 * if only ones then record differences (insertions and deletions) 467 * 468 * if shape changing, find broadcasted dimensions and disallow 469 * merging that include those dimensions 470 */ 471 public MdsReshape(final int[] oldShape, final int[] newShape) { 472 this.oldShape = oldShape; 473 this.newShape = newShape; 474 differences = null; 475 } 476 477 @Override 478 public Object processField(Field field, Object o) { 479 Annotation a = field.getAnnotation(Reshapeable.class); 480 if (a != null) { // cannot be null 481 matchRank = ((Reshapeable) a).matchRank(); 482 } 483 return o; 484 } 485 486 @Override 487 public Class<? extends Annotation> getAnnClass() { 488 return Reshapeable.class; 489 } 490 491 @Override 492 public int change(int axis) { 493 if (matchRank) { 494 if (differences == null) 495 init(); 496 497 if (onesOnly) { 498 return differences[axis]; 499 } 500 throw new UnsupportedOperationException("TODO support other shape operations"); 501 } 502 return 0; 503 } 504 505 @Override 506 public int getNewRank() { 507 return matchRank ? newShape.length : -1; 508 } 509 510 private void init() { 511 int or = oldShape.length - 1; 512 int nr = newShape.length - 1; 513 if (or < 0 || nr < 0) { // zero-rank shapes 514 onesOnly = true; 515 differences = new int[1]; 516 differences[0] = or < 0 ? nr + 1 : or + 1; 517 return; 518 } 519 int ob = 0; 520 int nb = 0; 521 onesOnly = true; 522 do { 523 while (oldShape[ob] == 1 && ob < or) { 524 ob++; // next non-unit dimension 525 } 526 while (newShape[nb] == 1 && nb < nr) { 527 nb++; 528 } 529 if (oldShape[ob++] != newShape[nb++]) { 530 onesOnly = false; 531 break; 532 } 533 } while (ob <= or && nb <= nr); 534 535 ob = 0; 536 nb = 0; 537 differences = new int[or + 2]; 538 if (onesOnly) { 539 // work out unit dimensions removed from or add to old 540 int j = 0; 541 do { 542 if (oldShape[ob] != 1 && newShape[nb] != 1) { 543 ob++; 544 nb++; 545 } else { 546 while (oldShape[ob] == 1 && ob < or) { 547 ob++; 548 differences[j]--; 549 } 550 while (newShape[nb] == 1 && nb < nr) { 551 nb++; 552 differences[j]++; 553 } 554 } 555 j++; 556 } while (ob <= or && nb <= nr && j <= or); 557 while (ob <= or && oldShape[ob] == 1) { 558 ob++; 559 differences[j]--; 560 } 561 while (nb <= nr && newShape[nb] == 1) { 562 nb++; 563 differences[j]++; 564 } 565 } else { 566 if (matchRank) { 567 logger.error("Combining dimensions is currently not supported"); 568 throw new IllegalArgumentException("Combining dimensions is currently not supported"); 569 } 570 // work out mapping: contiguous dimensions can be grouped or split 571 while (ob <= or && nb <= nr) { 572 int ol = oldShape[ob]; 573 while (ol == 1 && ol <= or) { 574 ob++; 575 ol = oldShape[ob]; 576 } 577 int oe = ob + 1; 578 int nl = newShape[nb]; 579 while (nl == 1 && nl <= nr) { 580 nb++; 581 nl = newShape[nb]; 582 } 583 int ne = nb + 1; 584 if (ol < nl) { 585 differences[ob] = 1; 586 do { // case where new shape combines several dimensions into one dimension 587 if (oe == (or + 1)) { 588 break; 589 } 590 differences[oe] = 1; 591 ol *= oldShape[oe++]; 592 } while (ol < nl); 593 differences[oe - 1] = oe - ob; // signal end with difference 594 if (nl != ol) { 595 logger.error("Single dimension is incompatible with subshape"); 596 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 597 } 598 } else if (ol > nl) { 599 do { // case where new shape spreads single dimension over several dimensions 600 if (ne == (nr + 1)) { 601 break; 602 } 603 nl *= newShape[ne++]; 604 } while (nl < ol); 605 if (nl != ol) { 606 logger.error("Subshape is incompatible with single dimension"); 607 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 608 } 609 610 } 611 612 ob = oe; 613 nb = ne; 614 } 615 616 } 617 } 618 619 @Override 620 public ILazyDataset run(ILazyDataset lz) { 621 if (differences == null) 622 init(); 623 624 int[] lshape = lz.getShape(); 625 if (Arrays.equals(newShape, lshape)) { 626 return lz; 627 } 628 int or = lz.getRank(); 629 int nr = newShape.length; 630 int[] nshape = new int[nr]; 631 Arrays.fill(nshape, 1); 632 if (onesOnly) { 633 // ignore omit removed dimensions 634 for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) { 635 int c = differences[i]; 636 if (c == 0) { 637 nshape[di++] = lshape[si++]; 638 } else if (c > 0) { 639 while (c-- > 0 && di < nr) { 640 di++; 641 } 642 } else if (c < 0) { 643 si -= c; // remove dimensions by skipping forward in source array 644 } 645 } 646 } else { 647 boolean[] broadcast = new boolean[or]; 648 for (int ob = 0; ob < or; ob++) { 649 broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1; 650 } 651 int osize = lz.getSize(); 652 653 // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...) 654 int ob = 0; 655 int nsize = 1; 656 for (int i = 0; i < nr; i++) { 657 if (ob < or && broadcast[ob]) { 658 if (differences[ob] != 0) { 659 logger.error("Metadata contains a broadcast axis which cannot be reshaped"); 660 throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped"); 661 } 662 } else { 663 nshape[i] = nsize < osize ? newShape[i] : 1; 664 } 665 nsize *= nshape[i]; 666 ob++; 667 } 668 } 669 670 ILazyDataset nlz = lz.getSliceView(); 671 if (lz instanceof Dataset) { 672 nlz = ((Dataset) lz).reshape(nshape); 673 } else { 674 nlz = lz.getSliceView(); 675 nlz.setShape(nshape); 676 } 677 return nlz; 678 } 679 } 680 681 class MdsTranspose implements MetadatasetAnnotationOperation { 682 int[] map; 683 684 public MdsTranspose(final int[] axesMap) { 685 map = axesMap; 686 } 687 688 @SuppressWarnings({ "rawtypes", "unchecked" }) 689 @Override 690 public Object processField(Field f, Object o) { 691 // reorder arrays and lists according the axes map 692 if (o.getClass().isArray()) { 693 int l = Array.getLength(o); 694 if (l == map.length) { 695 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 696 for (int i = 0; i < l; i++) { 697 Array.set(narray, i, Array.get(o, map[i])); 698 } 699 for (int i = 0; i < l; i++) { 700 Array.set(o, i, Array.get(narray, i)); 701 } 702 } 703 } else if (o instanceof List<?>) { 704 List list = (List) o; 705 int l = list.size(); 706 if (l == map.length) { 707 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 708 for (int i = 0; i < l; i++) { 709 Array.set(narray, i, list.get(map[i])); 710 } 711 list.clear(); 712 for (int i = 0; i < l; i++) { 713 list.add(Array.get(narray, i)); 714 } 715 } 716 } 717 return o; 718 } 719 720 @Override 721 public Class<? extends Annotation> getAnnClass() { 722 return Transposable.class; 723 } 724 725 @Override 726 public int change(int axis) { 727 return 0; 728 } 729 730 @Override 731 public int getNewRank() { 732 return -1; 733 } 734 735 @Override 736 public ILazyDataset run(ILazyDataset lz) { 737 return lz.getTransposedView(map); 738 } 739 } 740 741 class MdsDirty implements MetadatasetAnnotationOperation { 742 743 @Override 744 public Object processField(Field f, Object o) { 745 // throw exception if not boolean??? 746 Class<?> t = f.getType(); 747 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 748 if (o.equals(false)) { 749 o = true; 750 } 751 } 752 return o; 753 } 754 755 @Override 756 public Class<? extends Annotation> getAnnClass() { 757 return Dirtiable.class; 758 } 759 760 @Override 761 public int change(int axis) { 762 return 0; 763 } 764 765 @Override 766 public int getNewRank() { 767 return -1; 768 } 769 770 @Override 771 public ILazyDataset run(ILazyDataset lz) { 772 return lz; 773 } 774 } 775 776 /** 777 * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced 778 * dataset after cloning the metadata 779 * @param asView if true then just a view 780 * @param slice 781 */ 782 protected void sliceMetadata(boolean asView, final SliceND slice) { 783 processAnnotatedMetadata(new MdsSlice(asView, slice.getStart(), slice.getStop(), slice.getStep(), slice.getSourceShape()), true); 784 } 785 786 /** 787 * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing 788 * or setting the shape 789 * 790 * @param newShape 791 */ 792 protected void reshapeMetadata(final int[] oldShape, final int[] newShape) { 793 processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true); 794 } 795 796 /** 797 * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed 798 * dataset after cloning the metadata 799 * @param axesMap 800 */ 801 protected void transposeMetadata(final int[] axesMap) { 802 processAnnotatedMetadata(new MdsTranspose(axesMap), true); 803 } 804 805 /** 806 * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified 807 * @since 2.0 808 */ 809 protected void dirtyMetadata() { 810 processAnnotatedMetadata(new MdsDirty(), true); 811 } 812 813 @SuppressWarnings("unchecked") 814 private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) { 815 if (metadata == null) 816 return; 817 818 for (Class<? extends MetadataType> c : metadata.keySet()) { 819 for (MetadataType m : metadata.get(c)) { 820 if (m == null) 821 continue; 822 823 Class<? extends MetadataType> mc = m.getClass(); 824 do { // iterate over super-classes 825 processClass(op, m, mc, throwException); 826 Class<?> sclazz = mc.getSuperclass(); 827 if (!MetadataType.class.isAssignableFrom(sclazz)) 828 break; 829 mc = (Class<? extends MetadataType>) sclazz; 830 } while (true); 831 } 832 } 833 } 834 835 @SuppressWarnings({ "unchecked", "rawtypes" }) 836 private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) { 837 for (Field f : mc.getDeclaredFields()) { 838 if (!f.isAnnotationPresent(op.getAnnClass())) 839 continue; 840 841 try { 842 f.setAccessible(true); 843 Object o = f.get(m); 844 if (o == null) 845 continue; 846 847 Object no = op.processField(f, o); 848 if (no != o) { 849 f.set(m, no); 850 continue; 851 } 852 Object r = null; 853 if (o instanceof ILazyDataset) { 854 try { 855 f.set(m, op.run((ILazyDataset) o)); 856 } catch (Exception e) { 857 logger.error("Problem processing " + o, e); 858 if (!catchExceptions) 859 throw e; 860 } 861 } else if (o.getClass().isArray()) { 862 int l = Array.getLength(o); 863 if (l <= 0) 864 continue; 865 866 for (int i = 0; r == null && i < l; i++) { 867 r = Array.get(o, i); 868 } 869 int n = op.getNewRank(); 870 if (r == null) { 871 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 872 f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n)); 873 } 874 continue; 875 } 876 if (n < 0) 877 n = l; 878 Object narray = Array.newInstance(r.getClass(), n); 879 for (int i = 0, si = 0, di = 0; di < n && si < l; i++) { 880 int c = op.change(i); 881 if (c == 0) { 882 Array.set(narray, di++, processObject(op, Array.get(o, si++))); 883 } else if (c > 0) { 884 di += c; // add nulls by skipping forward in destination array 885 } else if (c < 0) { 886 si -= c; // remove dimensions by skipping forward in source array 887 } 888 } 889 if (n == l) { 890 for (int i = 0; i < l; i++) { 891 Array.set(o, i, Array.get(narray, i)); 892 } 893 } else { 894 f.set(m, narray); 895 } 896 } else if (o instanceof List<?>) { 897 List list = (List) o; 898 int l = list.size(); 899 if (l <= 0) 900 continue; 901 902 for (int i = 0; r == null && i < l; i++) { 903 r = list.get(i); 904 } 905 int n = op.getNewRank(); 906 if (r == null) { 907 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 908 list.clear(); 909 for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) { 910 list.add(null); 911 } 912 } 913 continue; 914 } 915 916 if (n < 0) 917 n = l; 918 Object narray = Array.newInstance(r.getClass(), n); 919 for (int i = 0, si = 0, di = 0; i < l && si < l; i++) { 920 int c = op.change(i); 921 if (c == 0) { 922 Array.set(narray, di++, processObject(op, list.get(si++))); 923 } else if (c > 0) { 924 di += c; // add nulls by skipping forward in destination array 925 } else if (c < 0) { 926 si -= c; // remove dimensions by skipping forward in source array 927 } 928 } 929 list.clear(); 930 for (int i = 0; i < n; i++) { 931 list.add(Array.get(narray, i)); 932 } 933 } else if (o instanceof Map<?,?>) { 934 Map map = (Map) o; 935 for (Object k : map.keySet()) { 936 map.put(k, processObject(op, map.get(k))); 937 } 938 } 939 } catch (Exception e) { 940 logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e); 941 if (throwException) 942 throw new RuntimeException(e); 943 } 944 } 945 } 946 947 @SuppressWarnings({ "unchecked", "rawtypes" }) 948 private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception { 949 if (o == null) 950 return o; 951 952 if (o instanceof ILazyDataset) { 953 try { 954 return op.run((ILazyDataset) o); 955 } catch (Exception e) { 956 logger.error("Problem processing " + o, e); 957 if (!catchExceptions) 958 throw e; 959 } 960 } else if (o.getClass().isArray()) { 961 int l = Array.getLength(o); 962 for (int i = 0; i < l; i++) { 963 Array.set(o, i, processObject(op, Array.get(o, i))); 964 } 965 } else if (o instanceof List<?>) { 966 List list = (List) o; 967 for (int i = 0, imax = list.size(); i < imax; i++) { 968 list.set(i, processObject(op, list.get(i))); 969 } 970 } else if (o instanceof Map<?,?>) { 971 Map map = (Map) o; 972 for (Object k : map.keySet()) { 973 map.put(k, processObject(op, map.get(k))); 974 } 975 } 976 return o; 977 } 978 979 protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) { 980 for (Class<? extends MetadataType> mc : oldMetadata.keySet()) { 981 metadata.put(mc, oldMetadata.get(mc)); 982 } 983 } 984 985 protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) { 986 ILazyDataset d = null; 987 if (blob instanceof ILazyDataset) { 988 d = (ILazyDataset) blob; 989 if (d instanceof IDataset) { 990 Dataset ed = DatasetUtils.convertToDataset((IDataset) d); 991 int is = ed.getElementsPerItem(); 992 if (is != 1 && is != getElementsPerItem()) { 993 throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset"); 994 } 995 d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 996 } else if (!keepLazy) { 997 final int is = getElementsPerItem(); 998 try { 999 d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 1000 } catch (DatasetException e) { 1001 logger.error("Could not get data from lazy dataset", e); 1002 return null; 1003 } 1004 } 1005 } else { 1006 final int is = getElementsPerItem(); 1007 if (is == 1) { 1008 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1009 } else { 1010 try { 1011 d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob); 1012 } catch (IllegalArgumentException e) { // if only single value supplied try again 1013 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1014 } 1015 } 1016 if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) { 1017 d.setShape(shape.clone()); 1018 } 1019 } 1020 List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape()); 1021 d.setShape(s.get(0)); 1022 1023 return d; 1024 } 1025 1026 @Override 1027 public void setErrors(Serializable errors) { 1028 if (shape == null) { 1029 throw new IllegalArgumentException("Cannot set errors for null dataset"); 1030 } 1031 if (errors == null) { 1032 clearMetadata(ErrorMetadata.class); 1033 return; 1034 } 1035 if (errors == this) { 1036 logger.warn("Ignoring setting error to itself as this will lead to infinite recursion"); 1037 return; 1038 } 1039 1040 ILazyDataset errorData = createFromSerializable(errors, true); 1041 1042 ErrorMetadata emd = getErrorMetadata(); 1043 if (emd == null) { 1044 try { 1045 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1046 setMetadata(emd); 1047 } catch (MetadataException me) { 1048 logger.error("Could not create metadata", me); 1049 } 1050 } 1051 emd.setError(errorData); 1052 } 1053 1054 protected ErrorMetadata getErrorMetadata() { 1055 try { 1056 List<ErrorMetadata> el = getMetadata(ErrorMetadata.class); 1057 if (el != null && !el.isEmpty()) { 1058 return el.get(0); 1059 } 1060 } catch (Exception e) { 1061 } 1062 return null; 1063 } 1064 1065 @Override 1066 public ILazyDataset getErrors() { 1067 ErrorMetadata emd = getErrorMetadata(); 1068 return emd == null ? null : emd.getError(); 1069 } 1070 1071 @Override 1072 public boolean hasErrors() { 1073 return LazyDatasetBase.this.getErrors() != null; 1074 } 1075 1076 /** 1077 * Check permutation axes 1078 * @param shape 1079 * @param axes 1080 * @return cleaned up axes or null if trivial 1081 */ 1082 public static int[] checkPermutatedAxes(int[] shape, int... axes) { 1083 int rank = shape == null ? 0 : shape.length; 1084 1085 if (axes == null || axes.length == 0) { 1086 axes = new int[rank]; 1087 for (int i = 0; i < rank; i++) { 1088 axes[i] = rank - 1 - i; 1089 } 1090 } 1091 1092 if (axes.length != rank) { 1093 logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank); 1094 throw new IllegalArgumentException("axis permutation does not match shape of dataset"); 1095 } 1096 1097 // check all permutation values are within bounds 1098 for (int i = 0; i < rank; i++) { 1099 axes[i] = ShapeUtils.checkAxis(rank, axes[i]); 1100 } 1101 1102 // check for a valid permutation (is this an unnecessary restriction?) 1103 int[] perm = axes.clone(); 1104 Arrays.sort(perm); 1105 1106 for (int i = 0; i < rank; i++) { 1107 if (perm[i] != i) { 1108 logger.error("axis permutation is not valid: it does not contain complete set of axes"); 1109 throw new IllegalArgumentException("axis permutation does not contain complete set of axes"); 1110 } 1111 } 1112 1113 if (Arrays.equals(axes, perm)) 1114 return null; // signal identity or trivial permutation 1115 1116 return axes; 1117 } 1118}