001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.annotation.Annotation; 017import java.lang.reflect.Array; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.Collection; 022import java.util.concurrent.ConcurrentMap; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.List; 025import java.util.Map; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.MetadataException; 029import org.eclipse.january.metadata.Dirtiable; 030import org.eclipse.january.metadata.ErrorMetadata; 031import org.eclipse.january.metadata.IMetadata; 032import org.eclipse.january.metadata.MetadataFactory; 033import org.eclipse.january.metadata.MetadataType; 034import org.eclipse.january.metadata.Reshapeable; 035import org.eclipse.january.metadata.Sliceable; 036import org.eclipse.january.metadata.Transposable; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * Common base for both lazy and normal dataset implementations 042 */ 043public abstract class LazyDatasetBase implements ILazyDataset, Serializable { 044 045 private static final long serialVersionUID = 767926846438976050L; 046 047 protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class); 048 049 protected static boolean catchExceptions; 050 051 static { 052 /** 053 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE 054 */ 055 try { 056 catchExceptions = Boolean.getBoolean("run.in.eclipse"); 057 } catch (SecurityException e) { 058 // set a default for when the security manager does not allow access to the requested key 059 catchExceptions = false; 060 } 061 } 062 063 transient private boolean dirty = true; // indicate dirty state of metadata 064 protected String name = ""; 065 066 /** 067 * The shape or dimensions of the dataset 068 */ 069 protected int[] shape; 070 071 protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null; 072 073 /** 074 * @return type of dataset item 075 */ 076 abstract public int getDType(); 077 078 @Override 079 public Class<?> getElementClass() { 080 return DTypeUtils.getElementClass(getDType()); 081 } 082 083 @Override 084 public LazyDatasetBase clone() { 085 return null; 086 } 087 088 @Override 089 public boolean equals(Object obj) { 090 if (this == obj) { 091 return true; 092 } 093 if (obj == null) { 094 return false; 095 } 096 if (!getClass().equals(obj.getClass())) { 097 return false; 098 } 099 100 LazyDatasetBase other = (LazyDatasetBase) obj; 101 if (getDType() != other.getDType()) { 102 return false; 103 } 104 if (getElementsPerItem() != other.getElementsPerItem()) { 105 return false; 106 } 107 if (!Arrays.equals(shape, other.shape)) { 108 return false; 109 } 110 return true; 111 } 112 113 @Override 114 public int hashCode() { 115 int hash = getDType() * 17 + getElementsPerItem(); 116 int rank = shape.length; 117 for (int i = 0; i < rank; i++) { 118 hash = hash*17 + shape[i]; 119 } 120 return hash; 121 } 122 123 @Override 124 public String getName() { 125 return name; 126 } 127 128 @Override 129 public void setName(String name) { 130 this.name = name; 131 } 132 133 @Override 134 public int[] getShape() { 135 return shape.clone(); 136 } 137 138 @Override 139 public int getRank() { 140 return shape.length; 141 } 142 143 /** 144 * This method allows anything that dirties the dataset to clear various metadata values 145 * so that the other methods can work correctly. 146 * @since 2.1 147 */ 148 public void setDirty() { 149 dirty = true; 150 } 151 152 /** 153 * Find first sub-interface of (or class that directly implements) MetadataType 154 * @param clazz 155 * @return sub-interface 156 * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it 157 */ 158 @SuppressWarnings("unchecked") 159 public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) { 160 if (clazz.equals(MetadataType.class)) { 161 throw new IllegalArgumentException("Cannot accept MetadataType"); 162 } 163 164 if (clazz.isInterface()) { 165 return clazz; 166 } 167 168 if (clazz.isAnonymousClass()) { // special case 169 Class<?> s = clazz.getSuperclass(); 170 if (!s.equals(Object.class)) { 171 // only use super class if it is not an anonymous class of an interface 172 clazz = (Class<? extends MetadataType>) s; 173 } 174 } 175 176 for (Class<?> c : clazz.getInterfaces()) { 177 if (c.equals(MetadataType.class)) { 178 if (clazz.isAnonymousClass()) { 179 throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType"); 180 } 181 return clazz; 182 } 183 if (MetadataType.class.isAssignableFrom(c)) { 184 return (Class<? extends MetadataType>) c; 185 } 186 } 187 188 Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class 189 if (c != null) { 190 return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c); 191 } 192 193 logger.error("Somehow the search for metadata type interface ended in a bad place"); 194 assert false; // should not be able to get here!!! 195 return null; 196 } 197 198 @Override 199 public void setMetadata(MetadataType metadata) { 200 addMetadata(metadata, true); 201 } 202 203 @Override 204 public void addMetadata(MetadataType metadata) { 205 addMetadata(metadata, false); 206 } 207 208 private synchronized void addMetadata(MetadataType metadata, boolean clear) { 209 if (metadata == null) 210 return; 211 212 if (this.metadata == null) { 213 this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 214 } 215 216 Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass()); 217 if (!this.metadata.containsKey(clazz)) { 218 this.metadata.put(clazz, new ArrayList<MetadataType>()); 219 } else if (clear) { 220 this.metadata.get(clazz).clear(); 221 } 222 this.metadata.get(clazz).add(metadata); 223 224 // add for special case of sub-interfaces of IMetadata 225 if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) { 226 clazz = IMetadata.class; 227 if (!this.metadata.containsKey(clazz)) { 228 this.metadata.put(clazz, new ArrayList<MetadataType>()); 229 } else if (clear) { 230 this.metadata.get(clazz).clear(); 231 } 232 this.metadata.get(clazz).add(metadata); 233 } 234 } 235 236 @Override 237 @Deprecated 238 public synchronized IMetadata getMetadata() { 239 return getFirstMetadata(IMetadata.class); 240 } 241 242 @SuppressWarnings("unchecked") 243 @Override 244 public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException { 245 if (metadata == null) { 246 dirty = false; 247 return null; 248 } 249 250 if (dirty) { 251 dirtyMetadata(); 252 dirty = false; 253 } 254 255 if (clazz == null) { 256 List<S> all = new ArrayList<S>(); 257 for (Class<? extends MetadataType> c : metadata.keySet()) { 258 all.addAll((Collection<S>) metadata.get(c)); 259 } 260 return all; 261 } 262 263 return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz)); 264 } 265 266 @Override 267 public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) { 268 try { 269 List<S> ml = getMetadata(clazz); 270 if (ml == null) { 271 return null; 272 } 273 for (S t : ml) { 274 if (clazz.isInstance(t)) { 275 return t; 276 } 277 } 278 } catch (Exception e) { 279 logger.error("Get metadata failed!",e); 280 } 281 282 return null; 283 } 284 285 @Override 286 public synchronized void clearMetadata(Class<? extends MetadataType> clazz) { 287 if (metadata == null) 288 return; 289 290 if (clazz == null) { 291 metadata.clear(); 292 return; 293 } 294 295 List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz)); 296 if( list != null) { 297 list.clear(); 298 } 299 } 300 301 /** 302 * @since 2.0 303 */ 304 protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() { 305 return copyMetadata(metadata, null); 306 } 307 308 /** 309 * @since 2.0 310 */ 311 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) { 312 return copyMetadata(metadata, null); 313 } 314 315 /** 316 * Copy metadata. If oMetadata is not null, then copy from that when it has the corresponding items 317 * @since 2.1 318 * @param metadata 319 * @param oMetadata can be null 320 * @return 321 */ 322 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata, Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata) { 323 if (metadata == null) 324 return null; 325 326 ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 327 328 for (Class<? extends MetadataType> c : metadata.keySet()) { 329 List<MetadataType> l = metadata.get(c); 330 if (!l.isEmpty() && oMetadata != null && oMetadata.containsKey(c)) { // only override when not empty 331 l = oMetadata.get(c); 332 } 333 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 334 map.put(c, nl); 335 for (MetadataType m : l) { 336 nl.add(m.clone()); 337 } 338 } 339 return map; 340 } 341 342 interface MetadatasetAnnotationOperation { 343 /** 344 * Process value of given field 345 * <p> 346 * When the field is not a container then the returned value 347 * may replace the old value 348 * @param f given field 349 * @param o value of field 350 * @return transformed field 351 */ 352 Object processField(Field f, Object o); 353 354 /** 355 * @return annotated class 356 */ 357 Class<? extends Annotation> getAnnClass(); 358 359 /** 360 * @param axis 361 * @return number of dimensions to insert or remove 362 */ 363 int change(int axis); 364 365 /** 366 * 367 * @return rank or -1 to match 368 */ 369 int getNewRank(); 370 371 /** 372 * Run on given lazy dataset 373 * @param lz 374 * @return 375 */ 376 ILazyDataset run(ILazyDataset lz); 377 } 378 379 class MdsSlice implements MetadatasetAnnotationOperation { 380 private boolean asView; 381 private SliceND slice; 382 private int[] oShape; 383 private long oSize; 384 385 public MdsSlice(boolean asView, SliceND slice) { 386 this.asView = asView; 387 this.slice = slice; 388 oShape = slice.getSourceShape(); 389 oSize = ShapeUtils.calcLongSize(oShape); 390 } 391 392 @Override 393 public Object processField(Field field, Object o) { 394 return o; 395 } 396 397 @Override 398 public Class<? extends Annotation> getAnnClass() { 399 return Sliceable.class; 400 } 401 402 @Override 403 public int change(int axis) { 404 return 0; 405 } 406 407 @Override 408 public int getNewRank() { 409 return -1; 410 } 411 412 @Override 413 public ILazyDataset run(ILazyDataset lz) { 414 int rank = lz.getRank(); 415 if (slice.getStart().length != rank) { 416 throw new IllegalArgumentException("Slice rank does not match dataset!"); 417 } 418 419 int[] shape = lz.getShape(); 420 SliceND nslice; 421 if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) { 422 nslice = new SliceND(shape); 423 for (int i = 0; i < rank; i++) { 424 int s = shape[i]; 425 int os = oShape[i]; 426 if (s >= os) { 427 nslice.setSlice(i, 0, os, 1); 428 } else if (s == 1) { 429 nslice.setSlice(i, 0, 1, 1); 430 } else { 431 throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!"); 432 } 433 } 434 lz = lz.getSliceView(nslice); 435 } 436 if (lz.getSize() == oSize) { 437 nslice = slice; 438 } else { 439 nslice = slice.clone(); 440 } 441 for (int i = 0; i < rank; i++) { 442 int s = shape[i]; 443 if (s >= oShape[i]) { 444 continue; 445 } else if (s == 1) { 446 nslice.setSlice(i, 0, 1, 1); 447 } else { 448 throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!"); 449 } 450 } 451 452 if (asView || (lz instanceof IDataset)) { 453 return lz.getSliceView(nslice); 454 } 455 try { 456 return lz.getSlice(nslice); 457 } catch (DatasetException e) { 458 logger.error("Could not slice dataset in metadata", e); 459 return null; 460 } 461 } 462 } 463 464 class MdsReshape implements MetadatasetAnnotationOperation { 465 private boolean matchRank; 466 private int[] oldShape; 467 private int[] newShape; 468 boolean onesOnly; 469 int[] differences; 470 471 /* 472 * if only ones then record differences (insertions and deletions) 473 * 474 * if shape changing, find broadcasted dimensions and disallow 475 * merging that include those dimensions 476 */ 477 public MdsReshape(final int[] oldShape, final int[] newShape) { 478 this.oldShape = oldShape; 479 this.newShape = newShape; 480 differences = null; 481 } 482 483 @Override 484 public Object processField(Field field, Object o) { 485 Annotation a = field.getAnnotation(Reshapeable.class); 486 if (a != null) { // cannot be null 487 matchRank = ((Reshapeable) a).matchRank(); 488 } 489 return o; 490 } 491 492 @Override 493 public Class<? extends Annotation> getAnnClass() { 494 return Reshapeable.class; 495 } 496 497 @Override 498 public int change(int axis) { 499 if (matchRank) { 500 if (differences == null) 501 init(); 502 503 if (onesOnly) { 504 return differences[axis]; 505 } 506 throw new UnsupportedOperationException("TODO support other shape operations"); 507 } 508 return 0; 509 } 510 511 @Override 512 public int getNewRank() { 513 return matchRank ? newShape.length : -1; 514 } 515 516 private void init() { 517 int or = oldShape.length - 1; 518 int nr = newShape.length - 1; 519 if (or < 0 || nr < 0) { // zero-rank shapes 520 onesOnly = true; 521 differences = new int[1]; 522 differences[0] = or < 0 ? nr + 1 : or + 1; 523 return; 524 } 525 int ob = 0; 526 int nb = 0; 527 onesOnly = true; 528 do { 529 while (oldShape[ob] == 1 && ob < or) { 530 ob++; // next non-unit dimension 531 } 532 while (newShape[nb] == 1 && nb < nr) { 533 nb++; 534 } 535 if (oldShape[ob++] != newShape[nb++]) { 536 onesOnly = false; 537 break; 538 } 539 } while (ob <= or && nb <= nr); 540 541 ob = 0; 542 nb = 0; 543 differences = new int[or + 2]; 544 if (onesOnly) { 545 // work out unit dimensions removed from or add to old 546 int j = 0; 547 do { 548 if (oldShape[ob] != 1 && newShape[nb] != 1) { 549 ob++; 550 nb++; 551 } else { 552 while (oldShape[ob] == 1 && ob < or) { 553 ob++; 554 differences[j]--; 555 } 556 while (newShape[nb] == 1 && nb < nr) { 557 nb++; 558 differences[j]++; 559 } 560 } 561 j++; 562 } while (ob <= or && nb <= nr && j <= or); 563 while (ob <= or && oldShape[ob] == 1) { 564 ob++; 565 differences[j]--; 566 } 567 while (nb <= nr && newShape[nb] == 1) { 568 nb++; 569 differences[j]++; 570 } 571 } else { 572 if (matchRank) { 573 logger.error("Combining dimensions is currently not supported"); 574 throw new IllegalArgumentException("Combining dimensions is currently not supported"); 575 } 576 // work out mapping: contiguous dimensions can be grouped or split 577 while (ob <= or && nb <= nr) { 578 int ol = oldShape[ob]; 579 while (ol == 1 && ol <= or) { 580 ob++; 581 ol = oldShape[ob]; 582 } 583 int oe = ob + 1; 584 int nl = newShape[nb]; 585 while (nl == 1 && nl <= nr) { 586 nb++; 587 nl = newShape[nb]; 588 } 589 int ne = nb + 1; 590 if (ol < nl) { 591 differences[ob] = 1; 592 do { // case where new shape combines several dimensions into one dimension 593 if (oe == (or + 1)) { 594 break; 595 } 596 differences[oe] = 1; 597 ol *= oldShape[oe++]; 598 } while (ol < nl); 599 differences[oe - 1] = oe - ob; // signal end with difference 600 if (nl != ol) { 601 logger.error("Single dimension is incompatible with subshape"); 602 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 603 } 604 } else if (ol > nl) { 605 do { // case where new shape spreads single dimension over several dimensions 606 if (ne == (nr + 1)) { 607 break; 608 } 609 nl *= newShape[ne++]; 610 } while (nl < ol); 611 if (nl != ol) { 612 logger.error("Subshape is incompatible with single dimension"); 613 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 614 } 615 616 } 617 618 ob = oe; 619 nb = ne; 620 } 621 622 } 623 } 624 625 @Override 626 public ILazyDataset run(ILazyDataset lz) { 627 if (differences == null) 628 init(); 629 630 int[] lshape = lz.getShape(); 631 if (Arrays.equals(newShape, lshape)) { 632 return lz; 633 } 634 int or = lz.getRank(); 635 int nr = newShape.length; 636 int[] nshape = new int[nr]; 637 Arrays.fill(nshape, 1); 638 if (onesOnly) { 639 // ignore omit removed dimensions 640 for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) { 641 int c = differences[i]; 642 if (c == 0) { 643 nshape[di++] = lshape[si++]; 644 } else if (c > 0) { 645 while (c-- > 0 && di < nr) { 646 di++; 647 } 648 } else if (c < 0) { 649 si -= c; // remove dimensions by skipping forward in source array 650 } 651 } 652 } else { 653 boolean[] broadcast = new boolean[or]; 654 for (int ob = 0; ob < or; ob++) { 655 broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1; 656 } 657 int osize = lz.getSize(); 658 659 // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...) 660 int ob = 0; 661 int nsize = 1; 662 for (int i = 0; i < nr; i++) { 663 if (ob < or && broadcast[ob]) { 664 if (differences[ob] != 0) { 665 logger.error("Metadata contains a broadcast axis which cannot be reshaped"); 666 throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped"); 667 } 668 } else { 669 nshape[i] = nsize < osize ? newShape[i] : 1; 670 } 671 nsize *= nshape[i]; 672 ob++; 673 } 674 } 675 676 ILazyDataset nlz = lz.getSliceView(); 677 if (lz instanceof Dataset) { 678 nlz = ((Dataset) lz).reshape(nshape); 679 } else { 680 nlz = lz.getSliceView(); 681 nlz.setShape(nshape); 682 } 683 return nlz; 684 } 685 } 686 687 class MdsTranspose implements MetadatasetAnnotationOperation { 688 int[] map; 689 690 public MdsTranspose(final int[] axesMap) { 691 map = axesMap; 692 } 693 694 @SuppressWarnings({ "rawtypes", "unchecked" }) 695 @Override 696 public Object processField(Field f, Object o) { 697 // reorder arrays and lists according the axes map 698 if (o.getClass().isArray()) { 699 int l = Array.getLength(o); 700 if (l == map.length) { 701 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 702 for (int i = 0; i < l; i++) { 703 Array.set(narray, i, Array.get(o, map[i])); 704 } 705 for (int i = 0; i < l; i++) { 706 Array.set(o, i, Array.get(narray, i)); 707 } 708 } 709 } else if (o instanceof List<?>) { 710 List list = (List) o; 711 int l = list.size(); 712 if (l == map.length) { 713 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 714 for (int i = 0; i < l; i++) { 715 Array.set(narray, i, list.get(map[i])); 716 } 717 list.clear(); 718 for (int i = 0; i < l; i++) { 719 list.add(Array.get(narray, i)); 720 } 721 } 722 } 723 return o; 724 } 725 726 @Override 727 public Class<? extends Annotation> getAnnClass() { 728 return Transposable.class; 729 } 730 731 @Override 732 public int change(int axis) { 733 return 0; 734 } 735 736 @Override 737 public int getNewRank() { 738 return -1; 739 } 740 741 @Override 742 public ILazyDataset run(ILazyDataset lz) { 743 return lz.getTransposedView(map); 744 } 745 } 746 747 class MdsDirty implements MetadatasetAnnotationOperation { 748 749 @Override 750 public Object processField(Field f, Object o) { 751 // throw exception if not boolean??? 752 Class<?> t = f.getType(); 753 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 754 if (o.equals(false)) { 755 o = true; 756 } 757 } 758 return o; 759 } 760 761 @Override 762 public Class<? extends Annotation> getAnnClass() { 763 return Dirtiable.class; 764 } 765 766 @Override 767 public int change(int axis) { 768 return 0; 769 } 770 771 @Override 772 public int getNewRank() { 773 return -1; 774 } 775 776 @Override 777 public ILazyDataset run(ILazyDataset lz) { 778 return lz; 779 } 780 } 781 782 /** 783 * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced 784 * dataset after cloning the metadata 785 * @param asView if true then just a view 786 * @param slice 787 */ 788 protected void sliceMetadata(boolean asView, final SliceND slice) { 789 processAnnotatedMetadata(new MdsSlice(asView, slice), true); 790 } 791 792 /** 793 * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing 794 * or setting the shape 795 * 796 * @param newShape 797 */ 798 protected void reshapeMetadata(final int[] oldShape, final int[] newShape) { 799 processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true); 800 } 801 802 /** 803 * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed 804 * dataset after cloning the metadata 805 * @param axesMap 806 */ 807 protected void transposeMetadata(final int[] axesMap) { 808 processAnnotatedMetadata(new MdsTranspose(axesMap), true); 809 } 810 811 /** 812 * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified 813 * @since 2.0 814 */ 815 protected void dirtyMetadata() { 816 processAnnotatedMetadata(new MdsDirty(), true); 817 } 818 819 @SuppressWarnings("unchecked") 820 private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) { 821 if (metadata == null) 822 return; 823 824 for (Class<? extends MetadataType> c : metadata.keySet()) { 825 for (MetadataType m : metadata.get(c)) { 826 if (m == null) 827 continue; 828 829 Class<? extends MetadataType> mc = m.getClass(); 830 do { // iterate over super-classes 831 processClass(op, m, mc, throwException); 832 Class<?> sclazz = mc.getSuperclass(); 833 if (!MetadataType.class.isAssignableFrom(sclazz)) 834 break; 835 mc = (Class<? extends MetadataType>) sclazz; 836 } while (true); 837 } 838 } 839 } 840 841 @SuppressWarnings({ "unchecked", "rawtypes" }) 842 private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) { 843 for (Field f : mc.getDeclaredFields()) { 844 if (!f.isAnnotationPresent(op.getAnnClass())) 845 continue; 846 847 try { 848 f.setAccessible(true); 849 Object o = f.get(m); 850 if (o == null) 851 continue; 852 853 Object no = op.processField(f, o); 854 if (no != o) { 855 f.set(m, no); 856 continue; 857 } 858 Object r = null; 859 if (o instanceof ILazyDataset) { 860 try { 861 f.set(m, op.run((ILazyDataset) o)); 862 } catch (Exception e) { 863 logger.error("Problem processing " + o, e); 864 if (!catchExceptions) 865 throw e; 866 } 867 } else if (o.getClass().isArray()) { 868 int l = Array.getLength(o); 869 if (l <= 0) 870 continue; 871 872 for (int i = 0; r == null && i < l; i++) { 873 r = Array.get(o, i); 874 } 875 int n = op.getNewRank(); 876 if (r == null) { 877 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 878 f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n)); 879 } 880 continue; 881 } 882 if (n < 0) 883 n = l; 884 Object narray = Array.newInstance(r.getClass(), n); 885 for (int i = 0, si = 0, di = 0; di < n && si < l; i++) { 886 int c = op.change(i); 887 if (c == 0) { 888 Array.set(narray, di++, processObject(op, Array.get(o, si++))); 889 } else if (c > 0) { 890 di += c; // add nulls by skipping forward in destination array 891 } else if (c < 0) { 892 si -= c; // remove dimensions by skipping forward in source array 893 } 894 } 895 if (n == l) { 896 for (int i = 0; i < l; i++) { 897 Array.set(o, i, Array.get(narray, i)); 898 } 899 } else { 900 f.set(m, narray); 901 } 902 } else if (o instanceof List<?>) { 903 List list = (List) o; 904 int l = list.size(); 905 if (l <= 0) 906 continue; 907 908 for (int i = 0; r == null && i < l; i++) { 909 r = list.get(i); 910 } 911 int n = op.getNewRank(); 912 if (r == null) { 913 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 914 list.clear(); 915 for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) { 916 list.add(null); 917 } 918 } 919 continue; 920 } 921 922 if (n < 0) 923 n = l; 924 Object narray = Array.newInstance(r.getClass(), n); 925 for (int i = 0, si = 0, di = 0; i < l && si < l; i++) { 926 int c = op.change(i); 927 if (c == 0) { 928 Array.set(narray, di++, processObject(op, list.get(si++))); 929 } else if (c > 0) { 930 di += c; // add nulls by skipping forward in destination array 931 } else if (c < 0) { 932 si -= c; // remove dimensions by skipping forward in source array 933 } 934 } 935 list.clear(); 936 for (int i = 0; i < n; i++) { 937 list.add(Array.get(narray, i)); 938 } 939 } else if (o instanceof Map<?,?>) { 940 Map map = (Map) o; 941 for (Object k : map.keySet()) { 942 map.put(k, processObject(op, map.get(k))); 943 } 944 } 945 } catch (Exception e) { 946 logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e); 947 if (throwException) 948 throw new RuntimeException(e); 949 } 950 } 951 } 952 953 @SuppressWarnings({ "unchecked", "rawtypes" }) 954 private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception { 955 if (o == null) 956 return o; 957 958 if (o instanceof ILazyDataset) { 959 try { 960 return op.run((ILazyDataset) o); 961 } catch (Exception e) { 962 logger.error("Problem processing " + o, e); 963 if (!catchExceptions) 964 throw e; 965 } 966 } else if (o.getClass().isArray()) { 967 int l = Array.getLength(o); 968 for (int i = 0; i < l; i++) { 969 Array.set(o, i, processObject(op, Array.get(o, i))); 970 } 971 } else if (o instanceof List<?>) { 972 List list = (List) o; 973 for (int i = 0, imax = list.size(); i < imax; i++) { 974 list.set(i, processObject(op, list.get(i))); 975 } 976 } else if (o instanceof Map<?,?>) { 977 Map map = (Map) o; 978 for (Object k : map.keySet()) { 979 map.put(k, processObject(op, map.get(k))); 980 } 981 } 982 return o; 983 } 984 985 protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) { 986 for (Class<? extends MetadataType> mc : oldMetadata.keySet()) { 987 metadata.put(mc, oldMetadata.get(mc)); 988 } 989 } 990 991 protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) { 992 ILazyDataset d = null; 993 if (blob instanceof ILazyDataset) { 994 d = (ILazyDataset) blob; 995 if (d instanceof IDataset) { 996 Dataset ed = DatasetUtils.convertToDataset((IDataset) d); 997 int is = ed.getElementsPerItem(); 998 if (is != 1 && is != getElementsPerItem()) { 999 throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset"); 1000 } 1001 d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 1002 } else if (!keepLazy) { 1003 final int is = getElementsPerItem(); 1004 try { 1005 d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 1006 } catch (DatasetException e) { 1007 logger.error("Could not get data from lazy dataset", e); 1008 return null; 1009 } 1010 } 1011 } else { 1012 final int is = getElementsPerItem(); 1013 if (is == 1) { 1014 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1015 } else { 1016 try { 1017 d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob); 1018 } catch (IllegalArgumentException e) { // if only single value supplied try again 1019 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1020 } 1021 } 1022 if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) { 1023 d.setShape(shape.clone()); 1024 } 1025 } 1026 List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape()); 1027 d.setShape(s.get(0)); 1028 1029 return d; 1030 } 1031 1032 @Override 1033 public void setErrors(Serializable errors) { 1034 if (shape == null) { 1035 throw new IllegalArgumentException("Cannot set errors for null dataset"); 1036 } 1037 if (errors == null) { 1038 clearMetadata(ErrorMetadata.class); 1039 return; 1040 } 1041 if (errors == this) { 1042 logger.warn("Ignoring setting error to itself as this will lead to infinite recursion"); 1043 return; 1044 } 1045 1046 ILazyDataset errorData = createFromSerializable(errors, true); 1047 1048 ErrorMetadata emd = getErrorMetadata(); 1049 if (emd == null) { 1050 try { 1051 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1052 setMetadata(emd); 1053 } catch (MetadataException me) { 1054 logger.error("Could not create metadata", me); 1055 } 1056 } 1057 emd.setError(errorData); 1058 } 1059 1060 protected ErrorMetadata getErrorMetadata() { 1061 try { 1062 List<ErrorMetadata> el = getMetadata(ErrorMetadata.class); 1063 if (el != null && !el.isEmpty()) { 1064 return el.get(0); 1065 } 1066 } catch (Exception e) { 1067 } 1068 return null; 1069 } 1070 1071 @Override 1072 public ILazyDataset getErrors() { 1073 ErrorMetadata emd = getErrorMetadata(); 1074 return emd == null ? null : emd.getError(); 1075 } 1076 1077 @Override 1078 public boolean hasErrors() { 1079 return LazyDatasetBase.this.getErrors() != null; 1080 } 1081 1082 /** 1083 * Check permutation axes 1084 * @param shape 1085 * @param axes 1086 * @return cleaned up axes or null if trivial 1087 */ 1088 public static int[] checkPermutatedAxes(int[] shape, int... axes) { 1089 int rank = shape == null ? 0 : shape.length; 1090 1091 if (axes == null || axes.length == 0) { 1092 axes = new int[rank]; 1093 for (int i = 0; i < rank; i++) { 1094 axes[i] = rank - 1 - i; 1095 } 1096 } 1097 1098 if (axes.length != rank) { 1099 logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank); 1100 throw new IllegalArgumentException("axis permutation does not match shape of dataset"); 1101 } 1102 1103 // check all permutation values are within bounds 1104 for (int i = 0; i < rank; i++) { 1105 axes[i] = ShapeUtils.checkAxis(rank, axes[i]); 1106 } 1107 1108 // check for a valid permutation (is this an unnecessary restriction?) 1109 int[] perm = axes.clone(); 1110 Arrays.sort(perm); 1111 1112 for (int i = 0; i < rank; i++) { 1113 if (perm[i] != i) { 1114 logger.error("axis permutation is not valid: it does not contain complete set of axes"); 1115 throw new IllegalArgumentException("axis permutation does not contain complete set of axes"); 1116 } 1117 } 1118 1119 if (Arrays.equals(axes, perm)) 1120 return null; // signal identity or trivial permutation 1121 1122 return axes; 1123 } 1124}