001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.IOException; 016import java.io.Serializable; 017import java.lang.annotation.Annotation; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.HashMap; 022import java.util.LinkedList; 023import java.util.List; 024import java.util.Map; 025import java.util.Objects; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.IMonitor; 029import org.eclipse.january.io.ILazyLoader; 030import org.eclipse.january.metadata.MetadataFactory; 031import org.eclipse.january.metadata.MetadataType; 032import org.eclipse.january.metadata.OriginMetadata; 033import org.eclipse.january.metadata.Reshapeable; 034import org.eclipse.january.metadata.Sliceable; 035import org.eclipse.january.metadata.Transposable; 036import org.slf4j.Logger; 037import org.slf4j.LoggerFactory; 038 039public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable { 040 private static final long serialVersionUID = 2467865859867440242L; 041 042 private static final Logger logger = LoggerFactory.getLogger(LazyDataset.class); 043 044 protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null; 045 protected int[] oShape; // original shape 046 protected long size; // number of items 047 private Class<? extends Dataset> clazz = null; 048 protected int isize; // number of elements per item 049 050 protected ILazyLoader loader; 051 052 // relative to loader 053 protected int[] begSlice = null; // slice begin 054 protected int[] delSlice = null; // slice delta 055 /** 056 * @since 2.2 057 */ 058 protected int[] sShape = null; // sliced shape 059 060 /** 061 * @since 2.2 062 */ 063 protected int[] padding = null; // differences in shape from original (or sliced) shape 064 protected int[] map; // transposition map (same length as current shape) 065 066 /** 067 * Create a lazy dataset 068 * @param loader lazy loader 069 * @param name of dataset 070 * @param elements item size 071 * @param clazz dataset sub-interface 072 * @param shape dataset shape 073 * @since 2.3 074 */ 075 public LazyDataset(ILazyLoader loader, String name, int elements, Class<? extends Dataset> clazz, int... shape) { 076 this.loader = loader; 077 this.name = name; 078 this.isize = elements; 079 this.clazz = clazz; 080 this.shape = shape.clone(); 081 this.oShape = this.shape; 082 try { 083 size = ShapeUtils.calcLongSize(shape); 084 } catch (IllegalArgumentException e) { 085 size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 086 } 087 } 088 089 /** 090 * Create a lazy dataset 091 * @param loader lazy loader 092 * @param name of dataset 093 * @param clazz dataset sub-interface 094 * @param shape dataset shape 095 * @since 2.3 096 */ 097 public LazyDataset(ILazyLoader loader, String name, Class<? extends Dataset> clazz, int... shape) { 098 this(loader, name, 1, clazz, shape); 099 } 100 101 /** 102 * Create a lazy dataset 103 * @param name of dataset 104 * @param dtype dataset type 105 * @param elements item size 106 * @param shape dataset shape 107 * @param loader lazy loader 108 * @deprecated Use {@link #LazyDataset(ILazyLoader, String, int, Class, int[])} 109 */ 110 @Deprecated 111 public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) { 112 this(loader, name, elements, DTypeUtils.getInterface(dtype), shape); 113 } 114 115 /** 116 * Create a lazy dataset 117 * @param name of dataset 118 * @param dtype dataset type 119 * @param shape dataset shape 120 * @param loader lazy loader 121 * @deprecated Use {@link #LazyDataset(ILazyLoader, String, int, Class, int[])} 122 */ 123 @Deprecated 124 public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) { 125 this(name, dtype, 1, shape, loader); 126 } 127 128 LazyDataset(LazyDataset other) { 129 name = other.name; 130 shape = other.shape.clone(); 131 metadata = other.copyMetadata(); 132 oMetadata = other.oMetadata; 133 oShape = other.oShape; 134 size = other.size; 135 clazz = other.clazz; 136 isize = other.isize; 137 loader = other.loader; 138 139 begSlice = other.begSlice; 140 delSlice = other.delSlice; 141 sShape = other.sShape; 142 padding = other.padding; 143 map = other.map; 144 } 145 146 /** 147 * Create a lazy dataset based on in-memory data (handy for testing) 148 * @param dataset input 149 * @return lazy dataset 150 */ 151 public static LazyDataset createLazyDataset(final Dataset dataset) { 152 return new LazyDataset(new ILazyLoader() { 153 private static final long serialVersionUID = -6725268922780517523L; 154 155 final Dataset d = dataset; 156 157 @Override 158 public boolean isFileReadable() { 159 return true; 160 } 161 162 @Override 163 public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException { 164 return d.getSlice(mon, slice); 165 } 166 }, dataset.getName(), dataset.getElementsPerItem(), dataset.getClass(), dataset.getShapeRef()); 167 } 168 169 @Override 170 public Class<?> getElementClass() { 171 return InterfaceUtils.getElementClass(clazz); 172 } 173 174 /** 175 * Can return -1 when dataset interface is not defined 176 */ 177 @Override 178 public int getDType() { 179 return clazz == null ? -1 : DTypeUtils.getDType(clazz); 180 } 181 182 /** 183 * @return dataset interface that supports element class and number of elements. Can be null when undefined 184 * @since 2.3 185 */ 186 public Class<? extends Dataset> getInterface() { 187 return clazz; 188 } 189 190 /** 191 * Set interface 192 * @param clazz dataset sub-interface 193 * @since 2.3 194 */ 195 public void setInterface(Class<? extends Dataset> clazz) { 196 this.clazz = clazz; 197 } 198 199 /** 200 * Can return -1 for unknown 201 */ 202 @Override 203 public int getElementsPerItem() { 204 return isize; 205 } 206 207 @Override 208 public int getSize() { 209 return (int) size; 210 } 211 212 @Override 213 public String toString() { 214 StringBuilder out = new StringBuilder(); 215 216 if (name != null && name.length() > 0) { 217 out.append("Lazy dataset '"); 218 out.append(name); 219 out.append("' has shape ["); 220 } else { 221 out.append("Lazy dataset shape is ["); 222 } 223 int rank = shape == null ? 0 : shape.length; 224 225 if (rank > 0 && shape[0] >= 0) { 226 out.append(shape[0]); 227 } 228 for (int i = 1; i < rank; i++) { 229 out.append(", " + shape[i]); 230 } 231 out.append(']'); 232 233 return out.toString(); 234 } 235 236 @Override 237 public int hashCode() { 238 final int prime = 31; 239 int result = super.hashCode(); 240 result = prime * result + Arrays.hashCode(oShape); 241 result = prime * result + (int) (size ^ (size >>> 32)); 242 result = prime * result + Objects.hashCode(clazz); 243 result = prime * result + isize; 244 result = prime * result + Objects.hashCode(loader); 245 result = prime * result + Arrays.hashCode(begSlice); 246 result = prime * result + Arrays.hashCode(delSlice); 247 result = prime * result + Arrays.hashCode(sShape); 248 result = prime * result + Arrays.hashCode(padding); 249 result = prime * result + Arrays.hashCode(map); 250 return result; 251 } 252 253 @Override 254 public boolean equals(Object obj) { 255 if (this == obj) { 256 return true; 257 } 258 if (!super.equals(obj)) { 259 return false; 260 } 261 262 LazyDataset other = (LazyDataset) obj; 263 if (!Arrays.equals(oShape, other.oShape)) { 264 return false; 265 } 266 if (size != other.size) { 267 return false; 268 } 269 if (!Objects.equals(clazz, other.clazz)) { 270 return false; 271 } 272 if (isize != other.isize) { 273 return false; 274 } 275 276 if (loader != other.loader) { 277 return false; 278 } 279 280 if (!Arrays.equals(begSlice, other.begSlice)) { 281 return false; 282 } 283 if (!Arrays.equals(delSlice, other.delSlice)) { 284 return false; 285 } 286 if (!Arrays.equals(sShape, other.sShape)) { 287 return false; 288 } 289 if (!Arrays.equals(padding, other.padding)) { 290 return false; 291 } 292 if (!Arrays.equals(map, other.map)) { 293 return false; 294 } 295 296 return true; 297 } 298 299 @Override 300 public LazyDataset clone() { 301 return new LazyDataset(this); 302 } 303 304 @Override 305 public void setShape(int... shape) { 306 setShapeInternal(shape.clone()); 307 } 308 309 @Override 310 public LazyDataset squeezeEnds() { 311 setShapeInternal(ShapeUtils.squeezeShape(shape, true)); 312 return this; 313 } 314 315 @Override 316 public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException { 317 return getSlice(null, start, stop, step); 318 } 319 320 @Override 321 public Dataset getSlice(Slice... slice) throws DatasetException { 322 if (slice == null || slice.length == 0) { 323 return internalGetSlice(null, new SliceND(shape)); 324 } 325 return internalGetSlice(null, new SliceND(shape, slice)); 326 } 327 328 @Override 329 public Dataset getSlice(SliceND slice) throws DatasetException { 330 return getSlice(null, slice); 331 } 332 333 @Override 334 public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException { 335 if (slice == null || slice.length == 0) { 336 return internalGetSlice(monitor, new SliceND(shape)); 337 } 338 return internalGetSlice(monitor, new SliceND(shape, slice)); 339 } 340 341 @Override 342 public LazyDataset getSliceView(Slice... slice) { 343 if (slice == null || slice.length == 0) { 344 return internalGetSliceView(new SliceND(shape)); 345 } 346 return internalGetSliceView(new SliceND(shape, slice)); 347 } 348 349 /** 350 * @param nShape 351 */ 352 private void setShapeInternal(int... nShape) { 353 // work out transposed (sliced) shape (instead of removing padding from current shape) 354 if (size != 0) { 355 int[] pShape = calcTransposed(map, sShape == null ? oShape : sShape); 356 padding = ShapeUtils.calcShapePadding(pShape, nShape); 357 } 358 359 if (metadata != null) { 360 storeMetadata(metadata, Reshapeable.class); 361 metadata = copyMetadata(); 362 reshapeMetadata(shape, nShape); 363 } 364 shape = nShape; 365 } 366 367 @Override 368 public LazyDataset getSliceView(int[] start, int[] stop, int[] step) { 369 return internalGetSliceView(new SliceND(shape, start, stop, step)); 370 } 371 372 @Override 373 public LazyDataset getSliceView(SliceND slice) { 374 if (slice != null) { 375 checkSliceND(slice); 376 } 377 return internalGetSliceView(slice); 378 } 379 380 protected LazyDataset internalGetSliceView(SliceND slice) { 381 LazyDataset view = clone(); 382 if (slice == null || slice.isAll()) { 383 return view; 384 } 385 386 SliceND nslice = calcTrueSlice(slice); 387 if (nslice != null) { 388 view.begSlice = nslice.getStart(); 389 view.delSlice = nslice.getStep(); 390 view.sShape = nslice.getShape(); 391 } 392 view.shape = slice.getShape(); 393 view.size = ShapeUtils.calcLongSize(view.shape); 394 view.storeMetadata(metadata, Sliceable.class); 395 396 view.sliceMetadata(true, slice); 397 return view; 398 } 399 400 @Override 401 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException { 402 return internalGetSlice(monitor, new SliceND(shape, start, stop, step)); 403 } 404 405 @Override 406 public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException { 407 if (slice != null) { 408 checkSliceND(slice); 409 } 410 return internalGetSlice(monitor, slice); 411 } 412 413 protected Dataset internalGetSlice(IMonitor monitor, SliceND slice) throws DatasetException { 414 if (loader != null && !loader.isFileReadable()) { 415 return null; 416 } 417 418 SliceND nslice = calcTrueSlice(slice); 419 420 Dataset a; 421 if (nslice == null) { 422 a = DatasetFactory.zeros(clazz == null ? DoubleDataset.class : clazz, slice == null ? shape : slice.getShape()); 423 } else { 424 try { 425 a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice)); 426 } catch (IOException e) { 427 logger.error("Problem getting {}: {}", slice == null ? "all" : String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()), 428 Arrays.toString(slice.getStep()), loader), e); 429 throw new DatasetException(e); 430 } 431 } 432 a.setName(name + AbstractDataset.BLOCK_OPEN + (nslice == null ? 433 (slice == null ? AbstractDataset.ELLIPSIS : slice) : nslice) + AbstractDataset.BLOCK_CLOSE); 434 if (metadata != null && a instanceof LazyDatasetBase) { 435 LazyDatasetBase ba = (LazyDatasetBase) a; 436 ba.metadata = copyMetadata(); 437 if (oMetadata != null) { 438 ba.restoreMetadata(oMetadata); 439 } 440 // metadata axis may be larger than data 441 if (nslice != null && (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape())) { 442 ba.sliceMetadata(true, nslice); 443 } 444 } 445 446 if (nslice != null) { 447 if (map != null) { 448 a = a.getTransposedView(map); 449 } 450 if (padding != null) { 451 a.setShape(slice == null ? shape : slice.getShape()); 452 } 453 } 454 a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice == null ? 455 (slice == null ? null : slice.convertToSlice()) : nslice.convertToSlice(), oShape, null, name)); 456 457 if (clazz == null) { 458 clazz = a.getClass(); 459 } 460 return a; 461 } 462 463 @Override 464 public LazyDataset getTransposedView(final int... axes) { 465 LazyDataset view = clone(); 466 467 int[] naxes = checkPermutatedAxes(shape, axes); 468 if (naxes == null) { 469 return view; 470 } 471 472 view.shape = calcTransposed(naxes, shape); 473 if (view.size != 0 && padding != null) { // work out transpose by reverting effect of padding 474 int or = oShape.length; 475 int nr = shape.length; 476 int j = 0; // naxes index 477 int[] mShape = calcTransposed(map, sShape == null ? oShape : sShape); // pre-padded shape 478 int m = 0; // shape index 479 int e = -1; // index of unit dimension 480 final List<Integer> uaxes = new LinkedList<>(); 481 for (int a : naxes) { 482 uaxes.add(a); 483 } 484 List<Integer> oList = new ArrayList<>(); // dimensions left out by padding (in order) 485 int np = padding.length; 486 for (int i = 0; i < np; i++) { 487 int p = padding[i]; 488 if (p > 0) { // remove added dimensions 489 for (int k = 0; k < p; k++, j++) { 490 uaxes.remove((Integer) j); 491 } 492 } else if (p == 0) { // leave alone 493 if (mShape[m] == 1) { // bump up last unit dimension index 494 e = m; 495 } 496 j++; 497 m++; 498 } else { // add omitted dimensions to list 499 p = -p; 500 for (int k = 0; k < p; k++) { 501 e = find(mShape, 1, e + 1); 502 oList.add(e); 503 } 504 } 505 } 506 507 int[] omitted = new int[oList.size()]; 508 j = 0; 509 for (Integer o : oList) { 510 omitted[j++] = o; 511 } 512 int[] used = new int[or - omitted.length]; // all dimensions not omitted in pre-padded shape 513 j = 0; 514 for (int i = 0; i < or; i++) { 515 if (Arrays.binarySearch(omitted, i) < 0) { 516 used[j++] = i; 517 } 518 } 519 520 int[] vaxes = new int[uaxes.size()]; 521 j = 0; 522 for (int i = 0; i < nr; i++) { // remap dimension numbering 523 int l = uaxes.indexOf(i); 524 if (l >= 0) { 525 vaxes[l] = used[j++]; 526 } 527 } 528 int[] taxes = new int[or]; 529 j = 0; 530 for (int i = 0; i < or; i++) { // reassemble map 531 if (Arrays.binarySearch(omitted, i) >= 0) { 532 taxes[i] = i; 533 } else { 534 taxes[i] = vaxes[j++]; 535 } 536 } 537 538 naxes = taxes; 539 } 540 541 view.map = map == null ? naxes : calcTransposed(naxes, map); 542 if (view.size != 0) { 543 // work out transposed (sliced) shape 544 int[] tShape = calcTransposed(view.map, sShape == null ? oShape : sShape); 545 try { 546 view.padding = ShapeUtils.calcShapePadding(tShape, view.shape); 547 } catch (IllegalArgumentException e) { 548 System.err.println(e.getMessage() + ": " + Arrays.toString(tShape) + " cf " + Arrays.toString(view.shape)); 549 } 550 } 551 view.storeMetadata(metadata, Transposable.class); 552 view.transposeMetadata(axes); 553 return view; 554 } 555 556 private static int find(int[] map, int m, int off) { 557 for (int i = off, imax = map.length; i < imax; i++) { 558 if (map[i] == m) { 559 return i; 560 } 561 } 562 return -1; 563 } 564 565 private static int[] calcTransposed(int[] map, int[] values) { 566 if (values == null) { 567 return null; 568 } 569 int r = values.length; 570 if (map == null || r < 2) { 571 return values; 572 } 573 int[] ovalues = new int[r]; 574 for (int i = 0; i < r; i++) { 575 ovalues[i] = values[map[i]]; 576 } 577 return ovalues; 578 } 579 580 /** 581 * Calculate absolute slice 582 * @param slice an n-D slice 583 * @return true slice or null if zero-sized 584 */ 585 protected final SliceND calcTrueSlice(SliceND slice) { 586 /* 587 * Lazy dataset operations: getTransposedView (T), getSliceView (G), setShape/squeezeEnds (S+/S-): 588 * 589 * . T sets shape, base, and map in new view 590 * . G sets shape, size, begSlice and delSlice in new view 591 * . S sets shape, shapePadding in current view 592 * 593 * Then getSlice needs to interpret all info to find true slice, load data, get transposition (view) 594 * and set shape. Therefore: 595 * . S needs to update shapePadding only 596 * . T needs to update shapePadding too 597 * . G needs to work out true slice to update 598 * 599 * slice -> true slice 600 * adjusts for shape (S^-1) then remap dimensions (T^-1) 601 */ 602 603 if (slice == null) { 604 slice = new SliceND(shape); 605 } 606 607 if (ShapeUtils.calcLongSize(slice.getShape()) == 0) { 608 return null; 609 } 610 611 int[] nshape; 612 int[] nstart; 613 int[] nstep; 614 615 int r = oShape.length; 616 if (padding == null) { 617 nshape = slice.getShape(); 618 nstart = slice.getStart(); 619 nstep = slice.getStep(); 620 } else { 621 final int[] lshape = slice.getShape(); 622 final int[] lstart = slice.getStart(); 623 final int[] lstep = slice.getStep(); 624 625 nstart = new int[r]; 626 nstep = new int[r]; 627 nshape = new int[r]; 628 int i = 0; 629 int j = 0; 630 for (int p : padding) { // remove padding 631 if (p == 0) { 632 nshape[i] = lshape[j]; 633 nstart[i] = lstart[j]; 634 nstep[i] = lstep[j]; 635 i++; 636 j++; 637 } else if (p < 0) { 638 int imax = i - p; 639 while (i < imax) { 640 nshape[i] = 1; 641 nstep[i] = 1; 642 i++; 643 } 644 } else { 645 j += p; 646 } 647 } 648 } 649 650 if (map != null && r > 1) { // transpose dimensions 651 int[] pshape = new int[r]; 652 int[] pstart = new int[r]; 653 int[] pstep = new int[r]; 654 for (int i = 0; i < r; i++) { 655 int m = map[i]; 656 pshape[m] = nshape[i]; 657 pstart[m] = nstart[i]; 658 pstep[m] = nstep[i]; 659 } 660 661 nshape = pshape; 662 nstart = pstart; 663 nstep = pstep; 664 } 665 666 int[] nstop = new int[r]; 667 if (begSlice != null) { // find net slice 668 for (int i = 0; i < r; i++) { 669 int b = begSlice[i]; 670 int d = delSlice[i]; 671 nstart[i] = b + nstart[i] * d; 672 int nd = nstep[i] * d; 673 nstep[i] = nd; 674 nstop[i] = nstart[i] + (nshape[i] - 1) * nd + (nd >= 0 ? 1 : -1); 675 } 676 } else { 677 for (int i = 0; i < r; i++) { 678 int d = nstep[i]; 679 nstop[i] = nstart[i] + (nshape[i] - 1) * d + (d >= 0 ? 1 : -1); 680 } 681 } 682 683 return createSlice(nstart, nstop, nstep); 684 } 685 686 protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) { 687 return SliceND.createSlice(oShape, null, nstart, nstop, nstep); 688 } 689 690 /** 691 * Transform data so that it can be used in setSlice of saver 692 * @param data input 693 * @param tslice true slice 694 * @return data with dimensions adjusted and remapped 695 */ 696 final IDataset transformInput(IDataset data, SliceND tslice) { 697 if (padding != null) { // remove padding 698 data = data.getSliceView(); 699 int[] nshape = tslice == null ? shape : tslice.getShape(); 700 data.setShape(nshape); 701 } 702 703 return map == null ? data : data.getTransposedView(map); 704 } 705 706 /** 707 * Store metadata items that has given annotation 708 * @param origMetadata original metadata 709 * @param aclazz annotation class 710 */ 711 private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) { 712 List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz); 713 if (mclazzes.size() == 0) { 714 return; 715 } 716 717 if (oMetadata == null) { 718 oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>(); 719 } 720 for (Class<? extends MetadataType> mc : mclazzes) { 721 if (oMetadata.containsKey(mc)) { 722 continue; // do not overwrite original 723 } 724 725 List<MetadataType> l = origMetadata.get(mc); 726 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 727 for (MetadataType m : l) { 728 nl.add(m.clone()); 729 } 730 oMetadata.put(mc, nl); 731 } 732 } 733 734 @SuppressWarnings("unchecked") 735 private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) { 736 List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>(); 737 if (metadata == null) { 738 return mclazzes; 739 } 740 741 for (Class<? extends MetadataType> c : metadata.keySet()) { 742 boolean hasAnn = false; 743 for (MetadataType m : metadata.get(c)) { 744 if (m == null) { 745 continue; 746 } 747 748 Class<? extends MetadataType> mc = m.getClass(); 749 do { // iterate over super-classes 750 for (Field f : mc.getDeclaredFields()) { 751 if (f.isAnnotationPresent(aclazz)) { 752 hasAnn = true; 753 break; 754 } 755 } 756 Class<?> sclazz = mc.getSuperclass(); 757 if (!MetadataType.class.isAssignableFrom(sclazz)) { 758 break; 759 } 760 mc = (Class<? extends MetadataType>) sclazz; 761 } while (!hasAnn); 762 if (hasAnn) { 763 break; 764 } 765 } 766 if (hasAnn) { 767 mclazzes.add(c); 768 } 769 } 770 return mclazzes; 771 } 772 773 /** 774 * Gets the maximum size of a slice of a dataset in a given dimension 775 * which should normally fit in memory. Note that it might be possible 776 * to get more in memory, this is a conservative estimate and seems to 777 * almost always work at the size returned; providing Xmx is less than 778 * the physical memory. 779 * 780 * To get more in memory increase -Xmx setting or use an expression 781 * which calls a rolling function (like rmean) instead of slicing directly 782 * to memory. 783 * 784 * @param lazySet lazy dataset 785 * @param dimension to slice along 786 * @return maximum size of dimension that can be sliced. 787 */ 788 public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) { 789 // size in bytes of each item 790 final double size = InterfaceUtils.getItemBytes(lazySet.getElementsPerItem(), InterfaceUtils.getInterface(lazySet)); 791 792 // Max in bytes takes into account our minimum requirement 793 final double max = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory()); 794 795 // Firstly if the whole dataset it likely to fit in memory, then we allow it. 796 // Space specified in bytes per item available 797 final double space = max/lazySet.getSize(); 798 799 // If we have room for this whole dataset, then fine 800 int[] shape = lazySet.getShape(); 801 if (space >= size) { 802 return shape[dimension]; 803 } 804 805 // Otherwise estimate what we can fit in, conservatively. 806 // First get size of one slice, see it that fits, if not, still return 1 807 double sizeOneSlice = size; // in bytes 808 for (int dim = 0; dim < shape.length; dim++) { 809 if (dim == dimension) { 810 continue; 811 } 812 sizeOneSlice *= shape[dim]; 813 } 814 double avail = max / sizeOneSlice; 815 if (avail < 1) { 816 return 1; 817 } 818 819 // We fudge this to leave some room 820 return (int) Math.floor(avail/4d); 821 } 822}