Examples with SketchesArgumentException com.yahoo.sketches.SketchesArgumentException used on opensource projects

Example 31 with SketchesArgumentException

use of com.yahoo.sketches.SketchesArgumentException in project sketches-core by DataSketches.

the class DoublesMergeImpl method downSamplingMergeInto.

/**
   * Merges the source sketch into the target sketch that can have a smaller value of K.
   * However, it is required that the ratio of the two K values be a power of 2.
   * I.e., source.getK() = target.getK() * 2^(nonnegative integer).
   * The source is not modified.
   *
   * @param src The source sketch
   * @param tgt The target sketch
   */
//also used by DoublesSketch, DoublesUnionImpl and HeapDoublesSketchTest
static void downSamplingMergeInto(final DoublesSketch src, final UpdateDoublesSketch tgt) {
    final int srcK = src.getK();
    final int tgtK = tgt.getK();
    final long tgtN = tgt.getN();
    if ((srcK % tgtK) != 0) {
        throw new SketchesArgumentException("source.getK() must equal target.getK() * 2^(nonnegative integer).");
    }
    final int downFactor = srcK / tgtK;
    checkIfPowerOf2(downFactor, "source.getK()/target.getK() ratio");
    final int lgDownFactor = Integer.numberOfTrailingZeros(downFactor);
    final DoublesSketchAccessor srcSketchBuf = DoublesSketchAccessor.wrap(src);
    final long nFinal = tgtN + src.getN();
    for (int i = 0; i < srcSketchBuf.numItems(); i++) {
        // update only the base buffer
        tgt.update(srcSketchBuf.get(i));
    }
    final int spaceNeeded = DoublesUpdateImpl.getRequiredItemCapacity(tgtK, nFinal);
    final int curCombBufCap = tgt.getCombinedBufferItemCapacity();
    if (spaceNeeded > curCombBufCap) {
        //copies base buffer plus current levels
        tgt.growCombinedBuffer(curCombBufCap, spaceNeeded);
    }
    //working scratch buffers
    final DoublesArrayAccessor scratch2KAcc = DoublesArrayAccessor.initialize(2 * tgtK);
    final DoublesArrayAccessor downScratchKAcc = DoublesArrayAccessor.initialize(tgtK);
    final DoublesSketchAccessor tgtSketchBuf = DoublesSketchAccessor.wrap(tgt, true);
    long srcBitPattern = src.getBitPattern();
    long newTgtBitPattern = tgt.getBitPattern();
    for (int srcLvl = 0; srcBitPattern != 0L; srcLvl++, srcBitPattern >>>= 1) {
        if ((srcBitPattern & 1L) > 0L) {
            justZipWithStride(srcSketchBuf.setLevel(srcLvl), downScratchKAcc, tgtK, downFactor);
            newTgtBitPattern = DoublesUpdateImpl.inPlacePropagateCarry(//starting level
            srcLvl + lgDownFactor, //optSrcKBuf,
            downScratchKAcc, //size2KBuf,
            scratch2KAcc, //do mergeInto version
            false, tgtK, tgtSketchBuf, newTgtBitPattern);
            //off-heap is a no-op
            tgt.putBitPattern(newTgtBitPattern);
        }
    }
    if (tgt.isDirect() && (nFinal > 0)) {
        final WritableMemory mem = tgt.getMemory();
        mem.clearBits(FLAGS_BYTE, (byte) EMPTY_FLAG_MASK);
    }
    tgt.putN(nFinal);
    // internal consistency check
    assert tgt.getN() / (2 * tgtK) == newTgtBitPattern;
    final double srcMax = src.getMaxValue();
    final double srcMin = src.getMinValue();
    final double tgtMax = tgt.getMaxValue();
    final double tgtMin = tgt.getMinValue();
    if (srcMax > tgtMax) {
        tgt.putMaxValue(srcMax);
    }
    if (srcMin < tgtMin) {
        tgt.putMinValue(srcMin);
    }
}

Example 32 with SketchesArgumentException

use of com.yahoo.sketches.SketchesArgumentException in project sketches-core by DataSketches.

the class DoublesSketch method putMemory.

/**
   * Puts the current sketch into the given Memory if there is sufficient space, otherwise,
   * throws an error.
   *
   * @param dstMem the given memory.
   * @param compact if true, compacts and sorts the base buffer, which optimizes merge
   *                performance at the cost of slightly increased serialization time.
   */
public void putMemory(final WritableMemory dstMem, final boolean compact) {
    if (isDirect() && isCompact() == compact) {
        final Memory srcMem = getMemory();
        srcMem.copyTo(0, dstMem, 0, getStorageBytes());
    } else {
        final byte[] byteArr = toByteArray(compact);
        final int arrLen = byteArr.length;
        final long memCap = dstMem.getCapacity();
        if (memCap < arrLen) {
            throw new SketchesArgumentException("Destination Memory not large enough: " + memCap + " < " + arrLen);
        }
        dstMem.putByteArray(0, byteArr, 0, arrLen);
    }
}

Example 33 with SketchesArgumentException

use of com.yahoo.sketches.SketchesArgumentException in project sketches-core by DataSketches.

the class ItemsSketch method getInstance.

/**
   * Heapifies the given srcMem, which must be a Memory image of a ItemsSketch
   * @param <T> type of item
   * @param srcMem a Memory image of a sketch.
   * <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
   * @param comparator to compare items
   * @param serDe an instance of ArrayOfItemsSerDe
   * @return a ItemsSketch on the Java heap.
   */
public static <T> ItemsSketch<T> getInstance(final Memory srcMem, final Comparator<? super T> comparator, final ArrayOfItemsSerDe<T> serDe) {
    final long memCapBytes = srcMem.getCapacity();
    if (memCapBytes < 8) {
        throw new SketchesArgumentException("Memory too small: " + memCapBytes);
    }
    final int preambleLongs = extractPreLongs(srcMem);
    final int serVer = extractSerVer(srcMem);
    final int familyID = extractFamilyID(srcMem);
    final int flags = extractFlags(srcMem);
    final int k = extractK(srcMem);
    ItemsUtil.checkItemsSerVer(serVer);
    if ((serVer == 3) && ((flags & COMPACT_FLAG_MASK) == 0)) {
        throw new SketchesArgumentException("Non-compact Memory images are not supported.");
    }
    final boolean empty = Util.checkPreLongsFlagsCap(preambleLongs, flags, memCapBytes);
    Util.checkFamilyID(familyID);
    //checks k
    final ItemsSketch<T> qs = getInstance(k, comparator);
    if (empty) {
        return qs;
    }
    //Not empty, must have valid preamble + min, max
    final long n = extractN(srcMem);
    //can't check memory capacity here, not enough information
    //for min, max
    final int extra = 2;
    final int numMemItems = Util.computeRetainedItems(k, n) + extra;
    //set class members
    qs.n_ = n;
    qs.combinedBufferItemCapacity_ = Util.computeCombinedBufferItemCapacity(k, n);
    qs.baseBufferCount_ = computeBaseBufferItems(k, n);
    qs.bitPattern_ = computeBitPattern(k, n);
    qs.combinedBuffer_ = new Object[qs.combinedBufferItemCapacity_];
    final int srcMemItemsOffsetBytes = preambleLongs * Long.BYTES;
    final Memory mReg = srcMem.region(srcMemItemsOffsetBytes, srcMem.getCapacity() - srcMemItemsOffsetBytes);
    final T[] itemsArray = serDe.deserializeFromMemory(mReg, numMemItems);
    qs.itemsArrayToCombinedBuffer(itemsArray);
    return qs;
}

Example 34 with SketchesArgumentException

use of com.yahoo.sketches.SketchesArgumentException in project sketches-core by DataSketches.

the class ReservoirItemsUnion method heapify.

/**
   * Instantiates a Union from Memory
   *
   * @param <T> The type of item this sketch contains
   * @param srcMem Memory object containing a serialized union
   * @param serDe An instance of ArrayOfItemsSerDe
   * @return A ReservoirItemsUnion created from the provided Memory
   */
public static <T> ReservoirItemsUnion<T> heapify(final Memory srcMem, final ArrayOfItemsSerDe<T> serDe) {
    Family.RESERVOIR_UNION.checkFamilyID(srcMem.getByte(FAMILY_BYTE));
    final int numPreLongs = extractPreLongs(srcMem);
    final int serVer = extractSerVer(srcMem);
    final boolean isEmpty = (extractFlags(srcMem) & EMPTY_FLAG_MASK) != 0;
    int maxK = extractMaxK(srcMem);
    final boolean preLongsEqMin = (numPreLongs == Family.RESERVOIR_UNION.getMinPreLongs());
    final boolean preLongsEqMax = (numPreLongs == Family.RESERVOIR_UNION.getMaxPreLongs());
    if (!preLongsEqMin & !preLongsEqMax) {
        throw new SketchesArgumentException("Possible corruption: Non-empty union with only " + Family.RESERVOIR_UNION.getMinPreLongs() + "preLongs");
    }
    if (serVer != SER_VER) {
        if (serVer == 1) {
            final short encMaxK = extractEncodedReservoirSize(srcMem);
            maxK = ReservoirSize.decodeValue(encMaxK);
        } else {
            throw new SketchesArgumentException("Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
        }
    }
    final ReservoirItemsUnion<T> riu = new ReservoirItemsUnion<>(maxK);
    if (!isEmpty) {
        final int preLongBytes = numPreLongs << 3;
        final Memory sketchMem = srcMem.region(preLongBytes, srcMem.getCapacity() - preLongBytes);
        riu.update(sketchMem, serDe);
    }
    return riu;
}

Example 35 with SketchesArgumentException

use of com.yahoo.sketches.SketchesArgumentException in project sketches-core by DataSketches.

the class ReservoirLongsSketch method heapify.

/**
   * Returns a sketch instance of this class from the given srcMem, which must be a Memory
   * representation of this sketch class.
   *
   * @param srcMem a Memory representation of a sketch of this class. <a href=
   *        "{@docRoot}/resources/dictionary.html#mem">See Memory</a>
   * @return a sketch instance of this class
   */
public static ReservoirLongsSketch heapify(final Memory srcMem) {
    Family.RESERVOIR.checkFamilyID(srcMem.getByte(FAMILY_BYTE));
    final int numPreLongs = extractPreLongs(srcMem);
    final ResizeFactor rf = ResizeFactor.getRF(extractResizeFactor(srcMem));
    final int serVer = extractSerVer(srcMem);
    final boolean isEmpty = (extractFlags(srcMem) & EMPTY_FLAG_MASK) != 0;
    final long itemsSeen = (isEmpty ? 0 : extractN(srcMem));
    int k = extractK(srcMem);
    // Check values
    final boolean preLongsEqMin = (numPreLongs == Family.RESERVOIR.getMinPreLongs());
    final boolean preLongsEqMax = (numPreLongs == Family.RESERVOIR.getMaxPreLongs());
    if (!preLongsEqMin & !preLongsEqMax) {
        throw new SketchesArgumentException("Possible corruption: Non-empty sketch with only " + Family.RESERVOIR.getMinPreLongs() + "preLongs");
    }
    if (serVer != SER_VER) {
        if (serVer == 1) {
            final short encK = extractEncodedReservoirSize(srcMem);
            k = ReservoirSize.decodeValue(encK);
        } else {
            throw new SketchesArgumentException("Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
        }
    }
    if (isEmpty) {
        return new ReservoirLongsSketch(k, rf);
    }
    final int preLongBytes = numPreLongs << 3;
    final int numSketchLongs = (int) Math.min(itemsSeen, k);
    // default to full reservoir
    int allocatedSize = k;
    if (itemsSeen < k) {
        // under-full so determine size to allocate, using ceilingLog2(totalSeen) as minimum
        // casts to int are safe since under-full
        final int ceilingLgK = Util.toLog2(Util.ceilingPowerOf2(k), "heapify");
        final int minLgSize = Util.toLog2(Util.ceilingPowerOf2((int) itemsSeen), "heapify");
        final int initialLgSize = SamplingUtil.startingSubMultiple(ceilingLgK, rf.lg(), Math.max(minLgSize, MIN_LG_ARR_LONGS));
        allocatedSize = SamplingUtil.getAdjustedSize(k, 1 << initialLgSize);
    }
    final long[] data = new long[allocatedSize];
    srcMem.getLongArray(preLongBytes, data, 0, numSketchLongs);
    return new ReservoirLongsSketch(data, itemsSeen, rf, k);
}