Examples with SketchesArgumentException com.yahoo.sketches.SketchesArgumentException used on opensource projects

Example 21 with SketchesArgumentException

use of com.yahoo.sketches.SketchesArgumentException in project sketches-core by DataSketches.

the class VarOptItemsSketch method heapify.

/**
   * Returns a sketch instance of this class from the given srcMem,
   * which must be a Memory representation of this sketch class.
   *
   * @param <T>    The type of item this sketch contains
   * @param srcMem a Memory representation of a sketch of this class.
   *               <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
   * @param serDe  An instance of ArrayOfItemsSerDe
   * @return a sketch instance of this class
   */
@SuppressWarnings("null")
public static <T> VarOptItemsSketch<T> heapify(final Memory srcMem, final ArrayOfItemsSerDe<T> serDe) {
    final int numPreLongs = getAndCheckPreLongs(srcMem);
    final ResizeFactor rf = ResizeFactor.getRF(extractResizeFactor(srcMem));
    final int serVer = extractSerVer(srcMem);
    final int familyId = extractFamilyID(srcMem);
    final int flags = extractFlags(srcMem);
    final boolean isEmpty = (flags & EMPTY_FLAG_MASK) != 0;
    final boolean isGadget = (flags & GADGET_FLAG_MASK) != 0;
    // Check values
    if (numPreLongs != Family.VAROPT.getMinPreLongs() && numPreLongs != Family.VAROPT.getMaxPreLongs() && numPreLongs != PreambleUtil.VO_WARMUP_PRELONGS) {
        throw new SketchesArgumentException("Possible corruption: Must have " + Family.VAROPT.getMinPreLongs() + ", " + PreambleUtil.VO_WARMUP_PRELONGS + ", or " + Family.VAROPT.getMaxPreLongs() + " preLongs. Found: " + numPreLongs);
    }
    if (serVer != SER_VER) {
        throw new SketchesArgumentException("Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
    }
    final int reqFamilyId = Family.VAROPT.getID();
    if (familyId != reqFamilyId) {
        throw new SketchesArgumentException("Possible Corruption: FamilyID must be " + reqFamilyId + ": " + familyId);
    }
    final int k = extractK(srcMem);
    if (k < 2) {
        throw new SketchesArgumentException("Possible Corruption: k must be at least 2: " + k);
    }
    if (isEmpty) {
        assert numPreLongs == Family.VAROPT.getMinPreLongs();
        return new VarOptItemsSketch<>(k, rf);
    }
    final long n = extractN(srcMem);
    if (n < 0) {
        throw new SketchesArgumentException("Possible Corruption: n cannot be negative: " + n);
    }
    // get rest of preamble
    final int hCount = extractHRegionItemCount(srcMem);
    final int rCount = extractRRegionItemCount(srcMem);
    if (hCount < 0) {
        throw new SketchesArgumentException("Possible Corruption: H region count cannot be " + "negative: " + hCount);
    }
    if (rCount < 0) {
        throw new SketchesArgumentException("Possible Corruption: R region count cannot be " + "negative: " + rCount);
    }
    double totalRWeight = 0.0;
    if (numPreLongs == Family.VAROPT.getMaxPreLongs()) {
        if (rCount > 0) {
            totalRWeight = extractTotalRWeight(srcMem);
        } else {
            throw new SketchesArgumentException("Possible Corruption: " + Family.VAROPT.getMaxPreLongs() + " preLongs but no items in R region");
        }
    }
    final int preLongBytes = numPreLongs << 3;
    final int totalItems = hCount + rCount;
    // default to full
    int allocatedItems = k + 1;
    if (rCount == 0) {
        // Not in sampling mode, so determine size to allocate, using ceilingLog2(hCount) as minimum
        final int ceilingLgK = Util.toLog2(Util.ceilingPowerOf2(k), "heapify");
        final int minLgSize = Util.toLog2(Util.ceilingPowerOf2(hCount), "heapify");
        final int initialLgSize = SamplingUtil.startingSubMultiple(ceilingLgK, rf.lg(), Math.max(minLgSize, MIN_LG_ARR_ITEMS));
        allocatedItems = SamplingUtil.getAdjustedSize(k, 1 << initialLgSize);
        if (allocatedItems == k) {
            ++allocatedItems;
        }
    }
    // allocate full-sized ArrayLists, but we store only hCount weights at any moment
    final long weightOffsetBytes = TOTAL_WEIGHT_R_DOUBLE + (rCount > 0 ? Double.BYTES : 0);
    final ArrayList<Double> weightList = new ArrayList<>(allocatedItems);
    final double[] wts = new double[allocatedItems];
    srcMem.getDoubleArray(weightOffsetBytes, wts, 0, hCount);
    // can't use Arrays.asList(wts) since double[] rather than Double[]
    for (int i = 0; i < hCount; ++i) {
        if (wts[i] <= 0.0) {
            throw new SketchesArgumentException("Possible Corruption: " + "Non-positive weight in heapify(): " + wts[i]);
        }
        weightList.add(wts[i]);
    }
    // marks, if we have a gadget
    long markBytes = 0;
    int markCount = 0;
    ArrayList<Boolean> markList = null;
    if (isGadget) {
        final long markOffsetBytes = preLongBytes + (hCount * Double.BYTES);
        markBytes = ArrayOfBooleansSerDe.computeBytesNeeded(hCount);
        markList = new ArrayList<>(allocatedItems);
        final ArrayOfBooleansSerDe booleansSerDe = new ArrayOfBooleansSerDe();
        final Boolean[] markArray = booleansSerDe.deserializeFromMemory(srcMem.region(markOffsetBytes, (hCount >> 3) + 1), hCount);
        for (Boolean mark : markArray) {
            if (mark) {
                ++markCount;
            }
        }
        markList.addAll(Arrays.asList(markArray));
    }
    final long offsetBytes = preLongBytes + (hCount * Double.BYTES) + markBytes;
    final T[] data = serDe.deserializeFromMemory(srcMem.region(offsetBytes, srcMem.getCapacity() - offsetBytes), totalItems);
    final List<T> wrappedData = Arrays.asList(data);
    final ArrayList<T> dataList = new ArrayList<>(allocatedItems);
    dataList.addAll(wrappedData.subList(0, hCount));
    // Load items in R as needed
    if (rCount > 0) {
        // the gap
        weightList.add(-1.0);
        // the gap
        if (isGadget) {
            markList.add(false);
        }
        for (int i = 0; i < rCount; ++i) {
            weightList.add(-1.0);
            if (isGadget) {
                markList.add(false);
            }
        }
        // the gap
        dataList.add(null);
        dataList.addAll(wrappedData.subList(hCount, totalItems));
    }
    final VarOptItemsSketch<T> sketch = new VarOptItemsSketch<>(dataList, weightList, k, n, allocatedItems, rf, hCount, rCount, totalRWeight);
    if (isGadget) {
        sketch.marks_ = markList;
        sketch.numMarksInH_ = markCount;
    }
    return sketch;
}

Example 22 with SketchesArgumentException

use of com.yahoo.sketches.SketchesArgumentException in project sketches-core by DataSketches.

the class PreambleUtil method preambleToString.

/**
   * Returns a human readable string summary of the preamble state of the given Memory.
   * Note: other than making sure that the given Memory size is large
   * enough for just the preamble, this does not do much value checking of the contents of the
   * preamble as this is primarily a tool for debugging the preamble visually.
   *
   * @param mem the given Memory.
   * @return the summary preamble string.
   */
public static String preambleToString(final Memory mem) {
    // make sure we can get the assumed preamble
    final int preLongs = getAndCheckPreLongs(mem);
    final Family family = Family.idToFamily(mem.getByte(FAMILY_BYTE));
    switch(family) {
        case RESERVOIR:
        case VAROPT:
            return sketchPreambleToString(mem, family, preLongs);
        case RESERVOIR_UNION:
        case VAROPT_UNION:
            return unionPreambleToString(mem, family, preLongs);
        default:
            throw new SketchesArgumentException("Inspecting preamble with Sampling family's " + "PreambleUtil with object of family " + family.getFamilyName());
    }
}

Example 23 with SketchesArgumentException

use of com.yahoo.sketches.SketchesArgumentException in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkBadMemory.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkBadMemory() {
    byte[] bytes = new byte[4];
    Memory mem = Memory.wrap(bytes);
    try {
        PreambleUtil.getAndCheckPreLongs(mem);
        fail();
    } catch (final SketchesArgumentException e) {
    // expected
    }
    bytes = new byte[8];
    // only 1 preLong worth of items in bytearray
    bytes[0] = 2;
    mem = Memory.wrap(bytes);
    PreambleUtil.getAndCheckPreLongs(mem);
}

Example 24 with SketchesArgumentException

use of com.yahoo.sketches.SketchesArgumentException in project sketches-core by DataSketches.

the class DirectQuickSelectSketchTest method checkConstructorSrcMemCorruptions.

@Test
public void checkConstructorSrcMemCorruptions() {
    //lgNomLongs = 10
    int k = 1024;
    //exact mode, lgArrLongs = 11
    int u = k;
    int bytes = Sketches.getMaxUpdateSketchBytes(k);
    byte[] arr1 = new byte[bytes];
    WritableMemory mem1 = WritableMemory.wrap(arr1);
    //0
    ResizeFactor rf = ResizeFactor.X1;
    UpdateSketch usk1 = UpdateSketch.builder().setNominalEntries(k).setResizeFactor(rf).build(mem1);
    for (int i = 0; i < u; i++) {
        usk1.update(i);
    }
    //println(PreambleUtil.toString(mem1));
    @SuppressWarnings("unused") UpdateSketch usk2;
    //corrupt Family by setting to Compact
    mem1.putByte(FAMILY_BYTE, (byte) 3);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem1, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //Pass
    }
    //fix Family
    mem1.putByte(FAMILY_BYTE, (byte) 2);
    //corrupt preLongs
    mem1.putByte(PREAMBLE_LONGS_BYTE, (byte) 1);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem1, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //pass
    }
    //fix preLongs
    mem1.putByte(PREAMBLE_LONGS_BYTE, (byte) 3);
    //corrupt serVer
    mem1.putByte(SER_VER_BYTE, (byte) 2);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem1, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //pass
    }
    //fix serVer
    mem1.putByte(SER_VER_BYTE, (byte) 3);
    //corrupt theta and
    mem1.putLong(THETA_LONG, Long.MAX_VALUE >>> 1);
    //corrupt lgArrLongs
    mem1.putByte(LG_ARR_LONGS_BYTE, (byte) 10);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem1, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //pass
    }
    //fix theta and
    mem1.putLong(THETA_LONG, Long.MAX_VALUE);
    //fix lgArrLongs
    mem1.putByte(LG_ARR_LONGS_BYTE, (byte) 11);
    byte badFlags = (byte) (BIG_ENDIAN_FLAG_MASK | COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK | ORDERED_FLAG_MASK);
    mem1.putByte(FLAGS_BYTE, badFlags);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem1, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //pass
    }
    //corrupt length
    byte[] arr2 = Arrays.copyOfRange(arr1, 0, bytes - 1);
    WritableMemory mem2 = WritableMemory.wrap(arr2);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem2, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //pass
    }
}

Example 25 with SketchesArgumentException

use of com.yahoo.sketches.SketchesArgumentException in project sketches-core by DataSketches.

the class ForwardCompatibilityTest method convertSerV3toSerV2.

/**
   * Converts a SerVer3 CompactSketch to a SerVer2 SetSketch.
   * @param v3mem a SerVer3 Compact, Ordered Sketch.
   * @return a SerVer2 SetSketch as Memory object
   */
public static WritableMemory convertSerV3toSerV2(Memory v3mem) {
    //validate that v3mem is in the right form
    int serVer = v3mem.getByte(SER_VER_BYTE);
    int famId = v3mem.getByte(FAMILY_BYTE);
    int flags = v3mem.getByte(FLAGS_BYTE);
    if ((serVer != 3) || (famId != 3) || ((flags & 24) != 24))
        throw new SketchesArgumentException("Memory must be V3, Compact, Ordered");
    //compute size
    int preLongs = v3mem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F;
    int entries = (preLongs == 1) ? 0 : v3mem.getInt(RETAINED_ENTRIES_INT);
    int v2bytes = (preLongs + entries) << 3;
    //create new mem and do complete copy
    WritableMemory v2mem = WritableMemory.wrap(new byte[v2bytes]);
    v3mem.copyTo(0, v2mem, 0, v2bytes);
    //set serVer2
    v2mem.putByte(SER_VER_BYTE, (byte) 2);
    //adjust the flags
    byte v2flags = (byte) (2 | ((preLongs == 1) ? 4 : 0));
    v2mem.putByte(FLAGS_BYTE, v2flags);
    return v2mem;
}