Examples with ResizeFactor com.yahoo.sketches.ResizeFactor used on opensource projects

Example 16 with ResizeFactor

use of com.yahoo.sketches.ResizeFactor in project sketches-core by DataSketches.

the class DirectQuickSelectSketchTest method checkConstructorSrcMemCorruptions.

@Test
public void checkConstructorSrcMemCorruptions() {
    //lgNomLongs = 10
    int k = 1024;
    //exact mode, lgArrLongs = 11
    int u = k;
    int bytes = Sketches.getMaxUpdateSketchBytes(k);
    byte[] arr1 = new byte[bytes];
    WritableMemory mem1 = WritableMemory.wrap(arr1);
    //0
    ResizeFactor rf = ResizeFactor.X1;
    UpdateSketch usk1 = UpdateSketch.builder().setNominalEntries(k).setResizeFactor(rf).build(mem1);
    for (int i = 0; i < u; i++) {
        usk1.update(i);
    }
    //println(PreambleUtil.toString(mem1));
    @SuppressWarnings("unused") UpdateSketch usk2;
    //corrupt Family by setting to Compact
    mem1.putByte(FAMILY_BYTE, (byte) 3);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem1, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //Pass
    }
    //fix Family
    mem1.putByte(FAMILY_BYTE, (byte) 2);
    //corrupt preLongs
    mem1.putByte(PREAMBLE_LONGS_BYTE, (byte) 1);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem1, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //pass
    }
    //fix preLongs
    mem1.putByte(PREAMBLE_LONGS_BYTE, (byte) 3);
    //corrupt serVer
    mem1.putByte(SER_VER_BYTE, (byte) 2);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem1, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //pass
    }
    //fix serVer
    mem1.putByte(SER_VER_BYTE, (byte) 3);
    //corrupt theta and
    mem1.putLong(THETA_LONG, Long.MAX_VALUE >>> 1);
    //corrupt lgArrLongs
    mem1.putByte(LG_ARR_LONGS_BYTE, (byte) 10);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem1, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //pass
    }
    //fix theta and
    mem1.putLong(THETA_LONG, Long.MAX_VALUE);
    //fix lgArrLongs
    mem1.putByte(LG_ARR_LONGS_BYTE, (byte) 11);
    byte badFlags = (byte) (BIG_ENDIAN_FLAG_MASK | COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK | ORDERED_FLAG_MASK);
    mem1.putByte(FLAGS_BYTE, badFlags);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem1, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //pass
    }
    //corrupt length
    byte[] arr2 = Arrays.copyOfRange(arr1, 0, bytes - 1);
    WritableMemory mem2 = WritableMemory.wrap(arr2);
    try {
        usk2 = DirectQuickSelectSketch.writableWrap(mem2, DEFAULT_UPDATE_SEED);
        fail("Expected SketchesArgumentException");
    } catch (SketchesArgumentException e) {
    //pass
    }
}

Example 17 with ResizeFactor

use of com.yahoo.sketches.ResizeFactor in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkBadConstructorArgs.

@Test
public void checkBadConstructorArgs() {
    final ArrayList<String> data = new ArrayList<>(128);
    for (int i = 0; i < 128; ++i) {
        data.add(Integer.toString(i));
    }
    final ResizeFactor rf = ResizeFactor.X8;
    // no items
    try {
        ReservoirItemsSketch.<Byte>newInstance(null, 128, rf, 128);
        fail();
    } catch (final SketchesException e) {
        assertTrue(e.getMessage().contains("null reservoir"));
    }
    // size too small
    try {
        ReservoirItemsSketch.newInstance(data, 128, rf, 1);
        fail();
    } catch (final SketchesException e) {
        assertTrue(e.getMessage().contains("size less than 2"));
    }
    // configured reservoir size smaller than items length
    try {
        ReservoirItemsSketch.newInstance(data, 128, rf, 64);
        fail();
    } catch (final SketchesException e) {
        assertTrue(e.getMessage().contains("max size less than array length"));
    }
    // too many items seen vs items length, full sketch
    try {
        ReservoirItemsSketch.newInstance(data, 512, rf, 256);
        fail();
    } catch (final SketchesException e) {
        assertTrue(e.getMessage().contains("too few samples"));
    }
    // too many items seen vs items length, under-full sketch
    try {
        ReservoirItemsSketch.newInstance(data, 256, rf, 256);
        fail();
    } catch (final SketchesException e) {
        assertTrue(e.getMessage().contains("too few samples"));
    }
}

Example 18 with ResizeFactor

use of com.yahoo.sketches.ResizeFactor in project sketches-core by DataSketches.

the class ReservoirLongsSketchTest method checkBadConstructorArgs.

@Test
public void checkBadConstructorArgs() {
    final long[] data = new long[128];
    for (int i = 0; i < 128; ++i) {
        data[i] = i;
    }
    final ResizeFactor rf = ResizeFactor.X8;
    // no items
    try {
        ReservoirLongsSketch.getInstance(null, 128, rf, 128);
        fail();
    } catch (final SketchesException e) {
        assertTrue(e.getMessage().contains("null reservoir"));
    }
    // size too small
    try {
        ReservoirLongsSketch.getInstance(data, 128, rf, 1);
        fail();
    } catch (final SketchesException e) {
        assertTrue(e.getMessage().contains("size less than 2"));
    }
    // configured reservoir size smaller than items length
    try {
        ReservoirLongsSketch.getInstance(data, 128, rf, 64);
        fail();
    } catch (final SketchesException e) {
        assertTrue(e.getMessage().contains("max size less than array length"));
    }
    // too many items seen vs items length, full sketch
    try {
        ReservoirLongsSketch.getInstance(data, 512, rf, 256);
        fail();
    } catch (final SketchesException e) {
        assertTrue(e.getMessage().contains("too few samples"));
    }
    // too many items seen vs items length, under-full sketch
    try {
        ReservoirLongsSketch.getInstance(data, 256, rf, 256);
        fail();
    } catch (final SketchesException e) {
        assertTrue(e.getMessage().contains("too few samples"));
    }
}

Example 19 with ResizeFactor

use of com.yahoo.sketches.ResizeFactor in project sketches-core by DataSketches.

the class PreambleUtil method unionPreambleToString.

private static String unionPreambleToString(final Memory mem, final Family family, final int preLongs) {
    final ResizeFactor rf = ResizeFactor.getRF(extractResizeFactor(mem));
    final int serVer = extractSerVer(mem);
    // Flags
    final int flags = extractFlags(mem);
    final String flagsStr = zeroPad(Integer.toBinaryString(flags), 8) + ", " + (flags);
    //final boolean bigEndian = (flags & BIG_ENDIAN_FLAG_MASK) > 0;
    //final String nativeOrder = ByteOrder.nativeOrder().toString();
    //final boolean readOnly = (flags & READ_ONLY_FLAG_MASK) > 0;
    final boolean isEmpty = (flags & EMPTY_FLAG_MASK) > 0;
    final int k;
    if (serVer == 1) {
        final short encK = extractEncodedReservoirSize(mem);
        k = ReservoirSize.decodeValue(encK);
    } else {
        k = extractK(mem);
    }
    final long dataBytes = mem.getCapacity() - (preLongs << 3);
    return LS + "### END " + family.getFamilyName().toUpperCase() + " PREAMBLE SUMMARY" + LS + "Byte  0: Preamble Longs           : " + preLongs + LS + "Byte  0: ResizeFactor             : " + rf.toString() + LS + "Byte  1: Serialization Version    : " + serVer + LS + "Byte  2: Family                   : " + family.toString() + LS + "Byte  3: Flags Field              : " + flagsStr + LS + //+ "  READ_ONLY                       : " + readOnly + LS
    "  EMPTY                           : " + isEmpty + LS + "Bytes  4-7: Max Sketch Size (maxK): " + k + LS + "TOTAL Sketch Bytes                : " + mem.getCapacity() + LS + "  Preamble Bytes                  : " + (preLongs << 3) + LS + "  Sketch Bytes                    : " + dataBytes + LS + "### END " + family.getFamilyName().toUpperCase() + " PREAMBLE SUMMARY" + LS;
}

Example 20 with ResizeFactor

use of com.yahoo.sketches.ResizeFactor in project sketches-core by DataSketches.

the class ReservoirLongsSketch method heapify.

/**
   * Returns a sketch instance of this class from the given srcMem, which must be a Memory
   * representation of this sketch class.
   *
   * @param srcMem a Memory representation of a sketch of this class. <a href=
   *        "{@docRoot}/resources/dictionary.html#mem">See Memory</a>
   * @return a sketch instance of this class
   */
public static ReservoirLongsSketch heapify(final Memory srcMem) {
    Family.RESERVOIR.checkFamilyID(srcMem.getByte(FAMILY_BYTE));
    final int numPreLongs = extractPreLongs(srcMem);
    final ResizeFactor rf = ResizeFactor.getRF(extractResizeFactor(srcMem));
    final int serVer = extractSerVer(srcMem);
    final boolean isEmpty = (extractFlags(srcMem) & EMPTY_FLAG_MASK) != 0;
    final long itemsSeen = (isEmpty ? 0 : extractN(srcMem));
    int k = extractK(srcMem);
    // Check values
    final boolean preLongsEqMin = (numPreLongs == Family.RESERVOIR.getMinPreLongs());
    final boolean preLongsEqMax = (numPreLongs == Family.RESERVOIR.getMaxPreLongs());
    if (!preLongsEqMin & !preLongsEqMax) {
        throw new SketchesArgumentException("Possible corruption: Non-empty sketch with only " + Family.RESERVOIR.getMinPreLongs() + "preLongs");
    }
    if (serVer != SER_VER) {
        if (serVer == 1) {
            final short encK = extractEncodedReservoirSize(srcMem);
            k = ReservoirSize.decodeValue(encK);
        } else {
            throw new SketchesArgumentException("Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
        }
    }
    if (isEmpty) {
        return new ReservoirLongsSketch(k, rf);
    }
    final int preLongBytes = numPreLongs << 3;
    final int numSketchLongs = (int) Math.min(itemsSeen, k);
    // default to full reservoir
    int allocatedSize = k;
    if (itemsSeen < k) {
        // under-full so determine size to allocate, using ceilingLog2(totalSeen) as minimum
        // casts to int are safe since under-full
        final int ceilingLgK = Util.toLog2(Util.ceilingPowerOf2(k), "heapify");
        final int minLgSize = Util.toLog2(Util.ceilingPowerOf2((int) itemsSeen), "heapify");
        final int initialLgSize = SamplingUtil.startingSubMultiple(ceilingLgK, rf.lg(), Math.max(minLgSize, MIN_LG_ARR_LONGS));
        allocatedSize = SamplingUtil.getAdjustedSize(k, 1 << initialLgSize);
    }
    final long[] data = new long[allocatedSize];
    srcMem.getLongArray(preLongBytes, data, 0, numSketchLongs);
    return new ReservoirLongsSketch(data, itemsSeen, rf, k);
}