Examples with Family com.yahoo.sketches.Family used on opensource projects

Example 11 with Family

use of com.yahoo.sketches.Family in project sketches-core by DataSketches.

the class UpdateSketchTest method checkBuilder.

@Test
public void checkBuilder() {
    UpdateSketchBuilder bldr = UpdateSketch.builder();
    long seed = 12345L;
    bldr.setSeed(seed);
    assertEquals(seed, bldr.getSeed());
    float p = (float) 0.5;
    bldr.setP(p);
    assertEquals(p, bldr.getP());
    ResizeFactor rf = ResizeFactor.X4;
    bldr.setResizeFactor(rf);
    assertEquals(rf, bldr.getResizeFactor());
    Family fam = Family.ALPHA;
    bldr.setFamily(fam);
    assertEquals(fam, bldr.getFamily());
    int lgK = 10;
    int k = 1 << lgK;
    bldr.setNominalEntries(k);
    assertEquals(lgK, bldr.getLgNominalEntries());
    println(bldr.toString());
}

Example 12 with Family

use of com.yahoo.sketches.Family in project sketches-core by DataSketches.

the class SketchTest method checkBuilder.

@Test
public void checkBuilder() {
    int k = 2048;
    int lgK = Integer.numberOfTrailingZeros(k);
    long seed = 1021;
    float p = (float) 0.5;
    ResizeFactor rf = X4;
    Family fam = Family.ALPHA;
    UpdateSketch sk1 = UpdateSketch.builder().setSeed(seed).setP(p).setResizeFactor(rf).setFamily(fam).setNominalEntries(k).build();
    String nameS1 = sk1.getClass().getSimpleName();
    assertEquals(nameS1, "HeapAlphaSketch");
    assertEquals(sk1.getLgNomLongs(), lgK);
    assertEquals(sk1.getSeed(), seed);
    assertEquals(sk1.getP(), p);
    //check reset of defaults
    sk1 = UpdateSketch.builder().build();
    nameS1 = sk1.getClass().getSimpleName();
    assertEquals(nameS1, "HeapQuickSelectSketch");
    assertEquals(sk1.getLgNomLongs(), Integer.numberOfTrailingZeros(DEFAULT_NOMINAL_ENTRIES));
    assertEquals(sk1.getSeed(), DEFAULT_UPDATE_SEED);
    assertEquals(sk1.getP(), (float) 1.0);
    assertEquals(sk1.getResizeFactor(), ResizeFactor.X8);
}

Example 13 with Family

use of com.yahoo.sketches.Family in project sketches-core by DataSketches.

the class PreambleUtil method preambleToString.

/**
   * Returns a human readable string summary of the preamble state of the given Memory.
   * Note: other than making sure that the given Memory size is large
   * enough for just the preamble, this does not do much value checking of the contents of the
   * preamble as this is primarily a tool for debugging the preamble visually.
   *
   * @param srcMem the given Memory.
   * @return the summary preamble string.
   */
public static String preambleToString(final Memory srcMem) {
    //make sure we can get the assumed preamble
    final long pre0 = checkPreambleSize(srcMem);
    //byte 0
    final int preLongs = extractPreLongs(pre0);
    //byte 1
    final int serVer = extractSerVer(pre0);
    //byte 2
    final Family family = Family.idToFamily(extractFamilyID(pre0));
    //byte 3
    final int lgMaxMapSize = extractLgMaxMapSize(pre0);
    //byte 4
    final int lgCurMapSize = extractLgCurMapSize(pre0);
    //byte 5
    final int flags = extractFlags(pre0);
    //byte 6
    final int type = extractSerDeId(pre0);
    final String flagsStr = zeroPad(Integer.toBinaryString(flags), 8) + ", " + (flags);
    final boolean empty = (flags & EMPTY_FLAG_MASK) > 0;
    final int maxMapSize = 1 << lgMaxMapSize;
    final int curMapSize = 1 << lgCurMapSize;
    final int maxPreLongs = Family.FREQUENCY.getMaxPreLongs();
    //Assumed if preLongs == 1
    int activeItems = 0;
    long streamLength = 0;
    long offset = 0;
    if (preLongs == maxPreLongs) {
        //get full preamble
        final long[] preArr = new long[preLongs];
        srcMem.getLongArray(0, preArr, 0, preLongs);
        activeItems = extractActiveItems(preArr[1]);
        streamLength = preArr[2];
        offset = preArr[3];
    }
    final StringBuilder sb = new StringBuilder();
    sb.append(LS).append("### FREQUENCY SKETCH PREAMBLE SUMMARY:").append(LS).append("Byte  0: Preamble Longs       : ").append(preLongs).append(LS).append("Byte  1: Serialization Version: ").append(serVer).append(LS).append("Byte  2: Family               : ").append(family.toString()).append(LS).append("Byte  3: MaxMapSize           : ").append(maxMapSize).append(LS).append("Byte  4: CurMapSize           : ").append(curMapSize).append(LS).append("Byte  5: Flags Field          : ").append(flagsStr).append(LS).append("  EMPTY                       : ").append(empty).append(LS).append("Byte  6: Freq Sketch Type     : ").append(type).append(LS);
    if (preLongs == 1) {
        sb.append(" --ABSENT, ASSUMED:").append(LS);
    } else {
        //preLongs == maxPreLongs
        sb.append("Bytes 8-11 : ActiveItems    : ").append(activeItems).append(LS);
        sb.append("Bytes 16-23: StreamLength   : ").append(streamLength).append(LS).append("Bytes 24-31: Offset         : ").append(offset).append(LS);
    }
    sb.append("Preamble Bytes                : ").append(preLongs * 8).append(LS);
    sb.append("TOTAL Sketch Bytes            : ").append((preLongs + activeItems * 2) << 3).append(LS).append("### END FREQUENCY SKETCH PREAMBLE SUMMARY").append(LS);
    return sb.toString();
}

Example 14 with Family

use of com.yahoo.sketches.Family in project sketches-core by DataSketches.

the class Sketch method wrap.

/**
   * Wrap takes the sketch image in Memory and refers to it directly. There is no data copying onto
   * the java heap.  Only "Direct" Serialization Version 3 (i.e, OpenSource) sketches that have
   * been explicitly stored as direct objects can be wrapped.
   * An attempt to "wrap" earlier version sketches will result in a "heapified", normal
   * Java Heap version of the sketch where all data will be copied to the heap.
   * @param srcMem an image of a Sketch where the image seed hash matches the given seed hash.
   * <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
   * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See Update Hash Seed</a>.
   * Compact sketches store a 16-bit hash of the seed, but not the seed itself.
   * @return a UpdateSketch backed by the given Memory
   */
public static Sketch wrap(final Memory srcMem, final long seed) {
    final long pre0 = srcMem.getLong(0);
    final int preLongs = srcMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F;
    final int serVer = srcMem.getByte(SER_VER_BYTE) & 0XFF;
    final int familyID = srcMem.getByte(FAMILY_BYTE) & 0XFF;
    final Family family = Family.idToFamily(familyID);
    switch(family) {
        case QUICKSELECT:
            {
                //Hash Table structure
                if ((serVer == 3) && (preLongs == 3)) {
                    return DirectQuickSelectSketchR.readOnlyWrap(srcMem, seed);
                } else {
                    throw new SketchesArgumentException("Corrupted: " + family + " family image: must have SerVer = 3 and preLongs = 3");
                }
            }
        case COMPACT:
            {
                //serVer 1, 2, or 3, preLongs = 1, 2, or 3
                if (serVer == 1) {
                    return ForwardCompatibility.heapify1to3(srcMem, seed);
                } else if (serVer == 2) {
                    return ForwardCompatibility.heapify2to3(srcMem, seed);
                }
                final int flags = srcMem.getByte(FLAGS_BYTE);
                //used for corruption check
                final boolean compact = (flags & COMPACT_FLAG_MASK) > 0;
                final boolean ordered = (flags & ORDERED_FLAG_MASK) > 0;
                if (compact) {
                    return ordered ? DirectCompactOrderedSketch.wrapInstance(srcMem, pre0, seed) : DirectCompactSketch.wrapInstance(srcMem, pre0, seed);
                }
                throw new SketchesArgumentException("Corrupted: " + family + " family image must have compact flag set");
            }
        default:
            throw new SketchesArgumentException("Sketch cannot wrap family: " + family + " as a Sketch");
    }
}

Example 15 with Family

use of com.yahoo.sketches.Family in project sketches-core by DataSketches.

the class Sketch method constructHeapSketch.

/**
   * Instantiates a Heap Sketch from Memory.
   * @param famID the Family ID
   * @param ordered true if the sketch is of the Compact family and ordered
   * @param srcMem <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
   * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See Update Hash Seed</a>.
   * The seed required to instantiate a non-compact sketch.
   * @return a Sketch
   */
private static final Sketch constructHeapSketch(final byte famID, final boolean ordered, final Memory srcMem, final long seed) {
    final boolean compact = (srcMem.getByte(FLAGS_BYTE) & COMPACT_FLAG_MASK) != 0;
    final Family family = idToFamily(famID);
    switch(family) {
        case ALPHA:
            {
                if (compact) {
                    throw new SketchesArgumentException("Possibly Corrupted " + family + " image: cannot be compact");
                }
                return HeapAlphaSketch.heapifyInstance(srcMem, seed);
            }
        case QUICKSELECT:
            {
                return HeapQuickSelectSketch.heapifyInstance(srcMem, seed);
            }
        case COMPACT:
            {
                if (!compact) {
                    throw new SketchesArgumentException("Possibly Corrupted " + family + " image: must be compact");
                }
                return ordered ? HeapCompactOrderedSketch.heapifyInstance(srcMem, seed) : HeapCompactSketch.heapifyInstance(srcMem, seed);
            }
        default:
            {
                throw new SketchesArgumentException("Sketch cannot heapify family: " + family + " as a Sketch");
            }
    }
}