Examples with Memory com.yahoo.memory.Memory used on opensource projects

Example 76 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class LongsSketchTest method checkFreqLongsMemSerDe.

@Test
public void checkFreqLongsMemSerDe() {
    int minSize = 1 << LG_MIN_MAP_SIZE;
    LongsSketch sk1 = new LongsSketch(minSize);
    sk1.update(10, 100);
    sk1.update(10, 100);
    sk1.update(15, 3443);
    println(sk1.toString());
    sk1.update(1000001, 1010230);
    println(sk1.toString());
    sk1.update(1000002, 1010230);
    println(sk1.toString());
    byte[] bytearray0 = sk1.toByteArray();
    Memory mem0 = Memory.wrap(bytearray0);
    LongsSketch sk2 = LongsSketch.getInstance(mem0);
    checkEquality(sk1, sk2);
}

Example 77 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class VarOptItemsUnion method heapify.

/**
   * Instantiates a Union from Memory
   *
   * @param <T> The type of item this sketch contains
   * @param srcMem Memory object containing a serialized union
   * @param serDe An instance of ArrayOfItemsSerDe
   * @return A VarOptItemsUnion created from the provided Memory
   */
public static <T> VarOptItemsUnion<T> heapify(final Memory srcMem, final ArrayOfItemsSerDe<T> serDe) {
    Family.VAROPT_UNION.checkFamilyID(srcMem.getByte(FAMILY_BYTE));
    long n = 0;
    double outerTauNum = 0.0;
    long outerTauDenom = 0;
    // If we have read-only memory on heap (aka not-direct) then the backing array exists but is
    // not available to us, so srcMem.array() will fail. In that case, we can use the (slower)
    // Memory interface methods to read values directly.
    final int numPreLongs = extractPreLongs(srcMem);
    final int serVer = extractSerVer(srcMem);
    final boolean isEmpty = (extractFlags(srcMem) & EMPTY_FLAG_MASK) != 0;
    final int maxK = extractMaxK(srcMem);
    if (!isEmpty) {
        n = extractN(srcMem);
        outerTauNum = extractOuterTauNumerator(srcMem);
        outerTauDenom = extractOuterTauDenominator(srcMem);
    }
    if (serVer != SER_VER) {
        throw new SketchesArgumentException("Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
    }
    final boolean preLongsEqMin = (numPreLongs == Family.VAROPT_UNION.getMinPreLongs());
    final boolean preLongsEqMax = (numPreLongs == Family.VAROPT_UNION.getMaxPreLongs());
    if (!preLongsEqMin && !preLongsEqMax) {
        throw new SketchesArgumentException("Possible corruption: Non-empty union with only " + Family.VAROPT_UNION.getMinPreLongs() + "preLongs");
    }
    final VarOptItemsUnion<T> viu = new VarOptItemsUnion<>(maxK);
    if (isEmpty) {
        viu.gadget_ = VarOptItemsSketch.newInstanceAsGadget(maxK);
    } else {
        viu.n_ = n;
        viu.outerTauNumer = outerTauNum;
        viu.outerTauDenom = outerTauDenom;
        final int preLongBytes = numPreLongs << 3;
        final Memory sketchMem = srcMem.region(preLongBytes, srcMem.getCapacity() - preLongBytes);
        viu.gadget_ = VarOptItemsSketch.heapify(sketchMem, serDe);
    }
    return viu;
}

Example 78 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirLongsUnion method heapify.

/**
   * Instantiates a Union from Memory
   *
   * @param srcMem Memory object containing a serialized union
   * @return A ReservoirLongsUnion created from the provided Memory
   */
public static ReservoirLongsUnion heapify(final Memory srcMem) {
    Family.RESERVOIR_UNION.checkFamilyID(srcMem.getByte(FAMILY_BYTE));
    final int numPreLongs = extractPreLongs(srcMem);
    final int serVer = extractSerVer(srcMem);
    final boolean isEmpty = (extractFlags(srcMem) & EMPTY_FLAG_MASK) != 0;
    int maxK = extractMaxK(srcMem);
    final boolean preLongsEqMin = (numPreLongs == Family.RESERVOIR_UNION.getMinPreLongs());
    final boolean preLongsEqMax = (numPreLongs == Family.RESERVOIR_UNION.getMaxPreLongs());
    if (!preLongsEqMin & !preLongsEqMax) {
        throw new SketchesArgumentException("Possible corruption: Non-empty union with only " + Family.RESERVOIR_UNION.getMinPreLongs() + "preLongs");
    }
    if (serVer != SER_VER) {
        if (serVer == 1) {
            final short encMaxK = extractEncodedReservoirSize(srcMem);
            maxK = ReservoirSize.decodeValue(encMaxK);
        } else {
            throw new SketchesArgumentException("Possible Corruption: Ser Ver must be " + SER_VER + ": " + serVer);
        }
    }
    final ReservoirLongsUnion rlu = new ReservoirLongsUnion(maxK);
    if (!isEmpty) {
        final int preLongBytes = numPreLongs << 3;
        final Memory sketchMem = srcMem.region(preLongBytes, srcMem.getCapacity() - preLongBytes);
        rlu.update(sketchMem);
    }
    return rlu;
}

Example 79 with Memory

use of com.yahoo.memory.Memory in project druid by druid-io.

the class SketchBufferAggregator method init.

@Override
public void init(ByteBuffer buf, int position) {
    if (nm == null) {
        nm = new NativeMemory(buf);
    }
    Memory mem = new MemoryRegion(nm, position, maxIntermediateSize);
    unions.put(position, (Union) SetOperation.builder().initMemory(mem).build(size, Family.UNION));
}

Example 80 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class HeapIntersectionTest method checkHeapify.

@Test
public void checkHeapify() {
    int lgK = 9;
    int k = 1 << lgK;
    Intersection inter, inter2;
    UpdateSketch sk1, sk2;
    CompactSketch resultComp1, resultComp2;
    double est, est2;
    sk1 = UpdateSketch.builder().setNominalEntries(k).build();
    //est mode
    for (int i = 0; i < 2 * k; i++) sk1.update(i);
    CompactSketch compSkIn1 = sk1.compact(true, null);
    println("compSkIn1: " + compSkIn1.getEstimate());
    //1st call = valid
    inter = SetOperation.builder().buildIntersection();
    inter.update(compSkIn1);
    //2nd call = valid intersecting
    sk2 = UpdateSketch.builder().setNominalEntries(k).build();
    //est mode
    for (int i = 0; i < 2 * k; i++) sk2.update(i);
    CompactSketch compSkIn2 = sk2.compact(true, null);
    println("compSkIn2: " + compSkIn2.getEstimate());
    inter.update(compSkIn2);
    resultComp1 = inter.getResult(false, null);
    est = resultComp1.getEstimate();
    assertTrue(est > k);
    println("Est: " + est);
    byte[] byteArray = inter.toByteArray();
    Memory mem = Memory.wrap(byteArray);
    inter2 = (Intersection) SetOperation.heapify(mem);
    //inter2 = new Intersection(mem, seed);
    resultComp2 = inter2.getResult(false, null);
    est2 = resultComp2.getEstimate();
    println("Est2: " + est2);
    inter.reset();
    inter2.reset();
}