Examples with Memory com.yahoo.memory.Memory used on opensource projects

Example 71 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class UnionImpl method update.

@Override
public void update(final Sketch sketchIn) {
    if (sketchIn == null) {
        //null is interpreted as (Theta = 1.0, count = 0, empty = T).  Nothing changes
        return;
    }
    Util.checkSeedHashes(seedHash_, sketchIn.getSeedHash());
    final long thetaLongIn = sketchIn.getThetaLong();
    //Theta rule with incoming
    unionThetaLong_ = min(unionThetaLong_, thetaLongIn);
    final int curCountIn = sketchIn.getRetainedEntries(true);
    if (sketchIn.isOrdered()) {
        if (sketchIn.isDirect()) {
            //ordered, direct thus compact
            final Memory skMem = ((CompactSketch) sketchIn).getMemory();
            final int preambleLongs = skMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F;
            for (int i = 0; i < curCountIn; i++) {
                final int offsetBytes = (preambleLongs + i) << 3;
                final long hashIn = skMem.getLong(offsetBytes);
                // "early stop"
                if (hashIn >= unionThetaLong_) {
                    break;
                }
                //backdoor update, hash function is bypassed
                gadget_.hashUpdate(hashIn);
            }
        } else {
            //sketchIn is on the Java Heap, ordered, thus compact
            //not a copy!
            final long[] cacheIn = sketchIn.getCache();
            for (int i = 0; i < curCountIn; i++) {
                final long hashIn = cacheIn[i];
                // "early stop"
                if (hashIn >= unionThetaLong_) {
                    break;
                }
                //backdoor update, hash function is bypassed
                gadget_.hashUpdate(hashIn);
            }
        }
    } else //End ordered, compact
    {
        //either not-ordered compact or Hash Table form. A HT may have dirty values.
        //if off-heap this will be a copy
        final long[] cacheIn = sketchIn.getCache();
        final int arrLongs = cacheIn.length;
        for (int i = 0, c = 0; (i < arrLongs) && (c < curCountIn); i++) {
            final long hashIn = cacheIn[i];
            //rejects dirty values
            if ((hashIn <= 0L) || (hashIn >= unionThetaLong_)) {
                continue;
            }
            //backdoor update, hash function is bypassed
            gadget_.hashUpdate(hashIn);
            //insures against invalid state inside the incoming sketch
            c++;
        }
    }
    //Theta rule with gadget
    unionThetaLong_ = min(unionThetaLong_, gadget_.getThetaLong());
    if (gadget_.isDirect()) {
        gadget_.getMemory().putLong(UNION_THETA_LONG, unionThetaLong_);
    }
}

Example 72 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class DirectCompactDoublesSketchTest method wrapFromCompactSketch.

@Test
public void wrapFromCompactSketch() {
    final int k = 8;
    final int n = 177;
    // assuming ordered inserts
    final DirectCompactDoublesSketch qs = buildAndLoadDCQS(k, n);
    final byte[] qsBytes = qs.toByteArray();
    final Memory qsMem = Memory.wrap(qsBytes);
    final DirectCompactDoublesSketch compactQs = DirectCompactDoublesSketch.wrapInstance(qsMem);
    DoublesSketchTest.testSketchEquality(qs, compactQs);
    assertEquals(qsBytes.length, compactQs.getStorageBytes());
    final double[] combinedBuffer = compactQs.getCombinedBuffer();
    assertEquals(combinedBuffer.length, compactQs.getCombinedBufferItemCapacity());
}

Example 73 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class DirectCompactDoublesSketchTest method wrapFromUpdateSketch.

@Test(expectedExceptions = SketchesArgumentException.class)
public void wrapFromUpdateSketch() {
    final int k = 4;
    final int n = 27;
    final UpdateDoublesSketch qs = HeapUpdateDoublesSketchTest.buildAndLoadQS(k, n);
    final byte[] qsBytes = qs.toByteArray();
    final Memory qsMem = Memory.wrap(qsBytes);
    DirectCompactDoublesSketch.wrapInstance(qsMem);
    fail();
}

Example 74 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class DirectCompactDoublesSketchTest method wrapEmptyCompactSketch.

@Test
public void wrapEmptyCompactSketch() {
    final CompactDoublesSketch s1 = DoublesSketch.builder().build().compact();
    final Memory mem = Memory.wrap(ByteBuffer.wrap(s1.toByteArray()));
    final DoublesSketch s2 = DoublesSketch.wrap(mem);
    assertTrue(s2.isEmpty());
    assertEquals(s2.getN(), 0);
    assertEquals(s2.getMinValue(), Double.POSITIVE_INFINITY);
    assertEquals(s2.getMaxValue(), Double.NEGATIVE_INFINITY);
}

Example 75 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ToFromByteArrayTest method toFrom.

public void toFrom(int lgK, TgtHllType tgtHllType, int n) {
    HllSketch src = new HllSketch(lgK, tgtHllType);
    for (int i = 0; i < n; i++) {
        src.update(i);
    }
    println("n: " + n + ", lgK: " + lgK + ", type: " + tgtHllType);
    //printSketch(src, "SRC");
    byte[] byteArr = src.toByteArray();
    Memory mem = Memory.wrap(byteArr);
    HllSketch dst = HllSketch.heapify(mem);
    //printSketch(dst, "DST");
    assertEquals(dst.getEstimate(), src.getEstimate(), 0.0);
}