Examples with Memory org.apache.datasketches.memory.Memory used on opensource projects

Example 91 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class CompactSketchTest method checkHeapifyWrap.

// test combinations of compact ordered/not ordered and heap/direct
public void checkHeapifyWrap(int k, int u, boolean ordered) {
    UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build();
    for (int i = 0; i < u; i++) {
        // populate update sketch
        usk.update(i);
    }
    /**
     **ON HEAP MEMORY -- HEAPIFY***
     */
    CompactSketch refSk = usk.compact(ordered, null);
    byte[] barr = refSk.toByteArray();
    Memory srcMem = Memory.wrap(barr);
    CompactSketch testSk = (CompactSketch) Sketch.heapify(srcMem);
    checkByRange(refSk, testSk, u, ordered);
    /**
     *Via byte[]*
     */
    byte[] byteArray = refSk.toByteArray();
    Memory heapROMem = Memory.wrap(byteArray);
    testSk = (CompactSketch) Sketch.heapify(heapROMem);
    checkByRange(refSk, testSk, u, ordered);
    /**
     **OFF HEAP MEMORY -- WRAP***
     */
    // Prepare Memory for direct
    // for Compact
    int bytes = usk.getCompactBytes();
    try (WritableHandle wdh = WritableMemory.allocateDirect(bytes)) {
        WritableMemory directMem = wdh.getWritable();
        /**
         *Via CompactSketch.compact*
         */
        refSk = usk.compact(ordered, directMem);
        testSk = (CompactSketch) Sketch.wrap(directMem);
        checkByRange(refSk, testSk, u, ordered);
        /**
         *Via CompactSketch.compact*
         */
        testSk = (CompactSketch) Sketch.wrap(directMem);
        checkByRange(refSk, testSk, u, ordered);
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
}

Example 92 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class ConcurrentHeapQuickSelectSketchTest method checkMinReqBytes.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkMinReqBytes() {
    int lgK = 4;
    int k = 1 << lgK;
    SharedLocal sl = new SharedLocal(lgK);
    for (int i = 0; i < (4 * k); i++) {
        sl.local.update(i);
    }
    waitForBgPropagationToComplete(sl.shared);
    byte[] byteArray = sl.shared.toByteArray();
    // corrupt no. bytes
    byte[] badBytes = Arrays.copyOfRange(byteArray, 0, 24);
    Memory mem = Memory.wrap(badBytes);
    Sketch.heapify(mem);
}

Example 93 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class ConcurrentHeapQuickSelectSketchTest method checkHeapifyByteArrayEstimating.

@Test
public void checkHeapifyByteArrayEstimating() {
    int lgK = 12;
    int k = 1 << lgK;
    int u = 2 * k;
    SharedLocal sl = new SharedLocal(lgK);
    UpdateSketch local = sl.local;
    UpdateSketch shared = sl.shared;
    for (int i = 0; i < u; i++) {
        local.update(i);
    }
    waitForBgPropagationToComplete(shared);
    double localEst = local.getEstimate();
    double localLB = local.getLowerBound(2);
    double localUB = local.getUpperBound(2);
    assertEquals(local.isEstimationMode(), true);
    byte[] serArr = shared.toByteArray();
    Memory srcMem = Memory.wrap(serArr);
    UpdateSketch recoveredShared = UpdateSketch.heapify(srcMem, sl.seed);
    final int bytes = Sketch.getMaxUpdateSketchBytes(k);
    final WritableMemory wmem = WritableMemory.allocate(bytes);
    shared = sl.bldr.buildSharedFromSketch(recoveredShared, wmem);
    UpdateSketch local2 = sl.bldr.buildLocal(shared);
    assertEquals(local2.getEstimate(), localEst);
    assertEquals(local2.getLowerBound(2), localLB);
    assertEquals(local2.getUpperBound(2), localUB);
    assertEquals(local2.isEmpty(), false);
    assertEquals(local2.isEstimationMode(), true);
    assertEquals(local2.getResizeFactor(), local.getResizeFactor());
}

Example 94 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class DirectIntersectionTest method checkOverlappedDirect.

// Check Alex's bug intersecting 2 direct full sketches with only overlap of 2
// 
@Test
public void checkOverlappedDirect() {
    final int k = 1 << 4;
    // plenty of room
    final int memBytes = 2 * k * 16 + PREBYTES;
    final UpdateSketch sk1 = Sketches.updateSketchBuilder().setNominalEntries(k).build();
    final UpdateSketch sk2 = Sketches.updateSketchBuilder().setNominalEntries(k).build();
    for (int i = 0; i < k; i++) {
        sk1.update(i);
        // overlap by 2
        sk2.update(k - 2 + i);
    }
    final WritableMemory memIn1 = WritableMemory.writableWrap(new byte[memBytes]);
    final WritableMemory memIn2 = WritableMemory.writableWrap(new byte[memBytes]);
    final WritableMemory memInter = WritableMemory.writableWrap(new byte[memBytes]);
    final WritableMemory memComp = WritableMemory.writableWrap(new byte[memBytes]);
    final CompactSketch csk1 = sk1.compact(true, memIn1);
    final CompactSketch csk2 = sk2.compact(true, memIn2);
    final Intersection inter = Sketches.setOperationBuilder().buildIntersection(memInter);
    inter.intersect(csk1);
    inter.intersect(csk2);
    final CompactSketch cskOut = inter.getResult(true, memComp);
    assertEquals(cskOut.getEstimate(), 2.0, 0.0);
    final Intersection interRO = (Intersection) SetOperation.wrap((Memory) memInter);
    try {
        interRO.intersect(sk1, sk2);
        fail();
    } catch (final SketchesReadOnlyException e) {
    }
    try {
        interRO.reset();
        fail();
    } catch (final SketchesReadOnlyException e) {
    }
}

Example 95 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class UnionImpl method union.

@Override
public void union(final Sketch sketchIn) {
    if (sketchIn == null || sketchIn.isEmpty()) {
        // null and empty is interpreted as (Theta = 1.0, count = 0, empty = T).  Nothing changes
        return;
    }
    // sketchIn is valid and not empty
    Util.checkSeedHashes(expectedSeedHash_, sketchIn.getSeedHash());
    if (sketchIn instanceof SingleItemSketch) {
        gadget_.hashUpdate(sketchIn.getCache()[0]);
        return;
    }
    Sketch.checkSketchAndMemoryFlags(sketchIn);
    // Theta rule
    unionThetaLong_ = min(min(unionThetaLong_, sketchIn.getThetaLong()), gadget_.getThetaLong());
    unionEmpty_ = false;
    final int curCountIn = sketchIn.getRetainedEntries(true);
    if (curCountIn > 0) {
        if (sketchIn.isOrdered()) {
            // Ordered, thus compact
            if (sketchIn.hasMemory()) {
                final Memory skMem = ((CompactSketch) sketchIn).getMemory();
                final int preambleLongs = skMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F;
                for (int i = 0; i < curCountIn; i++) {
                    final int offsetBytes = preambleLongs + i << 3;
                    final long hashIn = skMem.getLong(offsetBytes);
                    // "early stop"
                    if (hashIn >= unionThetaLong_) {
                        break;
                    }
                    // backdoor update, hash function is bypassed
                    gadget_.hashUpdate(hashIn);
                }
            } else {
                // sketchIn is on the Java Heap or has array
                // not a copy!
                final long[] cacheIn = sketchIn.getCache();
                for (int i = 0; i < curCountIn; i++) {
                    final long hashIn = cacheIn[i];
                    // "early stop"
                    if (hashIn >= unionThetaLong_) {
                        break;
                    }
                    // backdoor update, hash function is bypassed
                    gadget_.hashUpdate(hashIn);
                }
            }
        } else // End ordered, compact
        {
            // either not-ordered compact or Hash Table form. A HT may have dirty values.
            // if off-heap this will be a copy
            final long[] cacheIn = sketchIn.getCache();
            final int arrLongs = cacheIn.length;
            for (int i = 0, c = 0; i < arrLongs && c < curCountIn; i++) {
                final long hashIn = cacheIn[i];
                // rejects dirty values
                if (hashIn <= 0L || hashIn >= unionThetaLong_) {
                    continue;
                }
                // backdoor update, hash function is bypassed
                gadget_.hashUpdate(hashIn);
                // ensures against invalid state inside the incoming sketch
                c++;
            }
        }
    }
    // Theta rule with gadget
    unionThetaLong_ = min(unionThetaLong_, gadget_.getThetaLong());
    if (gadget_.hasMemory()) {
        final WritableMemory wmem = (WritableMemory) gadget_.getMemory();
        PreambleUtil.insertUnionThetaLong(wmem, unionThetaLong_);
        PreambleUtil.clearEmpty(wmem);
    }
}