Examples with Memory org.apache.datasketches.memory.Memory used on opensource projects

Example 61 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class HeapUnionTest method checkSerVer2Handling.

@Test
public void checkSerVer2Handling() {
    // 4096
    final int lgK = 12;
    final int k = 1 << lgK;
    final int u1 = 2 * k;
    // smaller exact sketch forces early stop
    final int u2 = 1024;
    final int totU = u1 + u2;
    final UpdateSketch usk1 = UpdateSketch.builder().setNominalEntries(k).build();
    final UpdateSketch usk2 = UpdateSketch.builder().setNominalEntries(k).build();
    for (int i = 0; i < u1; i++) {
        // 2*k
        usk1.update(i);
    }
    for (int i = u1; i < totU; i++) {
        // 2*k + 1024 no overlap
        usk2.update(i);
    }
    final Memory v2mem1 = convertSerVer3toSerVer2(usk1.compact(true, null), Util.DEFAULT_UPDATE_SEED);
    final Memory v2mem2 = convertSerVer3toSerVer2(usk2.compact(true, null), Util.DEFAULT_UPDATE_SEED);
    final Union union = SetOperation.builder().setNominalEntries(k).buildUnion();
    union.union(v2mem1);
    union.union(v2mem2);
    final CompactSketch cOut = union.getResult(true, null);
    assertEquals(cOut.getEstimate(), totU, .05 * k);
}

Example 62 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class ReadOnlyMemoryTest method wrapCompactOrderedSketch.

@Test
public void wrapCompactOrderedSketch() {
    UpdateSketch updateSketch = UpdateSketch.builder().build();
    updateSketch.update(1);
    Memory mem = Memory.wrap(ByteBuffer.wrap(updateSketch.compact().toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder()));
    Sketch sketch = Sketch.wrap(mem);
    assertEquals(sketch.getEstimate(), 1.0);
}

Example 63 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class AdoubleTest method serDeTest.

@SuppressWarnings("deprecation")
@Test
public void serDeTest() {
    final int lgK = 12;
    final int K = 1 << lgK;
    final DoubleSketch a1Sk = new DoubleSketch(lgK, Mode.AlwaysOne);
    final int m = 2 * K;
    for (int key = 0; key < m; key++) {
        a1Sk.update(key, 1.0);
    }
    final double est1 = a1Sk.getEstimate();
    final Memory mem = Memory.wrap(a1Sk.toByteArray());
    final DoubleSketch a1Sk2 = new DoubleSketch(mem, Mode.AlwaysOne);
    final double est2 = a1Sk2.getEstimate();
    assertEquals(est1, est2);
}

Example 64 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class ToByteArrayImpl method toCouponByteArray.

// To byte array for coupons
static final byte[] toCouponByteArray(final AbstractCoupons impl, final boolean dstCompact) {
    final int srcCouponCount = impl.getCouponCount();
    final int srcLgCouponArrInts = impl.getLgCouponArrInts();
    final int srcCouponArrInts = 1 << srcLgCouponArrInts;
    final byte[] byteArrOut;
    final boolean list = impl.getCurMode() == CurMode.LIST;
    // prepare switch
    final int sw = (impl.isMemory() ? 0 : 4) | (impl.isCompact() ? 0 : 2) | (dstCompact ? 0 : 1);
    switch(sw) {
        case 0:
            {
                // Src Memory, Src Compact, Dst Compact
                final Memory srcMem = impl.getMemory();
                final int bytesOut = impl.getMemDataStart() + (srcCouponCount << 2);
                byteArrOut = new byte[bytesOut];
                srcMem.getByteArray(0, byteArrOut, 0, bytesOut);
                break;
            }
        case 1:
            {
                // Src Memory, Src Compact, Dst Updatable
                final int dataStart = impl.getMemDataStart();
                final int bytesOut = dataStart + (srcCouponArrInts << 2);
                byteArrOut = new byte[bytesOut];
                final WritableMemory memOut = WritableMemory.writableWrap(byteArrOut);
                copyCommonListAndSet(impl, memOut);
                insertCompactFlag(memOut, dstCompact);
                final int[] tgtCouponIntArr = new int[srcCouponArrInts];
                final PairIterator itr = impl.iterator();
                while (itr.nextValid()) {
                    final int pair = itr.getPair();
                    final int idx = find(tgtCouponIntArr, srcLgCouponArrInts, pair);
                    if (idx < 0) {
                        // found EMPTY
                        // insert
                        tgtCouponIntArr[~idx] = pair;
                        continue;
                    }
                    throw new SketchesStateException("Error: found duplicate.");
                }
                memOut.putIntArray(dataStart, tgtCouponIntArr, 0, srcCouponArrInts);
                if (list) {
                    insertListCount(memOut, srcCouponCount);
                } else {
                    insertHashSetCount(memOut, srcCouponCount);
                }
                break;
            }
        case 2:
            {
                // Src Memory, Src Updatable, Dst Compact
                final int dataStart = impl.getMemDataStart();
                final int bytesOut = dataStart + (srcCouponCount << 2);
                byteArrOut = new byte[bytesOut];
                final WritableMemory memOut = WritableMemory.writableWrap(byteArrOut);
                copyCommonListAndSet(impl, memOut);
                insertCompactFlag(memOut, dstCompact);
                final PairIterator itr = impl.iterator();
                int cnt = 0;
                while (itr.nextValid()) {
                    insertInt(memOut, dataStart + (cnt++ << 2), itr.getPair());
                }
                if (list) {
                    insertListCount(memOut, srcCouponCount);
                } else {
                    insertHashSetCount(memOut, srcCouponCount);
                }
                break;
            }
        case 3:
            {
                // Src Memory, Src Updatable, Dst Updatable
                final Memory srcMem = impl.getMemory();
                final int bytesOut = impl.getMemDataStart() + (srcCouponArrInts << 2);
                byteArrOut = new byte[bytesOut];
                srcMem.getByteArray(0, byteArrOut, 0, bytesOut);
                break;
            }
        case 6:
            {
                // Src Heap, Src Updatable, Dst Compact
                final int dataStart = impl.getMemDataStart();
                final int bytesOut = dataStart + (srcCouponCount << 2);
                byteArrOut = new byte[bytesOut];
                final WritableMemory memOut = WritableMemory.writableWrap(byteArrOut);
                copyCommonListAndSet(impl, memOut);
                insertCompactFlag(memOut, dstCompact);
                final PairIterator itr = impl.iterator();
                int cnt = 0;
                while (itr.nextValid()) {
                    insertInt(memOut, dataStart + (cnt++ << 2), itr.getPair());
                }
                if (list) {
                    insertListCount(memOut, srcCouponCount);
                } else {
                    insertHashSetCount(memOut, srcCouponCount);
                }
                break;
            }
        case 7:
            {
                // Src Heap, Src Updatable, Dst Updatable
                final int dataStart = impl.getMemDataStart();
                final int bytesOut = dataStart + (srcCouponArrInts << 2);
                byteArrOut = new byte[bytesOut];
                final WritableMemory memOut = WritableMemory.writableWrap(byteArrOut);
                copyCommonListAndSet(impl, memOut);
                memOut.putIntArray(dataStart, impl.getCouponIntArr(), 0, srcCouponArrInts);
                if (list) {
                    insertListCount(memOut, srcCouponCount);
                } else {
                    insertHashSetCount(memOut, srcCouponCount);
                }
                break;
            }
        default:
            throw new SketchesStateException("Corruption, should not happen: " + sw);
    }
    return byteArrOut;
}

Example 65 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class MurmurHash3 method hash.

// --Hash of Memory-------------------------------------------------------
/**
 * Hash the given Memory.
 *
 * <p>Note: if you want to hash only a portion of Memory, convert it to the
 * appropriate Region first with ByteOrder = Little Endian. If it is not
 * Little Endian a new region view will be created as Little Endian.
 * This does not change the underlying data.
 *
 * @param mem The input Memory. It must be non-null and non-empty.
 * @param seed A long valued seed.
 * @return a 128-bit hash of the input as a long array of size 2.
 */
public static long[] hash(final Memory mem, final long seed) {
    Objects.requireNonNull(mem);
    final long lengthBytes = mem.getCapacity();
    checkPositive(lengthBytes);
    final Memory memLE = mem.getTypeByteOrder() == ByteOrder.LITTLE_ENDIAN ? mem : mem.region(0, lengthBytes, ByteOrder.LITTLE_ENDIAN);
    final HashState hashState = new HashState(seed, seed);
    // Number of full 128-bit blocks of 16 bytes.
    // Possible exclusion of a remainder of up to 15 bytes.
    // bytes / 16
    final long nblocks = lengthBytes >>> 4;
    // Process the 128-bit blocks (the body) into the hash
    for (long i = 0; i < nblocks; i++) {
        // 16 bytes per block
        // 0, 16, 32, ...
        final long k1 = memLE.getLong(i << 4);
        // 8, 24, 40, ...
        final long k2 = memLE.getLong((i << 4) + 8);
        hashState.blockMix128(k1, k2);
    }
    // Get the tail index wrt hashed portion, remainder length
    // 16 bytes per block
    final long tail = nblocks << 4;
    // remainder bytes: 0,1,...,15
    final int rem = (int) (lengthBytes - tail);
    // Get the tail
    final long k1;
    final long k2;
    if (rem > 8) {
        // k1 -> whole; k2 -> partial
        k1 = memLE.getLong(tail);
        k2 = getLong(memLE, tail + 8, rem - 8);
    } else {
        // k1 -> whole, partial or 0; k2 == 0
        k1 = rem == 0 ? 0 : getLong(memLE, tail, rem);
        k2 = 0;
    }
    // Mix the tail into the hash and return
    return hashState.finalMix128(k1, k2, lengthBytes);
}