Examples with WritableMemory com.yahoo.memory.WritableMemory used on opensource projects

Example 96 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class HeapUpdateSketch method toByteArray.

byte[] toByteArray(final int preLongs, final byte familyID) {
    if (isDirty()) {
        rebuild();
    }
    final int preBytes = (preLongs << 3) & 0X3F;
    final int dataBytes = getCurrentDataLongs(false) << 3;
    final byte[] byteArrOut = new byte[preBytes + dataBytes];
    final WritableMemory memOut = WritableMemory.wrap(byteArrOut);
    //may be null
    final Object memObj = memOut.getArray();
    final long memAdd = memOut.getCumulativeOffset(0L);
    //preamble first 8 bytes. Note: only compact can be reduced to 8 bytes.
    final int lgRf = this.getResizeFactor().lg() & 3;
    final byte byte0 = (byte) ((lgRf << 6) | preLongs);
    insertPreLongs(memObj, memAdd, byte0);
    insertSerVer(memObj, memAdd, SER_VER);
    insertFamilyID(memObj, memAdd, familyID);
    insertLgNomLongs(memObj, memAdd, this.getLgNomLongs());
    insertLgArrLongs(memObj, memAdd, this.getLgArrLongs());
    insertSeedHash(memObj, memAdd, this.getSeedHash());
    insertCurCount(memObj, memAdd, this.getRetainedEntries(true));
    insertP(memObj, memAdd, this.getP());
    insertThetaLong(memObj, memAdd, this.getThetaLong());
    //Flags: BigEnd=0, ReadOnly=0, Empty=X, compact=0, ordered=0
    final byte flags = this.isEmpty() ? (byte) EMPTY_FLAG_MASK : 0;
    insertFlags(memObj, memAdd, flags);
    //Data
    final int arrLongs = 1 << this.getLgArrLongs();
    final long[] cache = this.getCache();
    //load byteArrOut
    memOut.putLongArray(preBytes, cache, 0, arrLongs);
    return byteArrOut;
}

Example 97 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class IntersectionImplR method toByteArray.

@Override
public byte[] toByteArray() {
    final int preBytes = CONST_PREAMBLE_LONGS << 3;
    final int dataBytes = (curCount_ > 0) ? 8 << lgArrLongs_ : 0;
    final byte[] byteArrOut = new byte[preBytes + dataBytes];
    if (mem_ != null) {
        mem_.getByteArray(0, byteArrOut, 0, preBytes + dataBytes);
    } else {
        final WritableMemory memOut = WritableMemory.wrap(byteArrOut);
        //preamble
        //RF not used = 0
        memOut.putByte(PREAMBLE_LONGS_BYTE, (byte) CONST_PREAMBLE_LONGS);
        memOut.putByte(SER_VER_BYTE, (byte) SER_VER);
        memOut.putByte(FAMILY_BYTE, (byte) objectToFamily(this).getID());
        //not used
        memOut.putByte(LG_NOM_LONGS_BYTE, (byte) 0);
        memOut.putByte(LG_ARR_LONGS_BYTE, (byte) lgArrLongs_);
        if (empty_) {
            memOut.setBits(FLAGS_BYTE, (byte) EMPTY_FLAG_MASK);
        } else {
            memOut.clearBits(FLAGS_BYTE, (byte) EMPTY_FLAG_MASK);
        }
        memOut.putShort(SEED_HASH_SHORT, seedHash_);
        memOut.putInt(RETAINED_ENTRIES_INT, curCount_);
        memOut.putFloat(P_FLOAT, (float) 1.0);
        memOut.putLong(THETA_LONG, thetaLong_);
        //data
        if (curCount_ > 0) {
            memOut.putLongArray(preBytes, hashTable_, 0, 1 << lgArrLongs_);
        }
    }
    return byteArrOut;
}

Example 98 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectUpdateDoublesSketchTest method checkPutCombinedBuffer.

@Test
public void checkPutCombinedBuffer() {
    final int k = PreambleUtil.DEFAULT_K;
    final int cap = 32 + ((2 * k) << 3);
    WritableMemory mem = WritableMemory.wrap(new byte[cap]);
    final UpdateDoublesSketch qs = DoublesSketch.builder().setK(k).build(mem);
    mem = qs.getMemory();
    assertEquals(mem.getCapacity(), cap);
    assertTrue(qs.isEmpty());
    final int n = 16;
    final double[] data = new double[n];
    for (int i = 0; i < n; ++i) {
        data[i] = i + 1;
    }
    qs.putBaseBufferCount(n);
    qs.putN(n);
    qs.putCombinedBuffer(data);
    final double[] combBuf = qs.getCombinedBuffer();
    assertEquals(combBuf, data);
    // shouldn't have changed min/max values
    assertEquals(qs.getMinValue(), Double.POSITIVE_INFINITY);
    assertEquals(qs.getMaxValue(), Double.NEGATIVE_INFINITY);
}

Example 99 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectUpdateDoublesSketchTest method serializeDeserialize.

@Test
public void serializeDeserialize() {
    int sizeBytes = DoublesSketch.getUpdatableStorageBytes(128, 2000);
    WritableMemory mem = WritableMemory.wrap(new byte[sizeBytes]);
    UpdateDoublesSketch sketch1 = DoublesSketch.builder().build(mem);
    for (int i = 0; i < 1000; i++) {
        sketch1.update(i);
    }
    UpdateDoublesSketch sketch2 = UpdateDoublesSketch.wrap(mem);
    for (int i = 0; i < 1000; i++) {
        sketch2.update(i + 1000);
    }
    assertEquals(sketch2.getMinValue(), 0.0);
    assertEquals(sketch2.getMaxValue(), 1999.0);
    assertEquals(sketch2.getQuantile(0.5), 1000.0, 10.0);
    byte[] arr2 = sketch2.toByteArray(false);
    assertEquals(arr2.length, sketch2.getStorageBytes());
    DoublesSketch sketch3 = DoublesSketch.wrap(WritableMemory.wrap(arr2));
    assertEquals(sketch3.getMinValue(), 0.0);
    assertEquals(sketch3.getMaxValue(), 1999.0);
    assertEquals(sketch3.getQuantile(0.5), 1000.0, 10.0);
}

Example 100 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class DoublesSketchBuilderTest method checkBuilder.

@Test
public void checkBuilder() {
    //default is 128
    int k = 256;
    DoublesSketchBuilder bldr = DoublesSketch.builder();
    bldr.setK(k);
    //confirms new k
    assertEquals(bldr.getK(), k);
    println(bldr.toString());
    int bytes = DoublesSketch.getUpdatableStorageBytes(k, 0);
    byte[] byteArr = new byte[bytes];
    WritableMemory mem = WritableMemory.wrap(byteArr);
    DoublesSketch ds = bldr.build(mem);
    assertTrue(ds.isDirect());
    println(bldr.toString());
    bldr = DoublesSketch.builder();
    assertEquals(bldr.getK(), PreambleUtil.DEFAULT_K);
}