Examples with WritableMemory org.apache.datasketches.memory.WritableMemory used on opensource projects

Example 6 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project druid by druid-io.

the class HashVectorGrouperTest method testTableStartIsNotMemoryStartIfNotMaxSized.

@Test
public void testTableStartIsNotMemoryStartIfNotMaxSized() {
    final int maxVectorSize = 512;
    final int keySize = 4;
    final int bufferSize = 100 * 1024;
    final WritableMemory keySpace = WritableMemory.allocate(keySize * maxVectorSize);
    final ByteBuffer buffer = ByteBuffer.wrap(new byte[bufferSize]);
    final AggregatorAdapters aggregatorAdapters = Mockito.mock(AggregatorAdapters.class);
    final HashVectorGrouper grouper = new HashVectorGrouper(Suppliers.ofInstance(buffer), keySize, aggregatorAdapters, 8, 0.f, 4);
    grouper.initVectorized(maxVectorSize);
    Assert.assertNotEquals(0, grouper.getTableStart());
}

Example 7 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project druid by druid-io.

the class HashVectorGrouperTest method testGrowTwice.

@Test
public void testGrowTwice() {
    final int maxVectorSize = 512;
    final int keySize = 4;
    final int aggSize = 8;
    final WritableMemory keySpace = WritableMemory.allocate(keySize * maxVectorSize);
    final AggregatorAdapters aggregatorAdapters = Mockito.mock(AggregatorAdapters.class);
    Mockito.when(aggregatorAdapters.spaceNeeded()).thenReturn(aggSize);
    int startingNumBuckets = 4;
    int maxBuckets = 32;
    final int bufferSize = (keySize + aggSize) * maxBuckets;
    final ByteBuffer buffer = ByteBuffer.wrap(new byte[bufferSize]);
    final HashVectorGrouper grouper = new HashVectorGrouper(Suppliers.ofInstance(buffer), keySize, aggregatorAdapters, maxBuckets, 0.f, startingNumBuckets);
    grouper.initVectorized(maxVectorSize);
    int tableStart = grouper.getTableStart();
    // two keys should not cause buffer to grow
    fillKeyspace(keySpace, maxVectorSize, 2);
    AggregateResult result = grouper.aggregateVector(keySpace, 0, maxVectorSize);
    Assert.assertTrue(result.isOk());
    Assert.assertEquals(tableStart, grouper.getTableStart());
    // 3rd key should cause buffer to grow
    // buffer should grow to next size, but is not full
    fillKeyspace(keySpace, maxVectorSize, 3);
    result = grouper.aggregateVector(keySpace, 0, maxVectorSize);
    Assert.assertTrue(result.isOk());
    Assert.assertTrue(grouper.getTableStart() > tableStart);
    // this time should be all the way
    fillKeyspace(keySpace, maxVectorSize, 6);
    result = grouper.aggregateVector(keySpace, 0, maxVectorSize);
    Assert.assertTrue(result.isOk());
    Assert.assertEquals(0, grouper.getTableStart());
}

Example 8 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project druid by druid-io.

the class HashTableUtilsTest method generateRandomButNotReallyRandomMemory.

private static WritableMemory generateRandomButNotReallyRandomMemory(final int length) {
    final WritableMemory randomMemory = WritableMemory.allocate(length);
    // Fill with random bytes, but use the same seed every run for consistency. This test should pass with
    // any seed unless something really pathological happens.
    final Random random = new Random(0);
    final byte[] randomBytes = new byte[length];
    random.nextBytes(randomBytes);
    randomMemory.putByteArray(0, randomBytes, 0, length);
    return randomMemory;
}

Example 9 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project druid by druid-io.

the class BufferArrayGrouper method iterator.

@Override
public CloseableIterator<Entry<Memory>> iterator() {
    final CloseableIterator<Entry<Integer>> iterator = iterator(false);
    final WritableMemory keyMemory = WritableMemory.allocate(Integer.BYTES);
    return new CloseableIterator<Entry<Memory>>() {

        @Override
        public boolean hasNext() {
            return iterator.hasNext();
        }

        @Override
        public Entry<Memory> next() {
            final Entry<Integer> integerEntry = iterator.next();
            keyMemory.putInt(0, integerEntry.getKey());
            return new Entry<>(keyMemory, integerEntry.getValues());
        }

        @Override
        public void close() throws IOException {
            iterator.close();
        }
    };
}

Example 10 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class CompressionCharacterization method doTrialsAtLgKAtN.

private void doTrialsAtLgKAtN(final int lgK, final long n, final int totalTrials) {
    final int k = 1 << lgK;
    final int minNK = (int) ((k < n) ? k : n);
    final double nOverK = (double) n / k;
    final int lgTotTrials = Integer.numberOfTrailingZeros(totalTrials);
    final int lgWaves = Math.max(lgTotTrials - 10, 0);
    final int trialsPerWave = 1 << (lgTotTrials - lgWaves);
    // printf("%d %d %d %d\n", totalTrials, lgTotTrials, 1 << lgWaves, trialsPerWave);
    streamSketches = new CpcSketch[trialsPerWave];
    compressedStates1 = new CompressedState[trialsPerWave];
    memoryArr = new WritableMemory[trialsPerWave];
    compressedStates2 = new CompressedState[trialsPerWave];
    unCompressedSketches = new CpcSketch[trialsPerWave];
    // update: fill, compress, uncompress sketches arrays in waves
    long totalC = 0;
    long totalW = 0;
    long sumCtor_nS = 0;
    long sumUpd_nS = 0;
    long sumCom_nS = 0;
    long sumSer_nS = 0;
    long sumDes_nS = 0;
    long sumUnc_nS = 0;
    long sumEqu_nS = 0;
    long nanoStart, nanoEnd;
    final long start = System.currentTimeMillis();
    // Wave Loop
    for (int w = 0; w < (1 << lgWaves); w++) {
        // Construct array with sketches loop
        nanoStart = System.nanoTime();
        for (int trial = 0; trial < trialsPerWave; trial++) {
            final CpcSketch sketch = new CpcSketch(lgK);
            streamSketches[trial] = sketch;
        }
        nanoEnd = System.nanoTime();
        sumCtor_nS += nanoEnd - nanoStart;
        nanoStart = nanoEnd;
        // Sketch Update loop
        for (int trial = 0; trial < trialsPerWave; trial++) {
            final CpcSketch sketch = streamSketches[trial];
            for (long i = 0; i < n; i++) {
                // increment loop
                sketch.update(vIn += iGoldenU64);
            }
        }
        nanoEnd = System.nanoTime();
        sumUpd_nS += nanoEnd - nanoStart;
        nanoStart = nanoEnd;
        // Compress loop
        for (int trial = 0; trial < trialsPerWave; trial++) {
            final CpcSketch sketch = streamSketches[trial];
            final CompressedState state = CompressedState.compress(sketch);
            compressedStates1[trial] = state;
            totalC += sketch.numCoupons;
            totalW += state.csvLengthInts + state.cwLengthInts;
        }
        nanoEnd = System.nanoTime();
        sumCom_nS += nanoEnd - nanoStart;
        nanoStart = nanoEnd;
        // State to Memory loop
        for (int trial = 0; trial < trialsPerWave; trial++) {
            final CompressedState state = compressedStates1[trial];
            final long cap = state.getRequiredSerializedBytes();
            final WritableMemory wmem = WritableMemory.allocate((int) cap);
            state.exportToMemory(wmem);
            memoryArr[trial] = wmem;
        }
        nanoEnd = System.nanoTime();
        sumSer_nS += nanoEnd - nanoStart;
        nanoStart = nanoEnd;
        // Memory to State loop
        for (int trial = 0; trial < trialsPerWave; trial++) {
            final Memory mem = memoryArr[trial];
            final CompressedState state = importFromMemory(mem);
            compressedStates2[trial] = state;
        }
        nanoEnd = System.nanoTime();
        sumDes_nS += nanoEnd - nanoStart;
        nanoStart = nanoEnd;
        // Uncompress loop
        for (int trial = 0; trial < trialsPerWave; trial++) {
            final CompressedState state = compressedStates2[trial];
            CpcSketch uncSk = null;
            uncSk = CpcSketch.uncompress(state, DEFAULT_UPDATE_SEED);
            unCompressedSketches[trial] = uncSk;
        }
        nanoEnd = System.nanoTime();
        sumUnc_nS += nanoEnd - nanoStart;
        nanoStart = nanoEnd;
        // Equals check
        for (int trial = 0; trial < trialsPerWave; trial++) {
            rtAssert(TestUtil.specialEquals(streamSketches[trial], unCompressedSketches[trial], false, false));
        }
        nanoEnd = System.nanoTime();
        sumEqu_nS += nanoEnd - nanoStart;
        nanoStart = nanoEnd;
    }
    // end wave loop
    final double total_S = (System.currentTimeMillis() - start) / 1E3;
    final double avgC = (1.0 * totalC) / totalTrials;
    final double avgCoK = avgC / k;
    final double avgWords = (1.0 * totalW) / totalTrials;
    final double avgBytes = (4.0 * totalW) / totalTrials;
    final double avgCtor_nS = Math.round((double) sumCtor_nS / totalTrials);
    final double avgUpd_nS = Math.round((double) sumUpd_nS / totalTrials);
    final double avgUpd_nSperN = avgUpd_nS / n;
    final double avgCom_nS = Math.round((double) sumCom_nS / totalTrials);
    final double avgCom_nSper2C = avgCom_nS / (2.0 * avgC);
    final double avgCom_nSperK = avgCom_nS / k;
    final double avgSer_nS = Math.round((double) sumSer_nS / totalTrials);
    final double avgSer_nSperW = avgSer_nS / avgWords;
    final double avgDes_nS = Math.round((double) sumDes_nS / totalTrials);
    final double avgDes_nSperW = avgDes_nS / avgWords;
    final double avgUnc_nS = Math.round((double) sumUnc_nS / totalTrials);
    final double avgUnc_nSper2C = avgUnc_nS / (2.0 * avgC);
    final double avgUnc_nSperK = avgUnc_nS / k;
    final double avgEqu_nS = Math.round((double) sumEqu_nS / totalTrials);
    final double avgEqu_nSperMinNK = avgEqu_nS / minNK;
    final int len = unCompressedSketches.length;
    final Flavor finFlavor = unCompressedSketches[len - 1].getFlavor();
    final String offStr = Integer.toString(unCompressedSketches[len - 1].windowOffset);
    final String flavorOff = finFlavor.toString() + String.format("%2s", offStr);
    printf(dfmt, lgK, totalTrials, n, minNK, avgCoK, flavorOff, nOverK, avgBytes, avgCtor_nS, avgUpd_nS, avgCom_nS, avgSer_nS, avgDes_nS, avgUnc_nS, avgEqu_nS, avgUpd_nSperN, avgCom_nSper2C, avgCom_nSperK, avgSer_nSperW, avgDes_nSperW, avgUnc_nSper2C, avgUnc_nSperK, avgEqu_nSperMinNK, total_S);
}