Search in sources :

Example 6 with BenchmarkOptions

use of com.carrotsearch.junitbenchmarks.BenchmarkOptions in project parquet-mr by apache.

the class BenchmarkDeltaLengthByteArray method benchmarkRandomStringsWithPlainValuesWriter.

@BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 4)
@Test
public void benchmarkRandomStringsWithPlainValuesWriter() throws IOException {
    PlainValuesWriter writer = new PlainValuesWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());
    BinaryPlainValuesReader reader = new BinaryPlainValuesReader();
    Utils.writeData(writer, values);
    ByteBufferInputStream data = writer.getBytes().toInputStream();
    Binary[] bin = Utils.readData(reader, data, values.length);
    System.out.println("size " + data.position());
}
Also used : PlainValuesWriter(org.apache.parquet.column.values.plain.PlainValuesWriter) BinaryPlainValuesReader(org.apache.parquet.column.values.plain.BinaryPlainValuesReader) DirectByteBufferAllocator(org.apache.parquet.bytes.DirectByteBufferAllocator) ByteBufferInputStream(org.apache.parquet.bytes.ByteBufferInputStream) Binary(org.apache.parquet.io.api.Binary) Test(org.junit.Test) BenchmarkOptions(com.carrotsearch.junitbenchmarks.BenchmarkOptions)

Example 7 with BenchmarkOptions

use of com.carrotsearch.junitbenchmarks.BenchmarkOptions in project parquet-mr by apache.

the class BenchmarkReadingRandomIntegers method readingDelta.

@BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 10)
@Test
public void readingDelta() throws IOException {
    for (int j = 0; j < 10; j++) {
        DeltaBinaryPackingValuesReader reader = new DeltaBinaryPackingValuesReader();
        readData(reader, deltaBytes);
    }
}
Also used : DeltaBinaryPackingValuesReader(org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesReader) Test(org.junit.Test) BenchmarkOptions(com.carrotsearch.junitbenchmarks.BenchmarkOptions)

Example 8 with BenchmarkOptions

use of com.carrotsearch.junitbenchmarks.BenchmarkOptions in project parquet-mr by apache.

the class BenchmarkReadingRandomIntegers method readingRLE.

@BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 10)
@Test
public void readingRLE() throws IOException {
    for (int j = 0; j < 10; j++) {
        ValuesReader reader = new RunLengthBitPackingHybridValuesReader(32);
        readData(reader, rleBytes);
    }
}
Also used : DeltaBinaryPackingValuesReader(org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesReader) RunLengthBitPackingHybridValuesReader(org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesReader) ValuesReader(org.apache.parquet.column.values.ValuesReader) RunLengthBitPackingHybridValuesReader(org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesReader) Test(org.junit.Test) BenchmarkOptions(com.carrotsearch.junitbenchmarks.BenchmarkOptions)

Example 9 with BenchmarkOptions

use of com.carrotsearch.junitbenchmarks.BenchmarkOptions in project parquet-mr by apache.

the class RandomWritingBenchmarkTest method writeRLETest.

@BenchmarkOptions(benchmarkRounds = 10, warmupRounds = 2)
@Test
public void writeRLETest() {
    ValuesWriter writer = new RunLengthBitPackingHybridValuesWriter(32, 100, 20000, new DirectByteBufferAllocator());
    runWriteTest(writer);
}
Also used : DirectByteBufferAllocator(org.apache.parquet.bytes.DirectByteBufferAllocator) RunLengthBitPackingHybridValuesWriter(org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter) ValuesWriter(org.apache.parquet.column.values.ValuesWriter) RunLengthBitPackingHybridValuesWriter(org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter) DeltaBinaryPackingValuesWriter(org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriter) Test(org.junit.Test) BenchmarkOptions(com.carrotsearch.junitbenchmarks.BenchmarkOptions)

Example 10 with BenchmarkOptions

use of com.carrotsearch.junitbenchmarks.BenchmarkOptions in project druid by druid-io.

the class BenchmarkIndexibleWrites method testConcurrentReads.

/**
 *   BenchmarkIndexibleWrites.TestConcurrentReads[0]: [measured 100 out of 200 rounds, threads: 1 (sequential)]
 *   round: 0.28 [+- 0.02], round.block: 0.00 [+- 0.00], round.gc: 0.02 [+- 0.00], GC.calls: 396, GC.time: 1.84, time.total: 59.98, time.warmup: 30.51, time.bench: 29.48
 *   BenchmarkIndexibleWrites.TestConcurrentReads[1]: [measured 100 out of 200 rounds, threads: 1 (sequential)]
 *   round: 0.12 [+- 0.01], round.block: 0.00 [+- 0.00], round.gc: 0.02 [+- 0.00], GC.calls: 396, GC.time: 2.05, time.total: 29.21, time.warmup: 14.65, time.bench: 14.55
 */
@BenchmarkOptions(warmupRounds = 100, benchmarkRounds = 100, clock = Clock.REAL_TIME, callgc = true)
@Ignore
@Test
public void testConcurrentReads() throws ExecutionException, InterruptedException {
    final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(concurrentThreads, new ThreadFactoryBuilder().setDaemon(false).setNameFormat("indexible-writes-benchmark-reader-%d").build()));
    final AtomicInteger index = new AtomicInteger(0);
    final AtomicInteger queryableIndex = new AtomicInteger(0);
    List<ListenableFuture<?>> futures = new ArrayList<>();
    final Integer loops = totalIndexSize / concurrentThreads;
    final AtomicBoolean done = new AtomicBoolean(false);
    final CountDownLatch start = new CountDownLatch(1);
    for (int i = 0; i < concurrentThreads; ++i) {
        futures.add(executorService.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    start.await();
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
                final Random rndGen = ThreadLocalRandom.current();
                while (!done.get()) {
                    Integer idx = rndGen.nextInt(queryableIndex.get() + 1);
                    Assert.assertEquals(idx, concurrentIndexible.get(idx));
                }
            }
        }));
    }
    {
        final Integer idx = index.getAndIncrement();
        concurrentIndexible.set(idx, idx);
        start.countDown();
    }
    for (int i = 1; i < totalIndexSize; ++i) {
        final Integer idx = index.getAndIncrement();
        concurrentIndexible.set(idx, idx);
        queryableIndex.incrementAndGet();
    }
    done.set(true);
    Futures.allAsList(futures).get();
    executorService.shutdown();
    Assert.assertTrue(StringUtils.format("Index too small %d, expected %d across %d loops", index.get(), totalIndexSize, loops), index.get() >= totalIndexSize);
    for (int i = 0; i < index.get(); ++i) {
        Assert.assertEquals(i, concurrentIndexible.get(i).intValue());
    }
    concurrentIndexible.clear();
    futures.clear();
}
Also used : ArrayList(java.util.ArrayList) CountDownLatch(java.util.concurrent.CountDownLatch) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Random(java.util.Random) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Ignore(org.junit.Ignore) Test(org.junit.Test) BenchmarkOptions(com.carrotsearch.junitbenchmarks.BenchmarkOptions)

Aggregations

BenchmarkOptions (com.carrotsearch.junitbenchmarks.BenchmarkOptions)42 Test (org.junit.Test)42 DirectByteBufferAllocator (org.apache.parquet.bytes.DirectByteBufferAllocator)10 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)6 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)6 ThreadFactoryBuilder (com.google.common.util.concurrent.ThreadFactoryBuilder)6 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)6 ByteBufferInputStream (org.apache.parquet.bytes.ByteBufferInputStream)6 Binary (org.apache.parquet.io.api.Binary)6 Ignore (org.junit.Ignore)6 ArrayList (java.util.ArrayList)4 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)4 Pointer (org.directmemory.memory.Pointer)4 BasicDBObject (com.mongodb.BasicDBObject)3 MutableBitmap (io.druid.collections.bitmap.MutableBitmap)3 LinkedList (java.util.LinkedList)3 MutableBitmap (org.apache.druid.collections.bitmap.MutableBitmap)3 DeltaBinaryPackingValuesWriter (org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriter)3 BinaryPlainValuesReader (org.apache.parquet.column.values.plain.BinaryPlainValuesReader)3 PlainValuesWriter (org.apache.parquet.column.values.plain.PlainValuesWriter)3