Search in sources :

Example 1 with Setup

use of org.openjdk.jmh.annotations.Setup in project druid by druid-io.

the class BoundFilterBenchmark method setup.

@Setup
public void setup() throws IOException {
    step = (END_INT - START_INT) / cardinality;
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new BitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add((i - START_INT) / step);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new BitmapIndexSelector() {

        @Override
        public Indexed<String> getDimensionValues(String dimension) {
            return dictionary;
        }

        @Override
        public int getNumRows() {
            throw new UnsupportedOperationException();
        }

        @Override
        public BitmapFactory getBitmapFactory() {
            return bitmapFactory;
        }

        @Override
        public ImmutableBitmap getBitmapIndex(String dimension, String value) {
            return bitmapIndex.getBitmap(bitmapIndex.getIndex(value));
        }

        @Override
        public BitmapIndex getBitmapIndex(String dimension) {
            return bitmapIndex;
        }

        @Override
        public ImmutableRTree getSpatialIndex(String dimension) {
            throw new UnsupportedOperationException();
        }
    };
}
Also used : ImmutableBitmap(io.druid.collections.bitmap.ImmutableBitmap) MutableBitmap(io.druid.collections.bitmap.MutableBitmap) BitmapIndex(io.druid.segment.column.BitmapIndex) Function(com.google.common.base.Function) ImmutableRTree(io.druid.collections.spatial.ImmutableRTree) RoaringBitmapSerdeFactory(io.druid.segment.data.RoaringBitmapSerdeFactory) BitmapIndexColumnPartSupplier(io.druid.segment.serde.BitmapIndexColumnPartSupplier) BitmapIndexSelector(io.druid.query.filter.BitmapIndexSelector) RoaringBitmapFactory(io.druid.collections.bitmap.RoaringBitmapFactory) BitmapFactory(io.druid.collections.bitmap.BitmapFactory) RoaringBitmapFactory(io.druid.collections.bitmap.RoaringBitmapFactory) Indexed(io.druid.segment.data.Indexed) GenericIndexed(io.druid.segment.data.GenericIndexed) RoaringBitmapSerdeFactory(io.druid.segment.data.RoaringBitmapSerdeFactory) BitmapSerdeFactory(io.druid.segment.data.BitmapSerdeFactory) Setup(org.openjdk.jmh.annotations.Setup)

Example 2 with Setup

use of org.openjdk.jmh.annotations.Setup in project elasticsearch by elastic.

the class AllocationBenchmark method setUp.

@Setup
public void setUp() throws Exception {
    final String[] params = indicesShardsReplicasNodes.split("\\|");
    int numIndices = toInt(params[0]);
    int numShards = toInt(params[1]);
    int numReplicas = toInt(params[2]);
    int numNodes = toInt(params[3]);
    strategy = Allocators.createAllocationService(Settings.builder().put("cluster.routing.allocation.awareness.attributes", "tag").build());
    MetaData.Builder mb = MetaData.builder();
    for (int i = 1; i <= numIndices; i++) {
        mb.put(IndexMetaData.builder("test_" + i).settings(Settings.builder().put("index.version.created", Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas));
    }
    MetaData metaData = mb.build();
    RoutingTable.Builder rb = RoutingTable.builder();
    for (int i = 1; i <= numIndices; i++) {
        rb.addAsNew(metaData.index("test_" + i));
    }
    RoutingTable routingTable = rb.build();
    DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
    for (int i = 1; i <= numNodes; i++) {
        nb.add(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags))));
    }
    initialClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).nodes(nb).build();
}
Also used : RoutingTable(org.elasticsearch.cluster.routing.RoutingTable) MetaData(org.elasticsearch.cluster.metadata.MetaData) IndexMetaData(org.elasticsearch.cluster.metadata.IndexMetaData) DiscoveryNodes(org.elasticsearch.cluster.node.DiscoveryNodes) Setup(org.openjdk.jmh.annotations.Setup)

Example 3 with Setup

use of org.openjdk.jmh.annotations.Setup in project pinot by linkedin.

the class BenchmarkFileRead method loadData.

@Setup
public void loadData() {
    try {
        file = new File("/Users/jfim/index_dir/sTest_OFFLINE/sTest_0_0/daysSinceEpoch.sv.unsorted.fwd");
        raf = new RandomAccessFile(file, "rw");
        length = (int) file.length();
        byteBuffer = ByteBuffer.allocate(length);
        raf.getChannel().read(byteBuffer);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
Also used : RandomAccessFile(java.io.RandomAccessFile) IOException(java.io.IOException) RandomAccessFile(java.io.RandomAccessFile) File(java.io.File) Setup(org.openjdk.jmh.annotations.Setup)

Example 4 with Setup

use of org.openjdk.jmh.annotations.Setup in project pinot by linkedin.

the class BenchmarkQueryEngine method startPinot.

@Setup
public void startPinot() throws Exception {
    System.out.println("Using table name " + TABLE_NAME);
    System.out.println("Using data directory " + DATA_DIRECTORY);
    System.out.println("Starting pinot");
    PerfBenchmarkDriverConf conf = new PerfBenchmarkDriverConf();
    conf.setStartBroker(true);
    conf.setStartController(true);
    conf.setStartServer(true);
    conf.setStartZookeeper(true);
    conf.setUploadIndexes(false);
    conf.setRunQueries(false);
    conf.setServerInstanceSegmentTarDir(null);
    conf.setServerInstanceDataDir(DATA_DIRECTORY);
    conf.setConfigureResources(false);
    _perfBenchmarkDriver = new PerfBenchmarkDriver(conf);
    _perfBenchmarkDriver.run();
    Set<String> tables = new HashSet<String>();
    File[] segments = new File(DATA_DIRECTORY, TABLE_NAME).listFiles();
    for (File segmentDir : segments) {
        SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(segmentDir);
        if (!tables.contains(segmentMetadata.getTableName())) {
            _perfBenchmarkDriver.configureTable(segmentMetadata.getTableName());
            tables.add(segmentMetadata.getTableName());
        }
        System.out.println("Adding segment " + segmentDir.getAbsolutePath());
        _perfBenchmarkDriver.addSegment(segmentMetadata);
    }
    ZkClient client = new ZkClient("localhost:2191", 10000, 10000, new ZNRecordSerializer());
    ZNRecord record = client.readData("/PinotPerfTestCluster/EXTERNALVIEW/" + TABLE_NAME);
    while (true) {
        System.out.println("record = " + record);
        Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS);
        int onlineSegmentCount = 0;
        for (Map<String, String> instancesAndStates : record.getMapFields().values()) {
            for (String state : instancesAndStates.values()) {
                if (state.equals("ONLINE")) {
                    onlineSegmentCount++;
                    break;
                }
            }
        }
        System.out.println(onlineSegmentCount + " segments online out of " + segments.length);
        if (onlineSegmentCount == segments.length) {
            break;
        }
        record = client.readData("/PinotPerfTestCluster/EXTERNALVIEW/" + TABLE_NAME);
    }
    ranOnce = false;
    System.out.println(_perfBenchmarkDriver.postQuery(QUERY_PATTERNS[queryPattern], optimizationFlags).toString(2));
}
Also used : ZkClient(org.I0Itec.zkclient.ZkClient) PerfBenchmarkDriver(com.linkedin.pinot.tools.perf.PerfBenchmarkDriver) SegmentMetadataImpl(com.linkedin.pinot.core.segment.index.SegmentMetadataImpl) PerfBenchmarkDriverConf(com.linkedin.pinot.tools.perf.PerfBenchmarkDriverConf) File(java.io.File) ZNRecord(org.apache.helix.ZNRecord) HashSet(java.util.HashSet) ZNRecordSerializer(org.apache.helix.manager.zk.ZNRecordSerializer) Setup(org.openjdk.jmh.annotations.Setup)

Example 5 with Setup

use of org.openjdk.jmh.annotations.Setup in project druid by druid-io.

the class LikeFilterBenchmark method setup.

@Setup
public void setup() throws IOException {
    step = (END_INT - START_INT) / cardinality;
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new BitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add((i - START_INT) / step);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new BitmapIndexSelector() {

        @Override
        public Indexed<String> getDimensionValues(String dimension) {
            return dictionary;
        }

        @Override
        public int getNumRows() {
            throw new UnsupportedOperationException();
        }

        @Override
        public BitmapFactory getBitmapFactory() {
            return bitmapFactory;
        }

        @Override
        public ImmutableBitmap getBitmapIndex(String dimension, String value) {
            return bitmapIndex.getBitmap(bitmapIndex.getIndex(value));
        }

        @Override
        public BitmapIndex getBitmapIndex(String dimension) {
            return bitmapIndex;
        }

        @Override
        public ImmutableRTree getSpatialIndex(String dimension) {
            throw new UnsupportedOperationException();
        }
    };
}
Also used : ImmutableBitmap(io.druid.collections.bitmap.ImmutableBitmap) MutableBitmap(io.druid.collections.bitmap.MutableBitmap) BitmapIndex(io.druid.segment.column.BitmapIndex) Function(com.google.common.base.Function) ImmutableRTree(io.druid.collections.spatial.ImmutableRTree) RoaringBitmapSerdeFactory(io.druid.segment.data.RoaringBitmapSerdeFactory) BitmapIndexColumnPartSupplier(io.druid.segment.serde.BitmapIndexColumnPartSupplier) BitmapIndexSelector(io.druid.query.filter.BitmapIndexSelector) RoaringBitmapFactory(io.druid.collections.bitmap.RoaringBitmapFactory) BitmapFactory(io.druid.collections.bitmap.BitmapFactory) RoaringBitmapFactory(io.druid.collections.bitmap.RoaringBitmapFactory) Indexed(io.druid.segment.data.Indexed) GenericIndexed(io.druid.segment.data.GenericIndexed) RoaringBitmapSerdeFactory(io.druid.segment.data.RoaringBitmapSerdeFactory) BitmapSerdeFactory(io.druid.segment.data.BitmapSerdeFactory) Setup(org.openjdk.jmh.annotations.Setup)

Aggregations

Setup (org.openjdk.jmh.annotations.Setup)297 Random (java.util.Random)61 File (java.io.File)32 ArrayList (java.util.ArrayList)24 InputRow (io.druid.data.input.InputRow)15 BenchmarkDataGenerator (io.druid.benchmark.datagen.BenchmarkDataGenerator)14 HyperUniquesSerde (io.druid.query.aggregation.hyperloglog.HyperUniquesSerde)14 HyperUniquesSerde (org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde)14 DataGenerator (org.apache.druid.segment.generator.DataGenerator)14 DataSegment (org.apache.druid.timeline.DataSegment)13 Scope (org.openjdk.jmh.annotations.Scope)13 State (org.openjdk.jmh.annotations.State)13 ByteBuffer (java.nio.ByteBuffer)12 Benchmark (org.openjdk.jmh.annotations.Benchmark)12 IOException (java.io.IOException)11 IndexSpec (io.druid.segment.IndexSpec)10 List (java.util.List)10 TopicPartition (org.apache.kafka.common.TopicPartition)10 LinearShardSpec (org.apache.druid.timeline.partition.LinearShardSpec)9 Function (com.google.common.base.Function)8