Examples with MapBasedInputRow org.apache.druid.data.input.MapBasedInputRow used on opensource projects

Example 11 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class SpatialFilterTest method makeMergedQueryableIndex.

private static QueryableIndex makeMergedQueryableIndex(IndexSpec indexSpec) {
    try {
        IncrementalIndex first = new OnheapIncrementalIndex.Builder().setIndexSchema(new IncrementalIndexSchema.Builder().withMinTimestamp(DATA_INTERVAL.getStartMillis()).withQueryGranularity(Granularities.DAY).withMetrics(METRIC_AGGS).withDimensionsSpec(DimensionsSpec.builder().setSpatialDimensions(Arrays.asList(new SpatialDimensionSchema("dim.geo", Arrays.asList("lat", "long")), new SpatialDimensionSchema("spatialIsRad", Arrays.asList("lat2", "long2")))).build()).build()).setMaxRowCount(1000).build();
        IncrementalIndex second = new OnheapIncrementalIndex.Builder().setIndexSchema(new IncrementalIndexSchema.Builder().withMinTimestamp(DATA_INTERVAL.getStartMillis()).withQueryGranularity(Granularities.DAY).withMetrics(METRIC_AGGS).withDimensionsSpec(DimensionsSpec.builder().setSpatialDimensions(Arrays.asList(new SpatialDimensionSchema("dim.geo", Arrays.asList("lat", "long")), new SpatialDimensionSchema("spatialIsRad", Arrays.asList("lat2", "long2")))).build()).build()).setMaxRowCount(1000).build();
        IncrementalIndex third = new OnheapIncrementalIndex.Builder().setIndexSchema(new IncrementalIndexSchema.Builder().withMinTimestamp(DATA_INTERVAL.getStartMillis()).withQueryGranularity(Granularities.DAY).withMetrics(METRIC_AGGS).withDimensionsSpec(DimensionsSpec.builder().setSpatialDimensions(Arrays.asList(new SpatialDimensionSchema("dim.geo", Arrays.asList("lat", "long")), new SpatialDimensionSchema("spatialIsRad", Arrays.asList("lat2", "long2")))).build()).build()).setMaxRowCount(NUM_POINTS).build();
        first.add(new MapBasedInputRow(DateTimes.of("2013-01-01").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-01").toString(), "dim", "foo", "lat", 0.0f, "long", 0.0f, "val", 17L)));
        first.add(new MapBasedInputRow(DateTimes.of("2013-01-02").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-02").toString(), "dim", "foo", "lat", 1.0f, "long", 3.0f, "val", 29L)));
        first.add(new MapBasedInputRow(DateTimes.of("2013-01-03").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-03").toString(), "dim", "foo", "lat", 4.0f, "long", 2.0f, "val", 13L)));
        first.add(new MapBasedInputRow(DateTimes.of("2013-01-05").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-05").toString(), "dim", "foo", "lat", "_mmx.unknown", "long", "_mmx.unknown", "val", 101L)));
        first.add(new MapBasedInputRow(DateTimes.of("2013-01-05").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-05").toString(), "dim", "foo", "dim.geo", "_mmx.unknown", "val", 501L)));
        second.add(new MapBasedInputRow(DateTimes.of("2013-01-04").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-04").toString(), "dim", "foo", "lat", 7.0f, "long", 3.0f, "val", 91L)));
        second.add(new MapBasedInputRow(DateTimes.of("2013-01-05").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-05").toString(), "dim", "foo", "lat", 8.0f, "long", 6.0f, "val", 47L)));
        second.add(new MapBasedInputRow(DateTimes.of("2013-01-05").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-05").toString(), "lat2", 0.0f, "long2", 0.0f, "val", 13L)));
        // Add a bunch of random points
        Random rand = ThreadLocalRandom.current();
        for (int i = 8; i < NUM_POINTS; i++) {
            third.add(new MapBasedInputRow(DateTimes.of("2013-01-01").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-01").toString(), "dim", "boo", "lat", (float) (rand.nextFloat() * 10 + 10.0), "long", (float) (rand.nextFloat() * 10 + 10.0), "val", i)));
        }
        File tmpFile = File.createTempFile("yay", "who");
        tmpFile.delete();
        File firstFile = new File(tmpFile, "first");
        File secondFile = new File(tmpFile, "second");
        File thirdFile = new File(tmpFile, "third");
        File mergedFile = new File(tmpFile, "merged");
        FileUtils.mkdirp(firstFile);
        FileUtils.mkdirp(secondFile);
        FileUtils.mkdirp(thirdFile);
        FileUtils.mkdirp(mergedFile);
        firstFile.deleteOnExit();
        secondFile.deleteOnExit();
        thirdFile.deleteOnExit();
        mergedFile.deleteOnExit();
        INDEX_MERGER.persist(first, DATA_INTERVAL, firstFile, indexSpec, null);
        INDEX_MERGER.persist(second, DATA_INTERVAL, secondFile, indexSpec, null);
        INDEX_MERGER.persist(third, DATA_INTERVAL, thirdFile, indexSpec, null);
        QueryableIndex mergedRealtime = INDEX_IO.loadIndex(INDEX_MERGER.mergeQueryableIndex(Arrays.asList(INDEX_IO.loadIndex(firstFile), INDEX_IO.loadIndex(secondFile), INDEX_IO.loadIndex(thirdFile)), true, METRIC_AGGS, mergedFile, indexSpec, null, -1));
        return mergedRealtime;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 12 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class SpatialFilterTest method makeIncrementalIndex.

private static IncrementalIndex makeIncrementalIndex() throws IOException {
    IncrementalIndex theIndex = new OnheapIncrementalIndex.Builder().setIndexSchema(new IncrementalIndexSchema.Builder().withMinTimestamp(DATA_INTERVAL.getStartMillis()).withQueryGranularity(Granularities.DAY).withMetrics(METRIC_AGGS).withDimensionsSpec(DimensionsSpec.builder().setSpatialDimensions(Arrays.asList(new SpatialDimensionSchema("dim.geo", Arrays.asList("lat", "long")), new SpatialDimensionSchema("spatialIsRad", Arrays.asList("lat2", "long2")))).build()).build()).setMaxRowCount(NUM_POINTS).build();
    theIndex.add(new MapBasedInputRow(DateTimes.of("2013-01-01").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-01").toString(), "dim", "foo", "lat", 0.0f, "long", 0.0f, "val", 17L)));
    theIndex.add(new MapBasedInputRow(DateTimes.of("2013-01-02").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-02").toString(), "dim", "foo", "lat", 1.0f, "long", 3.0f, "val", 29L)));
    theIndex.add(new MapBasedInputRow(DateTimes.of("2013-01-03").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-03").toString(), "dim", "foo", "lat", 4.0f, "long", 2.0f, "val", 13L)));
    theIndex.add(new MapBasedInputRow(DateTimes.of("2013-01-04").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-04").toString(), "dim", "foo", "lat", 7.0f, "long", 3.0f, "val", 91L)));
    theIndex.add(new MapBasedInputRow(DateTimes.of("2013-01-05").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-05").toString(), "dim", "foo", "lat", 8.0f, "long", 6.0f, "val", 47L)));
    theIndex.add(new MapBasedInputRow(DateTimes.of("2013-01-05").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-05").toString(), "dim", "foo", "lat", "_mmx.unknown", "long", "_mmx.unknown", "val", 101L)));
    theIndex.add(new MapBasedInputRow(DateTimes.of("2013-01-05").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-05").toString(), "dim", "foo", "dim.geo", "_mmx.unknown", "val", 501L)));
    theIndex.add(new MapBasedInputRow(DateTimes.of("2013-01-05").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-05").toString(), "lat2", 0.0f, "long2", 0.0f, "val", 13L)));
    // Add a bunch of random points
    Random rand = ThreadLocalRandom.current();
    for (int i = 8; i < NUM_POINTS; i++) {
        theIndex.add(new MapBasedInputRow(DateTimes.of("2013-01-01").getMillis(), DIMS, ImmutableMap.of("timestamp", DateTimes.of("2013-01-01").toString(), "dim", "boo", "lat", (float) (rand.nextFloat() * 10 + 10.0), "long", (float) (rand.nextFloat() * 10 + 10.0), "val", i)));
    }
    return theIndex;
}

Example 13 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IncrementalIndexIngestionTest method testMultithreadAddFactsUsingExpressionAndJavaScript.

@Test
public void testMultithreadAddFactsUsingExpressionAndJavaScript() throws Exception {
    final IncrementalIndex indexExpr = indexCreator.createIndex(new IncrementalIndexSchema.Builder().withQueryGranularity(Granularities.MINUTE).withMetrics(new LongSumAggregatorFactory("oddnum", null, "if(value%2==1,1,0)", TestExprMacroTable.INSTANCE)).withRollup(true).build());
    final IncrementalIndex indexJs = indexCreator.createIndex(new IncrementalIndexSchema.Builder().withQueryGranularity(Granularities.MINUTE).withMetrics(new JavaScriptAggregatorFactory("oddnum", ImmutableList.of("value"), "function(current, value) { if (value%2==1) current = current + 1; return current;}", "function() {return 0;}", "function(a, b) { return a + b;}", JavaScriptConfig.getEnabledInstance())).withRollup(true).build());
    final int addThreadCount = 2;
    Thread[] addThreads = new Thread[addThreadCount];
    for (int i = 0; i < addThreadCount; ++i) {
        addThreads[i] = new Thread(new Runnable() {

            @Override
            public void run() {
                final Random random = ThreadLocalRandom.current();
                try {
                    for (int j = 0; j < MAX_ROWS / addThreadCount; ++j) {
                        int randomInt = random.nextInt(100000);
                        MapBasedInputRow mapBasedInputRowExpr = new MapBasedInputRow(0, Collections.singletonList("billy"), ImmutableMap.of("billy", randomInt % 3, "value", randomInt));
                        MapBasedInputRow mapBasedInputRowJs = new MapBasedInputRow(0, Collections.singletonList("billy"), ImmutableMap.of("billy", randomInt % 3, "value", randomInt));
                        indexExpr.add(mapBasedInputRowExpr);
                        indexJs.add(mapBasedInputRowJs);
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        });
        addThreads[i].start();
    }
    for (int i = 0; i < addThreadCount; ++i) {
        addThreads[i].join();
    }
    long exprSum = 0;
    long jsSum = 0;
    for (IncrementalIndexRow row : indexExpr.getFacts().keySet()) {
        exprSum += indexExpr.getMetricLongValue(row.getRowIndex(), 0);
    }
    for (IncrementalIndexRow row : indexJs.getFacts().keySet()) {
        jsSum += indexJs.getMetricLongValue(row.getRowIndex(), 0);
    }
    Assert.assertEquals(exprSum, jsSum);
}

Example 14 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IncrementalIndexStorageAdapterTest method testSingleValueTopN.

@Test
public void testSingleValueTopN() throws IOException {
    IncrementalIndex index = indexCreator.createIndex();
    DateTime t = DateTimes.nowUtc();
    index.add(new MapBasedInputRow(t.minus(1).getMillis(), Collections.singletonList("sally"), ImmutableMap.of("sally", "bo")));
    try (CloseableStupidPool<ByteBuffer> pool = new CloseableStupidPool<>("TopNQueryEngine-bufferPool", () -> ByteBuffer.allocate(50000))) {
        TopNQueryEngine engine = new TopNQueryEngine(pool);
        final Iterable<Result<TopNResultValue>> results = engine.query(new TopNQueryBuilder().dataSource("test").granularity(Granularities.ALL).intervals(Collections.singletonList(new Interval(DateTimes.EPOCH, DateTimes.nowUtc()))).dimension("sally").metric("cnt").threshold(10).aggregators(new LongSumAggregatorFactory("cnt", "cnt")).build(), new IncrementalIndexStorageAdapter(index), null).toList();
        Assert.assertEquals(1, Iterables.size(results));
        Assert.assertEquals(1, results.iterator().next().getValue().getValue().size());
    }
}

Example 15 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IncrementalIndexStorageAdapterTest method testFilterByNull.

@Test
public void testFilterByNull() throws Exception {
    IncrementalIndex index = indexCreator.createIndex();
    index.add(new MapBasedInputRow(System.currentTimeMillis() - 1, Collections.singletonList("billy"), ImmutableMap.of("billy", "hi")));
    index.add(new MapBasedInputRow(System.currentTimeMillis() - 1, Collections.singletonList("sally"), ImmutableMap.of("sally", "bo")));
    try (CloseableStupidPool<ByteBuffer> pool = new CloseableStupidPool<>("GroupByQueryEngine-bufferPool", () -> ByteBuffer.allocate(50000))) {
        final GroupByQueryEngine engine = new GroupByQueryEngine(Suppliers.ofInstance(new GroupByQueryConfig() {

            @Override
            public int getMaxIntermediateRows() {
                return 5;
            }
        }), pool);
        final Sequence<Row> rows = engine.process(GroupByQuery.builder().setDataSource("test").setGranularity(Granularities.ALL).setInterval(new Interval(DateTimes.EPOCH, DateTimes.nowUtc())).addDimension("billy").addDimension("sally").addAggregator(new LongSumAggregatorFactory("cnt", "cnt")).setDimFilter(DimFilters.dimEquals("sally", (String) null)).build(), new IncrementalIndexStorageAdapter(index));
        final List<Row> results = rows.toList();
        Assert.assertEquals(1, results.size());
        MapBasedRow row = (MapBasedRow) results.get(0);
        Assert.assertEquals(ImmutableMap.of("billy", "hi", "cnt", 1L), row.getEvent());
    }
}