Examples with MapBasedInputRow org.apache.druid.data.input.MapBasedInputRow used on opensource projects

Example 26 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class TransformerTest method testTransformTimeColumn.

@Test
public void testTransformTimeColumn() {
    final Transformer transformer = new Transformer(new TransformSpec(null, ImmutableList.of(new ExpressionTransform("__time", "timestamp_shift(__time, 'P1D', -2)", TestExprMacroTable.INSTANCE))));
    final DateTime now = DateTimes.nowUtc();
    final InputRow row = new MapBasedInputRow(now, ImmutableList.of("dim"), ImmutableMap.of("__time", now, "dim", false));
    final InputRow actual = transformer.transform(row);
    Assert.assertNotNull(actual);
    Assert.assertEquals(now.minusDays(2), actual.getTimestamp());
}

Example 27 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class TransformerTest method testTransformWithStringTransformOnListColumnThrowingException.

@Ignore("Disabled until https://github.com/apache/druid/issues/9824 is fixed")
@Test
public void testTransformWithStringTransformOnListColumnThrowingException() {
    final Transformer transformer = new Transformer(new TransformSpec(null, ImmutableList.of(new ExpressionTransform("dim", "strlen(dim)", TestExprMacroTable.INSTANCE))));
    final InputRow row = new MapBasedInputRow(DateTimes.nowUtc(), ImmutableList.of("dim"), ImmutableMap.of("dim", ImmutableList.of(10, 20, 100)));
    final InputRow actual = transformer.transform(row);
    Assert.assertNotNull(actual);
    Assert.assertEquals(ImmutableList.of("dim"), actual.getDimensions());
    // Unlike for querying, Druid doesn't explode multi-valued columns automatically for ingestion.
    expectedException.expect(AssertionError.class);
    actual.getRaw("dim");
}

Example 28 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTestBase method testMismatchedDimensions.

@Test
public void testMismatchedDimensions() throws IOException {
    IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
    index1.add(new MapBasedInputRow(1L, Arrays.asList("d1", "d2"), ImmutableMap.of("d1", "a", "d2", "z", "A", 1)));
    closer.closeLater(index1);
    IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
    index2.add(new MapBasedInputRow(1L, Arrays.asList("d1", "d2"), ImmutableMap.of("d1", "a", "d2", "z", "A", 2, "C", 100)));
    closer.closeLater(index2);
    Interval interval = new Interval(DateTimes.EPOCH, DateTimes.nowUtc());
    RoaringBitmapFactory factory = new RoaringBitmapFactory();
    List<IndexableAdapter> toMerge = Arrays.asList(new IncrementalIndexAdapter(interval, index1, factory), new IncrementalIndexAdapter(interval, index2, factory));
    final File tmpDirMerged = temporaryFolder.newFolder();
    indexMerger.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") }, tmpDirMerged, indexSpec, -1);
}

Example 29 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTestBase method testMergeWithSupersetOrdering.

@Test
public void testMergeWithSupersetOrdering() throws Exception {
    IncrementalIndex toPersistA = getSingleDimIndex("dimA", Arrays.asList("1", "2"));
    IncrementalIndex toPersistB = getSingleDimIndex("dimB", Arrays.asList("1", "2", "3"));
    IncrementalIndex toPersistBA = getSingleDimIndex("dimB", Arrays.asList("1", "2", "3"));
    addDimValuesToIndex(toPersistBA, "dimA", Arrays.asList("1", "2"));
    IncrementalIndex toPersistBA2 = new OnheapIncrementalIndex.Builder().setSimpleTestingIndexSchema(new CountAggregatorFactory("count")).setMaxRowCount(1000).build();
    toPersistBA2.add(new MapBasedInputRow(1, Arrays.asList("dimB", "dimA"), ImmutableMap.of("dimB", "1")));
    toPersistBA2.add(new MapBasedInputRow(1, Arrays.asList("dimB", "dimA"), ImmutableMap.of("dimA", "1")));
    IncrementalIndex toPersistC = getSingleDimIndex("dimA", Arrays.asList("1", "2"));
    addDimValuesToIndex(toPersistC, "dimC", Arrays.asList("1", "2", "3"));
    final File tmpDirA = temporaryFolder.newFolder();
    final File tmpDirB = temporaryFolder.newFolder();
    final File tmpDirBA = temporaryFolder.newFolder();
    final File tmpDirBA2 = temporaryFolder.newFolder();
    final File tmpDirC = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    final File tmpDirMerged2 = temporaryFolder.newFolder();
    QueryableIndex indexA = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistA, tmpDirA, indexSpec, null)));
    QueryableIndex indexB = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistB, tmpDirB, indexSpec, null)));
    QueryableIndex indexBA = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistBA, tmpDirBA, indexSpec, null)));
    QueryableIndex indexBA2 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistBA2, tmpDirBA2, indexSpec, null)));
    QueryableIndex indexC = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistC, tmpDirC, indexSpec, null)));
    final QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexBA, indexBA2), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged, indexSpec, null, -1)));
    final QueryableIndex merged2 = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB, indexBA, indexC), true, new AggregatorFactory[] { new CountAggregatorFactory("count") }, tmpDirMerged2, indexSpec, null, -1)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
    final QueryableIndexIndexableAdapter adapter2 = new QueryableIndexIndexableAdapter(merged2);
    final List<DebugRow> rowList2 = RowIteratorHelper.toList(adapter2.getRows());
    Assert.assertEquals(ImmutableList.of("dimB", "dimA"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(5, rowList.size());
    Assert.assertEquals(Arrays.asList(null, "1"), rowList.get(0).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList.get(0).metricValues());
    Assert.assertEquals(Arrays.asList(null, "2"), rowList.get(1).dimensionValues());
    Assert.assertEquals(Collections.singletonList(2L), rowList.get(1).metricValues());
    Assert.assertEquals(Arrays.asList("1", null), rowList.get(2).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList.get(2).metricValues());
    Assert.assertEquals(Arrays.asList("2", null), rowList.get(3).dimensionValues());
    Assert.assertEquals(Collections.singletonList(2L), rowList.get(3).metricValues());
    Assert.assertEquals(Arrays.asList("3", null), rowList.get(4).dimensionValues());
    Assert.assertEquals(Collections.singletonList(2L), rowList.get(4).metricValues());
    checkBitmapIndex(Arrays.asList(2, 3, 4), adapter.getBitmapIndex("dimA", null));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimA", "1"));
    checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("dimA", "2"));
    checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dimB", null));
    checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimB", "1"));
    checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("dimB", "2"));
    checkBitmapIndex(Collections.singletonList(4), adapter.getBitmapIndex("dimB", "3"));
    Assert.assertEquals(ImmutableList.of("dimA", "dimB", "dimC"), ImmutableList.copyOf(adapter2.getDimensionNames()));
    Assert.assertEquals(12, rowList2.size());
    Assert.assertEquals(Arrays.asList(null, null, "1"), rowList2.get(0).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(0).metricValues());
    Assert.assertEquals(Arrays.asList(null, null, "2"), rowList2.get(1).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(1).metricValues());
    Assert.assertEquals(Arrays.asList(null, null, "3"), rowList2.get(2).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(2).metricValues());
    Assert.assertEquals(Arrays.asList(null, "1", null), rowList2.get(3).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(3).metricValues());
    Assert.assertEquals(Arrays.asList(null, "2", null), rowList2.get(4).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(4).metricValues());
    Assert.assertEquals(Arrays.asList(null, "3", null), rowList2.get(5).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(5).metricValues());
    Assert.assertEquals(Arrays.asList("1", null, null), rowList2.get(6).dimensionValues());
    Assert.assertEquals(Collections.singletonList(3L), rowList2.get(6).metricValues());
    Assert.assertEquals(Arrays.asList("2", null, null), rowList2.get(7).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(7).metricValues());
    Assert.assertEquals(Arrays.asList(null, "1", null), rowList2.get(8).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(8).metricValues());
    Assert.assertEquals(Arrays.asList(null, "2", null), rowList2.get(9).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(9).metricValues());
    Assert.assertEquals(Arrays.asList(null, "3", null), rowList2.get(10).dimensionValues());
    Assert.assertEquals(Collections.singletonList(1L), rowList2.get(10).metricValues());
    Assert.assertEquals(Arrays.asList("2", null, null), rowList2.get(11).dimensionValues());
    Assert.assertEquals(Collections.singletonList(2L), rowList2.get(11).metricValues());
    checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 8, 9, 10), adapter2.getBitmapIndex("dimA", null));
    checkBitmapIndex(Collections.singletonList(6), adapter2.getBitmapIndex("dimA", "1"));
    checkBitmapIndex(Arrays.asList(7, 11), adapter2.getBitmapIndex("dimA", "2"));
    checkBitmapIndex(Arrays.asList(0, 1, 2, 6, 7, 11), adapter2.getBitmapIndex("dimB", null));
    checkBitmapIndex(Arrays.asList(3, 8), adapter2.getBitmapIndex("dimB", "1"));
    checkBitmapIndex(Arrays.asList(4, 9), adapter2.getBitmapIndex("dimB", "2"));
    checkBitmapIndex(Arrays.asList(5, 10), adapter2.getBitmapIndex("dimB", "3"));
    checkBitmapIndex(Arrays.asList(3, 4, 5, 6, 7, 8, 9, 10, 11), adapter2.getBitmapIndex("dimC", null));
    checkBitmapIndex(Collections.singletonList(0), adapter2.getBitmapIndex("dimC", "1"));
    checkBitmapIndex(Collections.singletonList(1), adapter2.getBitmapIndex("dimC", "2"));
    checkBitmapIndex(Collections.singletonList(2), adapter2.getBitmapIndex("dimC", "3"));
}

Example 30 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTestBase method testMultivalDim_mergeAcrossSegments_rollupWorks.

@Test
public void testMultivalDim_mergeAcrossSegments_rollupWorks() throws Exception {
    List<String> dims = Arrays.asList("dimA", "dimMultiVal");
    IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder().withDimensionsSpec(new DimensionsSpec(ImmutableList.of(new StringDimensionSchema("dimA", MultiValueHandling.SORTED_ARRAY, true), new StringDimensionSchema("dimMultiVal", MultiValueHandling.SORTED_ARRAY, true)))).withMetrics(new LongSumAggregatorFactory("sumCount", "sumCount")).withRollup(true).build();
    IncrementalIndex toPersistA = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
    Map<String, Object> event1 = new HashMap<>();
    event1.put("dimA", "leek");
    event1.put("dimMultiVal", ImmutableList.of("1", "2", "4"));
    event1.put("sumCount", 1L);
    Map<String, Object> event2 = new HashMap<>();
    event2.put("dimA", "leek");
    event2.put("dimMultiVal", ImmutableList.of("1", "2", "3", "5"));
    event2.put("sumCount", 1L);
    toPersistA.add(new MapBasedInputRow(1, dims, event1));
    toPersistA.add(new MapBasedInputRow(1, dims, event2));
    IncrementalIndex toPersistB = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
    Map<String, Object> event3 = new HashMap<>();
    event3.put("dimA", "leek");
    event3.put("dimMultiVal", ImmutableList.of("1", "2", "4"));
    event3.put("sumCount", 1L);
    Map<String, Object> event4 = new HashMap<>();
    event4.put("dimA", "potato");
    event4.put("dimMultiVal", ImmutableList.of("0", "1", "4"));
    event4.put("sumCount", 1L);
    toPersistB.add(new MapBasedInputRow(1, dims, event3));
    toPersistB.add(new MapBasedInputRow(1, dims, event4));
    final File tmpDirA = temporaryFolder.newFolder();
    final File tmpDirB = temporaryFolder.newFolder();
    final File tmpDirMerged = temporaryFolder.newFolder();
    QueryableIndex indexA = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistA, tmpDirA, indexSpec, null)));
    QueryableIndex indexB = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistB, tmpDirB, indexSpec, null)));
    final QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(indexA, indexB), true, new AggregatorFactory[] { new LongSumAggregatorFactory("sumCount", "sumCount") }, tmpDirMerged, indexSpec, null, -1)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
    Assert.assertEquals(ImmutableList.of("dimA", "dimMultiVal"), ImmutableList.copyOf(adapter.getDimensionNames()));
    Assert.assertEquals(3, rowList.size());
    Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "2", "3", "5")), rowList.get(0).dimensionValues());
    Assert.assertEquals(1L, rowList.get(0).metricValues().get(0));
    Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "2", "4")), rowList.get(1).dimensionValues());
    Assert.assertEquals(2L, rowList.get(1).metricValues().get(0));
    Assert.assertEquals(Arrays.asList("potato", Arrays.asList("0", "1", "4")), rowList.get(2).dimensionValues());
    Assert.assertEquals(1L, rowList.get(2).metricValues().get(0));
    checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dimA", "leek"));
    checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimA", "potato"));
    checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("dimMultiVal", "0"));
    checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimMultiVal", "1"));
    checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("dimMultiVal", "2"));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimMultiVal", "3"));
    checkBitmapIndex(Arrays.asList(1, 2), adapter.getBitmapIndex("dimMultiVal", "4"));
    checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("dimMultiVal", "5"));
}