Examples with MapBasedInputRow org.apache.druid.data.input.MapBasedInputRow used on opensource projects

Example 21 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class SinkTest method testDedup.

@Test
public void testDedup() throws Exception {
    final DataSchema schema = new DataSchema("test", new TimestampSpec(null, null, null), DimensionsSpec.EMPTY, new AggregatorFactory[] { new CountAggregatorFactory("rows") }, new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), null);
    final Interval interval = Intervals.of("2013-01-01/2013-01-02");
    final String version = DateTimes.nowUtc().toString();
    RealtimeTuningConfig tuningConfig = new RealtimeTuningConfig(null, 100, null, null, new Period("P1Y"), null, null, null, null, null, null, null, null, 0, 0, null, null, null, null, "dedupColumn");
    final Sink sink = new Sink(interval, schema, tuningConfig.getShardSpec(), version, tuningConfig.getAppendableIndexSpec(), tuningConfig.getMaxRowsInMemory(), tuningConfig.getMaxBytesInMemoryOrDefault(), true, tuningConfig.getDedupColumn());
    int rows = sink.add(new MapBasedInputRow(DateTimes.of("2013-01-01"), ImmutableList.of("field", "dedupColumn"), ImmutableMap.of("field1", "value1", "dedupColumn", "v1")), false).getRowCount();
    Assert.assertTrue(rows > 0);
    // dedupColumn is null
    rows = sink.add(new MapBasedInputRow(DateTimes.of("2013-01-01"), ImmutableList.of("field", "dedupColumn"), ImmutableMap.of("field1", "value2")), false).getRowCount();
    Assert.assertTrue(rows > 0);
    // dedupColumn is null
    rows = sink.add(new MapBasedInputRow(DateTimes.of("2013-01-01"), ImmutableList.of("field", "dedupColumn"), ImmutableMap.of("field1", "value3")), false).getRowCount();
    Assert.assertTrue(rows > 0);
    rows = sink.add(new MapBasedInputRow(DateTimes.of("2013-01-01"), ImmutableList.of("field", "dedupColumn"), ImmutableMap.of("field1", "value4", "dedupColumn", "v2")), false).getRowCount();
    Assert.assertTrue(rows > 0);
    rows = sink.add(new MapBasedInputRow(DateTimes.of("2013-01-01"), ImmutableList.of("field", "dedupColumn"), ImmutableMap.of("field1", "value5", "dedupColumn", "v1")), false).getRowCount();
    Assert.assertTrue(rows == -2);
}

Example 22 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class StreamAppenderatorDriverTest method testMaxRowsPerSegment.

@Test
public void testMaxRowsPerSegment() throws Exception {
    final int numSegments = 3;
    final TestCommitterSupplier<Integer> committerSupplier = new TestCommitterSupplier<>();
    Assert.assertNull(driver.startJob(null));
    for (int i = 0; i < numSegments * MAX_ROWS_PER_SEGMENT; i++) {
        committerSupplier.setMetadata(i + 1);
        InputRow row = new MapBasedInputRow(DateTimes.of("2000T01"), ImmutableList.of("dim2"), ImmutableMap.of("dim2", StringUtils.format("bar-%d", i), "met1", 2.0));
        final AppenderatorDriverAddResult addResult = driver.add(row, "dummy", committerSupplier, false, true);
        Assert.assertTrue(addResult.isOk());
        if (addResult.getNumRowsInSegment() > MAX_ROWS_PER_SEGMENT) {
            driver.moveSegmentOut("dummy", ImmutableList.of(addResult.getSegmentIdentifier()));
        }
    }
    final SegmentsAndCommitMetadata published = driver.publish(makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy")).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    while (driver.getSegments().containsKey("dummy")) {
        Thread.sleep(100);
    }
    final SegmentsAndCommitMetadata segmentsAndCommitMetadata = driver.registerHandoff(published).get(HANDOFF_CONDITION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    Assert.assertEquals(numSegments, segmentsAndCommitMetadata.getSegments().size());
    Assert.assertEquals(numSegments * MAX_ROWS_PER_SEGMENT, segmentsAndCommitMetadata.getCommitMetadata());
}

Example 23 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class TransformerTest method testTransformWithStringTransformOnDoubleColumnTransformAfterCasting.

@Test
public void testTransformWithStringTransformOnDoubleColumnTransformAfterCasting() {
    final Transformer transformer = new Transformer(new TransformSpec(null, ImmutableList.of(new ExpressionTransform("dim", "strlen(dim)", TestExprMacroTable.INSTANCE))));
    final InputRow row = new MapBasedInputRow(DateTimes.nowUtc(), ImmutableList.of("dim"), ImmutableMap.of("dim", 200.5d));
    final InputRow actual = transformer.transform(row);
    Assert.assertNotNull(actual);
    Assert.assertEquals(ImmutableList.of("dim"), actual.getDimensions());
    Assert.assertEquals(5L, actual.getRaw("dim"));
    Assert.assertEquals(row.getTimestamp(), actual.getTimestamp());
}

Example 24 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class TransformerTest method testTransformWithStringTransformOnLongColumnTransformAfterCasting.

@Test
public void testTransformWithStringTransformOnLongColumnTransformAfterCasting() {
    final Transformer transformer = new Transformer(new TransformSpec(null, ImmutableList.of(new ExpressionTransform("dim", "strlen(dim)", TestExprMacroTable.INSTANCE))));
    final InputRow row = new MapBasedInputRow(DateTimes.nowUtc(), ImmutableList.of("dim"), ImmutableMap.of("dim", 10L));
    final InputRow actual = transformer.transform(row);
    Assert.assertNotNull(actual);
    Assert.assertEquals(ImmutableList.of("dim"), actual.getDimensions());
    Assert.assertEquals(2L, actual.getRaw("dim"));
    Assert.assertEquals(row.getTimestamp(), actual.getTimestamp());
}

Example 25 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class TransformerTest method testTransformWithSelectorFilterWithStringBooleanValueOnStringColumn.

@Test
public void testTransformWithSelectorFilterWithStringBooleanValueOnStringColumn() {
    final Transformer transformer = new Transformer(new TransformSpec(new SelectorDimFilter("dim", "false", null), null));
    final InputRow row = new MapBasedInputRow(DateTimes.nowUtc(), ImmutableList.of("dim"), ImmutableMap.of("dim", "false"));
    Assert.assertEquals(row, transformer.transform(row));
    final InputRow row2 = new MapBasedInputRow(DateTimes.nowUtc(), ImmutableList.of("dim"), ImmutableMap.of("dim", "true"));
    Assert.assertNull(transformer.transform(row2));
}