Examples with MapBasedInputRow org.apache.druid.data.input.MapBasedInputRow used on opensource projects

Example 36 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexIOTest method setUp.

@Before
public void setUp() throws IndexSizeExceededException {
    long timestamp = 0L;
    for (Map<String, Object> event : events1) {
        incrementalIndex1.add(new MapBasedInputRow(timestamp++, Lists.newArrayList(event.keySet()), event));
    }
    timestamp = 0L;
    for (Map<String, Object> event : events2) {
        incrementalIndex2.add(new MapBasedInputRow(timestamp++, Lists.newArrayList(event.keySet()), event));
    }
    adapter2 = new IncrementalIndexAdapter(DEFAULT_INTERVAL, incrementalIndex2, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory());
    adapter1 = new IncrementalIndexAdapter(DEFAULT_INTERVAL, incrementalIndex1, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory());
}

Example 37 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTestBase method testMultivalDim_persistAndMerge_dimensionValueOrderingRules.

@Test
public void testMultivalDim_persistAndMerge_dimensionValueOrderingRules() throws Exception {
    List<String> dims = Arrays.asList("dimA", "dimMultiVal");
    IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder().withDimensionsSpec(new DimensionsSpec(ImmutableList.of(new StringDimensionSchema("dimA", MultiValueHandling.SORTED_ARRAY, true), new StringDimensionSchema("dimMultiVal", MultiValueHandling.SORTED_ARRAY, true)))).withMetrics(new LongSumAggregatorFactory("sumCount", "sumCount")).withRollup(true).build();
    Map<String, Object> nullEvent = new HashMap<>();
    nullEvent.put("dimA", "leek");
    nullEvent.put("sumCount", 1L);
    Map<String, Object> nullEvent2 = new HashMap<>();
    nullEvent2.put("dimA", "leek");
    nullEvent2.put("dimMultiVal", null);
    nullEvent2.put("sumCount", 1L);
    Map<String, Object> emptyListEvent = new HashMap<>();
    emptyListEvent.put("dimA", "leek");
    emptyListEvent.put("dimMultiVal", ImmutableList.of());
    emptyListEvent.put("sumCount", 1L);
    List<String> listWithNull = new ArrayList<>();
    listWithNull.add(null);
    Map<String, Object> listWithNullEvent = new HashMap<>();
    listWithNullEvent.put("dimA", "leek");
    listWithNullEvent.put("dimMultiVal", listWithNull);
    listWithNullEvent.put("sumCount", 1L);
    Map<String, Object> emptyStringEvent = new HashMap<>();
    emptyStringEvent.put("dimA", "leek");
    emptyStringEvent.put("dimMultiVal", "");
    emptyStringEvent.put("sumCount", 1L);
    Map<String, Object> listWithEmptyStringEvent = new HashMap<>();
    listWithEmptyStringEvent.put("dimA", "leek");
    listWithEmptyStringEvent.put("dimMultiVal", ImmutableList.of(""));
    listWithEmptyStringEvent.put("sumCount", 1L);
    Map<String, Object> singleValEvent = new HashMap<>();
    singleValEvent.put("dimA", "leek");
    singleValEvent.put("dimMultiVal", "1");
    singleValEvent.put("sumCount", 1L);
    Map<String, Object> singleValEvent2 = new HashMap<>();
    singleValEvent2.put("dimA", "leek");
    singleValEvent2.put("dimMultiVal", "2");
    singleValEvent2.put("sumCount", 1L);
    Map<String, Object> singleValEvent3 = new HashMap<>();
    singleValEvent3.put("dimA", "potato");
    singleValEvent3.put("dimMultiVal", "2");
    singleValEvent3.put("sumCount", 1L);
    Map<String, Object> listWithSingleValEvent = new HashMap<>();
    listWithSingleValEvent.put("dimA", "leek");
    listWithSingleValEvent.put("dimMultiVal", ImmutableList.of("1"));
    listWithSingleValEvent.put("sumCount", 1L);
    Map<String, Object> listWithSingleValEvent2 = new HashMap<>();
    listWithSingleValEvent2.put("dimA", "leek");
    listWithSingleValEvent2.put("dimMultiVal", ImmutableList.of("2"));
    listWithSingleValEvent2.put("sumCount", 1L);
    Map<String, Object> listWithSingleValEvent3 = new HashMap<>();
    listWithSingleValEvent3.put("dimA", "potato");
    listWithSingleValEvent3.put("dimMultiVal", ImmutableList.of("2"));
    listWithSingleValEvent3.put("sumCount", 1L);
    Map<String, Object> multivalEvent = new HashMap<>();
    multivalEvent.put("dimA", "leek");
    multivalEvent.put("dimMultiVal", ImmutableList.of("1", "3"));
    multivalEvent.put("sumCount", 1L);
    Map<String, Object> multivalEvent2 = new HashMap<>();
    multivalEvent2.put("dimA", "leek");
    multivalEvent2.put("dimMultiVal", ImmutableList.of("1", "4"));
    multivalEvent2.put("sumCount", 1L);
    Map<String, Object> multivalEvent3 = new HashMap<>();
    multivalEvent3.put("dimA", "leek");
    multivalEvent3.put("dimMultiVal", ImmutableList.of("1", "3", "5"));
    multivalEvent3.put("sumCount", 1L);
    Map<String, Object> multivalEvent4 = new HashMap<>();
    multivalEvent4.put("dimA", "leek");
    multivalEvent4.put("dimMultiVal", ImmutableList.of("1", "2", "3"));
    multivalEvent4.put("sumCount", 1L);
    List<String> multivalEvent5List = Arrays.asList("1", "2", "3", null);
    Map<String, Object> multivalEvent5 = new HashMap<>();
    multivalEvent5.put("dimA", "leek");
    multivalEvent5.put("dimMultiVal", multivalEvent5List);
    multivalEvent5.put("sumCount", 1L);
    List<String> multivalEvent6List = Arrays.asList(null, "3");
    Map<String, Object> multivalEvent6 = new HashMap<>();
    multivalEvent6.put("dimA", "leek");
    multivalEvent6.put("dimMultiVal", multivalEvent6List);
    multivalEvent6.put("sumCount", 1L);
    Map<String, Object> multivalEvent7 = new HashMap<>();
    multivalEvent7.put("dimA", "leek");
    multivalEvent7.put("dimMultiVal", ImmutableList.of("1", "2", "3", ""));
    multivalEvent7.put("sumCount", 1L);
    Map<String, Object> multivalEvent8 = new HashMap<>();
    multivalEvent8.put("dimA", "leek");
    multivalEvent8.put("dimMultiVal", ImmutableList.of("", "3"));
    multivalEvent8.put("sumCount", 1L);
    Map<String, Object> multivalEvent9 = new HashMap<>();
    multivalEvent9.put("dimA", "potato");
    multivalEvent9.put("dimMultiVal", ImmutableList.of("1", "3"));
    multivalEvent9.put("sumCount", 1L);
    List<Map<String, Object>> events = ImmutableList.of(nullEvent, nullEvent2, emptyListEvent, listWithNullEvent, emptyStringEvent, listWithEmptyStringEvent, singleValEvent, singleValEvent2, singleValEvent3, listWithSingleValEvent, listWithSingleValEvent2, listWithSingleValEvent3, multivalEvent, multivalEvent2, multivalEvent3, multivalEvent4, multivalEvent5, multivalEvent6, multivalEvent7, multivalEvent8, multivalEvent9);
    IncrementalIndex toPersistA = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
    for (Map<String, Object> event : events) {
        toPersistA.add(new MapBasedInputRow(1, dims, event));
    }
    final File tmpDirA = temporaryFolder.newFolder();
    QueryableIndex indexA = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersistA, tmpDirA, indexSpec, null)));
    List<QueryableIndex> singleEventIndexes = new ArrayList<>();
    for (Map<String, Object> event : events) {
        IncrementalIndex toPersist = new OnheapIncrementalIndex.Builder().setIndexSchema(indexSchema).setMaxRowCount(1000).build();
        toPersist.add(new MapBasedInputRow(1, dims, event));
        final File tmpDir = temporaryFolder.newFolder();
        QueryableIndex queryableIndex = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist, tmpDir, indexSpec, null)));
        singleEventIndexes.add(queryableIndex);
    }
    singleEventIndexes.add(indexA);
    final File tmpDirMerged = temporaryFolder.newFolder();
    final QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(singleEventIndexes, true, new AggregatorFactory[] { new LongSumAggregatorFactory("sumCount", "sumCount") }, tmpDirMerged, indexSpec, null, -1)));
    final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged);
    final List<DebugRow> rowList = RowIteratorHelper.toList(adapter.getRows());
    Assert.assertEquals(ImmutableList.of("dimA", "dimMultiVal"), ImmutableList.copyOf(adapter.getDimensionNames()));
    if (NullHandling.replaceWithDefault()) {
        Assert.assertEquals(11, rowList.size());
        Assert.assertEquals(Arrays.asList("leek", null), rowList.get(0).dimensionValues());
        Assert.assertEquals(12L, rowList.get(0).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList(null, "1", "2", "3")), rowList.get(1).dimensionValues());
        Assert.assertEquals(4L, rowList.get(1).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList(null, "3")), rowList.get(2).dimensionValues());
        Assert.assertEquals(4L, rowList.get(2).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", "1"), rowList.get(3).dimensionValues());
        Assert.assertEquals(4L, rowList.get(3).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "2", "3")), rowList.get(4).dimensionValues());
        Assert.assertEquals(2L, rowList.get(4).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "3")), rowList.get(5).dimensionValues());
        Assert.assertEquals(2L, rowList.get(5).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "3", "5")), rowList.get(6).dimensionValues());
        Assert.assertEquals(2L, rowList.get(6).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "4")), rowList.get(7).dimensionValues());
        Assert.assertEquals(2L, rowList.get(7).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", "2"), rowList.get(8).dimensionValues());
        Assert.assertEquals(4L, rowList.get(8).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("potato", Arrays.asList("1", "3")), rowList.get(9).dimensionValues());
        Assert.assertEquals(2L, rowList.get(9).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("potato", "2"), rowList.get(10).dimensionValues());
        Assert.assertEquals(4L, rowList.get(10).metricValues().get(0));
        checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8), adapter.getBitmapIndex("dimA", "leek"));
        checkBitmapIndex(Arrays.asList(9, 10), adapter.getBitmapIndex("dimA", "potato"));
        checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimMultiVal", null));
        checkBitmapIndex(ImmutableList.of(), adapter.getBitmapIndex("dimMultiVal", ""));
        checkBitmapIndex(Arrays.asList(1, 3, 4, 5, 6, 7, 9), adapter.getBitmapIndex("dimMultiVal", "1"));
        checkBitmapIndex(Arrays.asList(1, 4, 8, 10), adapter.getBitmapIndex("dimMultiVal", "2"));
        checkBitmapIndex(Arrays.asList(1, 2, 4, 5, 6, 9), adapter.getBitmapIndex("dimMultiVal", "3"));
        checkBitmapIndex(Collections.singletonList(7), adapter.getBitmapIndex("dimMultiVal", "4"));
        checkBitmapIndex(Collections.singletonList(6), adapter.getBitmapIndex("dimMultiVal", "5"));
    } else {
        Assert.assertEquals(14, rowList.size());
        Assert.assertEquals(Arrays.asList("leek", null), rowList.get(0).dimensionValues());
        Assert.assertEquals(8L, rowList.get(0).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList(null, "1", "2", "3")), rowList.get(1).dimensionValues());
        Assert.assertEquals(2L, rowList.get(1).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList(null, "3")), rowList.get(2).dimensionValues());
        Assert.assertEquals(2L, rowList.get(2).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", ""), rowList.get(3).dimensionValues());
        Assert.assertEquals(4L, rowList.get(3).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("", "1", "2", "3")), rowList.get(4).dimensionValues());
        Assert.assertEquals(2L, rowList.get(4).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("", "3")), rowList.get(5).dimensionValues());
        Assert.assertEquals(2L, rowList.get(5).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", "1"), rowList.get(6).dimensionValues());
        Assert.assertEquals(4L, rowList.get(6).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "2", "3")), rowList.get(7).dimensionValues());
        Assert.assertEquals(2L, rowList.get(7).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "3")), rowList.get(8).dimensionValues());
        Assert.assertEquals(2L, rowList.get(8).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "3", "5")), rowList.get(9).dimensionValues());
        Assert.assertEquals(2L, rowList.get(9).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", Arrays.asList("1", "4")), rowList.get(10).dimensionValues());
        Assert.assertEquals(2L, rowList.get(10).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("leek", "2"), rowList.get(11).dimensionValues());
        Assert.assertEquals(4L, rowList.get(11).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("potato", Arrays.asList("1", "3")), rowList.get(12).dimensionValues());
        Assert.assertEquals(2L, rowList.get(12).metricValues().get(0));
        Assert.assertEquals(Arrays.asList("potato", "2"), rowList.get(13).dimensionValues());
        Assert.assertEquals(4L, rowList.get(13).metricValues().get(0));
        checkBitmapIndex(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), adapter.getBitmapIndex("dimA", "leek"));
        checkBitmapIndex(Arrays.asList(12, 13), adapter.getBitmapIndex("dimA", "potato"));
        checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("dimMultiVal", null));
        checkBitmapIndex(ImmutableList.of(3, 4, 5), adapter.getBitmapIndex("dimMultiVal", ""));
        checkBitmapIndex(Arrays.asList(1, 4, 6, 7, 8, 9, 10, 12), adapter.getBitmapIndex("dimMultiVal", "1"));
        checkBitmapIndex(Arrays.asList(1, 4, 7, 11, 13), adapter.getBitmapIndex("dimMultiVal", "2"));
        checkBitmapIndex(Arrays.asList(1, 2, 4, 5, 7, 8, 9, 12), adapter.getBitmapIndex("dimMultiVal", "3"));
        checkBitmapIndex(Collections.singletonList(10), adapter.getBitmapIndex("dimMultiVal", "4"));
        checkBitmapIndex(Collections.singletonList(9), adapter.getBitmapIndex("dimMultiVal", "5"));
    }
}

Example 38 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class InputRowSerde method fromBytes.

public static InputRow fromBytes(final Map<String, IndexSerdeTypeHelper> typeHelperMap, byte[] data, AggregatorFactory[] aggs) {
    try {
        ByteArrayDataInput in = ByteStreams.newDataInput(data);
        // Read timestamp
        long timestamp = in.readLong();
        Map<String, Object> event = new HashMap<>();
        // Read dimensions
        List<String> dimensions = new ArrayList<>();
        int dimNum = WritableUtils.readVInt(in);
        for (int i = 0; i < dimNum; i++) {
            String dimension = readString(in);
            dimensions.add(dimension);
            IndexSerdeTypeHelper typeHelper = typeHelperMap.get(dimension);
            if (typeHelper == null) {
                typeHelper = STRING_HELPER;
            }
            Object dimValues = typeHelper.deserialize(in);
            if (dimValues == null) {
                continue;
            }
            if (typeHelper.getType() == ValueType.STRING) {
                List<String> dimensionValues = (List<String>) dimValues;
                if (dimensionValues.size() == 1) {
                    event.put(dimension, dimensionValues.get(0));
                } else {
                    event.put(dimension, dimensionValues);
                }
            } else {
                event.put(dimension, dimValues);
            }
        }
        // Read metrics
        int metricSize = WritableUtils.readVInt(in);
        for (int i = 0; i < metricSize; i++) {
            final String metric = readString(in);
            final AggregatorFactory agg = getAggregator(metric, aggs, i);
            final ColumnType type = agg.getIntermediateType();
            final byte metricNullability = in.readByte();
            if (metricNullability == NullHandling.IS_NULL_BYTE) {
                // metric value is null.
                continue;
            }
            if (type.is(ValueType.FLOAT)) {
                event.put(metric, in.readFloat());
            } else if (type.is(ValueType.LONG)) {
                event.put(metric, WritableUtils.readVLong(in));
            } else if (type.is(ValueType.DOUBLE)) {
                event.put(metric, in.readDouble());
            } else {
                ComplexMetricSerde serde = getComplexMetricSerde(agg.getIntermediateType().getComplexTypeName());
                byte[] value = readBytes(in);
                event.put(metric, serde.fromBytes(value, 0, value.length));
            }
        }
        return new MapBasedInputRow(timestamp, dimensions, event);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}

Example 39 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexGeneratorCombinerTest method testMultipleRowsMerged.

@Test
public void testMultipleRowsMerged() throws Exception {
    long timestamp = System.currentTimeMillis();
    Bucket bucket = new Bucket(0, DateTimes.utc(timestamp), 0);
    SortableBytes keySortableBytes = new SortableBytes(bucket.toGroupKey(), new byte[0]);
    BytesWritable key = keySortableBytes.toBytesWritable();
    DimensionsSpec dimensionsSpec = new DimensionsSpec(Arrays.asList(new StringDimensionSchema("host"), new StringDimensionSchema("keywords")));
    Map<String, InputRowSerde.IndexSerdeTypeHelper> typeHelperMap = InputRowSerde.getTypeHelperMap(dimensionsSpec);
    InputRow row1 = new MapBasedInputRow(timestamp, ImmutableList.of("keywords"), ImmutableMap.of("host", "host1", "keywords", Arrays.asList("foo", "bar"), "visited", 10));
    InputRow row2 = new MapBasedInputRow(timestamp, ImmutableList.of("keywords"), ImmutableMap.of("host", "host2", "keywords", Arrays.asList("foo", "bar"), "visited", 5));
    List<BytesWritable> rows = Lists.newArrayList(new BytesWritable(InputRowSerde.toBytes(typeHelperMap, row1, aggregators).getSerializedRow()), new BytesWritable(InputRowSerde.toBytes(typeHelperMap, row2, aggregators).getSerializedRow()));
    Reducer.Context context = EasyMock.createNiceMock(Reducer.Context.class);
    Capture<BytesWritable> captureKey = Capture.newInstance();
    Capture<BytesWritable> captureVal = Capture.newInstance();
    context.write(EasyMock.capture(captureKey), EasyMock.capture(captureVal));
    EasyMock.replay(context);
    combiner.reduce(key, rows, context);
    EasyMock.verify(context);
    Assert.assertTrue(captureKey.getValue() == key);
    InputRow capturedRow = InputRowSerde.fromBytes(typeHelperMap, captureVal.getValue().getBytes(), aggregators);
    Assert.assertEquals(Arrays.asList("host", "keywords"), capturedRow.getDimensions());
    Assert.assertEquals(ImmutableList.of(), capturedRow.getDimension("host"));
    Assert.assertEquals(Arrays.asList("bar", "foo"), capturedRow.getDimension("keywords"));
    Assert.assertEquals(15, capturedRow.getMetric("visited_sum").longValue());
    Assert.assertEquals(2.0, (Double) HyperUniquesAggregatorFactory.estimateCardinality(capturedRow.getRaw("unique_hosts"), false), 0.001);
}

Example 40 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class InputRowSerdeTest method testDimensionNullOrDefaultForNumerics.

@Test
public void testDimensionNullOrDefaultForNumerics() {
    HashMap<String, Object> eventWithNulls = new HashMap<>();
    eventWithNulls.put("d1", null);
    eventWithNulls.put("d2", Arrays.asList("d2v1", "d2v2"));
    eventWithNulls.put("d3", null);
    eventWithNulls.put("d4", null);
    eventWithNulls.put("d5", null);
    InputRow in = new MapBasedInputRow(timestamp, dims, eventWithNulls);
    DimensionsSpec dimensionsSpec = new DimensionsSpec(Arrays.asList(new StringDimensionSchema("d1"), new StringDimensionSchema("d2"), new LongDimensionSchema("d3"), new FloatDimensionSchema("d4"), new DoubleDimensionSchema("d5")));
    byte[] result = InputRowSerde.toBytes(InputRowSerde.getTypeHelperMap(dimensionsSpec), in, new AggregatorFactory[0]).getSerializedRow();
    if (NullHandling.replaceWithDefault()) {
        long expected = 0;
        // timestamp bytes + dims length
        expected += 9;
        // dim_non_existing writes: 1 16 1 bytes
        expected += 18;
        // d1: writes 1 2 1 bytes
        expected += 4;
        // d2: writes 1 2 1 1 4 1 4 bytes
        expected += 14;
        // d3: writes 1 2 8 bytes
        expected += 11;
        // d4: writes 1 2 4 bytes
        expected += 7;
        // d5: writes 1 2 8 bytes
        expected += 11;
        // writes aggregator length
        expected += 1;
        Assert.assertEquals(expected, result.length);
        Assert.assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, Arrays.copyOfRange(result, 48, 56));
        Assert.assertArrayEquals(new byte[] { 0, 0, 0, 0 }, Arrays.copyOfRange(result, 59, 63));
        Assert.assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, Arrays.copyOfRange(result, 66, 74));
    } else {
        long expected = 9 + 18 + 4 + 14 + 4 + 4 + 4 + 1;
        Assert.assertEquals(expected, result.length);
        Assert.assertEquals(result[48], NullHandling.IS_NULL_BYTE);
        Assert.assertEquals(result[52], NullHandling.IS_NULL_BYTE);
        Assert.assertEquals(result[56], NullHandling.IS_NULL_BYTE);
    }
}