Examples with MapBasedInputRow org.apache.druid.data.input.MapBasedInputRow used on opensource projects

Example 96 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class SchemalessIndexTest method makeIncrementalIndex.

public static IncrementalIndex makeIncrementalIndex(final String resourceFilename, AggregatorFactory[] aggs) {
    URL resource = TestIndex.class.getClassLoader().getResource(resourceFilename);
    log.info("Realtime loading resource[%s]", resource);
    String filename = resource.getFile();
    log.info("Realtime loading index file[%s]", filename);
    final IncrementalIndex retVal = new OnheapIncrementalIndex.Builder().setIndexSchema(new IncrementalIndexSchema.Builder().withMinTimestamp(DateTimes.of("2011-01-12T00:00:00.000Z").getMillis()).withQueryGranularity(Granularities.MINUTE).withMetrics(aggs).build()).setMaxRowCount(1000).build();
    try {
        final List<Object> events = JSON_MAPPER.readValue(new File(filename), List.class);
        for (Object obj : events) {
            final Map<String, Object> event = JSON_MAPPER.convertValue(obj, Map.class);
            final List<String> dims = new ArrayList<>();
            for (Map.Entry<String, Object> entry : event.entrySet()) {
                if (!entry.getKey().equalsIgnoreCase(TIMESTAMP) && !METRICS.contains(entry.getKey())) {
                    dims.add(entry.getKey());
                }
            }
            retVal.add(new MapBasedInputRow(new DateTime(event.get(TIMESTAMP), ISOChronology.getInstanceUTC()).getMillis(), dims, event));
        }
    } catch (IOException e) {
        index = null;
        throw new RuntimeException(e);
    }
    return retVal;
}

Example 97 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTestBase method getIndexD3.

private IncrementalIndex getIndexD3() throws Exception {
    IncrementalIndex toPersist1 = new OnheapIncrementalIndex.Builder().setSimpleTestingIndexSchema(new CountAggregatorFactory("count")).setMaxRowCount(1000).build();
    toPersist1.add(new MapBasedInputRow(1, Arrays.asList("d3", "d1", "d2"), ImmutableMap.of("d1", "100", "d2", "4000", "d3", "30000")));
    toPersist1.add(new MapBasedInputRow(1, Arrays.asList("d3", "d1", "d2"), ImmutableMap.of("d1", "300", "d2", "2000", "d3", "40000")));
    toPersist1.add(new MapBasedInputRow(1, Arrays.asList("d3", "d1", "d2"), ImmutableMap.of("d1", "200", "d2", "3000", "d3", "50000")));
    return toPersist1;
}

Example 98 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTestBase method testPersistEmptyColumn.

@Test
public void testPersistEmptyColumn() throws Exception {
    final IncrementalIndex toPersist1 = new OnheapIncrementalIndex.Builder().setSimpleTestingIndexSchema().setMaxRowCount(10).build();
    final IncrementalIndex toPersist2 = new OnheapIncrementalIndex.Builder().setSimpleTestingIndexSchema().setMaxRowCount(10).build();
    final File tmpDir1 = temporaryFolder.newFolder();
    final File tmpDir2 = temporaryFolder.newFolder();
    final File tmpDir3 = temporaryFolder.newFolder();
    toPersist1.add(new MapBasedInputRow(1L, ImmutableList.of("dim1", "dim2"), ImmutableMap.of("dim1", ImmutableList.of(), "dim2", "foo")));
    toPersist2.add(new MapBasedInputRow(1L, ImmutableList.of("dim1", "dim2"), ImmutableMap.of("dim1", ImmutableList.of(), "dim2", "bar")));
    final QueryableIndex index1 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist1, tmpDir1, indexSpec, null)));
    final QueryableIndex index2 = closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist2, tmpDir2, indexSpec, null)));
    final QueryableIndex merged = closer.closeLater(indexIO.loadIndex(indexMerger.mergeQueryableIndex(Arrays.asList(index1, index2), true, new AggregatorFactory[] {}, tmpDir3, indexSpec, null, -1)));
    Assert.assertEquals(1, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(index1.getAvailableDimensions()));
    Assert.assertEquals(1, index2.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(index2.getAvailableDimensions()));
    Assert.assertEquals(2, merged.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength());
    Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(merged.getAvailableDimensions()));
    assertDimCompression(index1, indexSpec.getDimensionCompression());
    assertDimCompression(index2, indexSpec.getDimensionCompression());
    assertDimCompression(merged, indexSpec.getDimensionCompression());
}

Example 99 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTestBase method testPersistNullColumnSkipping.

@Test
public void testPersistNullColumnSkipping() throws Exception {
    // check that column d2 is skipped because it only has null values
    IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
    index1.add(new MapBasedInputRow(1L, Arrays.asList("d1", "d2"), ImmutableMap.of("d1", "a", "A", 1)));
    index1.add(new MapBasedInputRow(1L, Arrays.asList("d1", "d2"), ImmutableMap.of("d1", "b", "A", 1)));
    final File tempDir = temporaryFolder.newFolder();
    QueryableIndex index = closer.closeLater(indexIO.loadIndex(indexMerger.persist(index1, tempDir, indexSpec, null)));
    List<String> expectedColumnNames = Arrays.asList("A", "d1");
    List<String> actualColumnNames = Lists.newArrayList(index.getColumnNames());
    Collections.sort(expectedColumnNames);
    Collections.sort(actualColumnNames);
    Assert.assertEquals(expectedColumnNames, actualColumnNames);
    SmooshedFileMapper sfm = closer.closeLater(SmooshedFileMapper.load(tempDir));
    List<String> expectedFilenames = Arrays.asList("A", "__time", "d1", "index.drd", "metadata.drd");
    List<String> actualFilenames = new ArrayList<>(sfm.getInternalFilenames());
    Collections.sort(expectedFilenames);
    Collections.sort(actualFilenames);
    Assert.assertEquals(expectedFilenames, actualFilenames);
}

Example 100 with MapBasedInputRow

use of org.apache.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTestBase method testAddMetricsBothSidesNull.

@Test
public void testAddMetricsBothSidesNull() throws IOException {
    IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
    closer.closeLater(index1);
    long timestamp = System.currentTimeMillis();
    index1.add(new MapBasedInputRow(timestamp, Arrays.asList("dim1", "dim2"), ImmutableMap.of("dim1", "1", "dim2", "2", "A", 5)));
    IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
    index2.add(new MapBasedInputRow(timestamp, Arrays.asList("dim1", "dim2"), ImmutableMap.of("dim1", "1", "dim2", "2", "A", 5, "C", 6)));
    closer.closeLater(index2);
    IncrementalIndex index3 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
    index3.add(new MapBasedInputRow(timestamp, Arrays.asList("dim1", "dim2"), ImmutableMap.of("dim1", "1", "dim2", "2", "A", 5)));
    Interval interval = new Interval(DateTimes.EPOCH, DateTimes.nowUtc());
    RoaringBitmapFactory factory = new RoaringBitmapFactory();
    List<IndexableAdapter> toMerge = Arrays.asList(new IncrementalIndexAdapter(interval, index1, factory), new IncrementalIndexAdapter(interval, index2, factory), new IncrementalIndexAdapter(interval, index3, factory));
    final File tmpDirMerged = temporaryFolder.newFolder();
    File merged = indexMerger.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") }, tmpDirMerged, indexSpec, -1);
    final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(indexIO.loadIndex(merged)));
    Assert.assertEquals(ImmutableSet.of("A", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
}