Examples with MapBasedInputRow io.druid.data.input.MapBasedInputRow used on opensource projects

Example 46 with MapBasedInputRow

use of io.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class SchemalessIndexTest method makeIncrementalIndex.

private static IncrementalIndex makeIncrementalIndex(final String resourceFilename, AggregatorFactory[] aggs) {
    URL resource = TestIndex.class.getClassLoader().getResource(resourceFilename);
    log.info("Realtime loading resource[%s]", resource);
    String filename = resource.getFile();
    log.info("Realtime loading index file[%s]", filename);
    final IncrementalIndex retVal = new OnheapIncrementalIndex(new DateTime("2011-01-12T00:00:00.000Z").getMillis(), Granularities.MINUTE, aggs, 1000);
    try {
        final List<Object> events = jsonMapper.readValue(new File(filename), List.class);
        for (Object obj : events) {
            final Map<String, Object> event = jsonMapper.convertValue(obj, Map.class);
            final List<String> dims = Lists.newArrayList();
            for (Map.Entry<String, Object> entry : event.entrySet()) {
                if (!entry.getKey().equalsIgnoreCase(TIMESTAMP) && !METRICS.contains(entry.getKey())) {
                    dims.add(entry.getKey());
                }
            }
            retVal.add(new MapBasedInputRow(new DateTime(event.get(TIMESTAMP)).getMillis(), dims, event));
        }
    } catch (IOException e) {
        index = null;
        throw Throwables.propagate(e);
    }
    return retVal;
}

Example 47 with MapBasedInputRow

use of io.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class SchemalessIndexTest method makeRowPersistedIndexes.

private static void makeRowPersistedIndexes() {
    synchronized (log) {
        try {
            if (events.isEmpty()) {
                makeEvents();
            }
            for (final Map<String, Object> event : events) {
                final long timestamp = new DateTime(event.get(TIMESTAMP)).getMillis();
                final List<String> dims = Lists.newArrayList();
                for (Map.Entry<String, Object> entry : event.entrySet()) {
                    if (!entry.getKey().equalsIgnoreCase(TIMESTAMP) && !METRICS.contains(entry.getKey())) {
                        dims.add(entry.getKey());
                    }
                }
                final IncrementalIndex rowIndex = new OnheapIncrementalIndex(timestamp, Granularities.MINUTE, METRIC_AGGS, 1000);
                rowIndex.add(new MapBasedInputRow(timestamp, dims, event));
                File tmpFile = File.createTempFile("billy", "yay");
                tmpFile.delete();
                tmpFile.mkdirs();
                tmpFile.deleteOnExit();
                INDEX_MERGER.persist(rowIndex, tmpFile, indexSpec);
                rowPersistedIndexes.add(INDEX_IO.loadIndex(tmpFile));
            }
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }
}

Example 48 with MapBasedInputRow

use of io.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTest method testPersistNullColumnSkipping.

@Test
public void testPersistNullColumnSkipping() throws Exception {
    //check that column d2 is skipped because it only has null values
    IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
    index1.add(new MapBasedInputRow(1L, Lists.newArrayList("d1", "d2"), ImmutableMap.<String, Object>of("d1", "a", "d2", "", "A", 1)));
    index1.add(new MapBasedInputRow(1L, Lists.newArrayList("d1", "d2"), ImmutableMap.<String, Object>of("d1", "b", "d2", "", "A", 1)));
    final File tempDir = temporaryFolder.newFolder();
    QueryableIndex index = closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(index1, tempDir, indexSpec)));
    List<String> expectedColumnNames = Arrays.asList("A", "d1");
    List<String> actualColumnNames = Lists.newArrayList(index.getColumnNames());
    Collections.sort(expectedColumnNames);
    Collections.sort(actualColumnNames);
    Assert.assertEquals(expectedColumnNames, actualColumnNames);
    SmooshedFileMapper sfm = closer.closeLater(SmooshedFileMapper.load(tempDir));
    List<String> expectedFilenames = Arrays.asList("A", "__time", "d1", "index.drd", "metadata.drd");
    List<String> actualFilenames = new ArrayList<>(sfm.getInternalFilenames());
    Collections.sort(expectedFilenames);
    Collections.sort(actualFilenames);
    Assert.assertEquals(expectedFilenames, actualFilenames);
}

Example 49 with MapBasedInputRow

use of io.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTest method testMismatchedDimensions.

@Test
public void testMismatchedDimensions() throws IOException, IndexSizeExceededException {
    IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
    index1.add(new MapBasedInputRow(1L, Lists.newArrayList("d1", "d2"), ImmutableMap.<String, Object>of("d1", "a", "d2", "z", "A", 1)));
    closer.closeLater(index1);
    IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
    index2.add(new MapBasedInputRow(1L, Lists.newArrayList("d1", "d2"), ImmutableMap.<String, Object>of("d1", "a", "d2", "z", "A", 2, "C", 100)));
    closer.closeLater(index2);
    Interval interval = new Interval(0, new DateTime().getMillis());
    RoaringBitmapFactory factory = new RoaringBitmapFactory();
    ArrayList<IndexableAdapter> toMerge = Lists.<IndexableAdapter>newArrayList(new IncrementalIndexAdapter(interval, index1, factory), new IncrementalIndexAdapter(interval, index2, factory));
    final File tmpDirMerged = temporaryFolder.newFolder();
    INDEX_MERGER.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") }, tmpDirMerged, indexSpec);
}

Example 50 with MapBasedInputRow

use of io.druid.data.input.MapBasedInputRow in project druid by druid-io.

the class IndexMergerTest method getIndexD3.

private IncrementalIndex getIndexD3() throws Exception {
    IncrementalIndex toPersist1 = new OnheapIncrementalIndex(0L, Granularities.NONE, new AggregatorFactory[] { new CountAggregatorFactory("count") }, 1000);
    toPersist1.add(new MapBasedInputRow(1, Arrays.asList("d3", "d1", "d2"), ImmutableMap.<String, Object>of("d1", "100", "d2", "4000", "d3", "30000")));
    toPersist1.add(new MapBasedInputRow(1, Arrays.asList("d3", "d1", "d2"), ImmutableMap.<String, Object>of("d1", "300", "d2", "2000", "d3", "40000")));
    toPersist1.add(new MapBasedInputRow(1, Arrays.asList("d3", "d1", "d2"), ImmutableMap.<String, Object>of("d1", "200", "d2", "3000", "d3", "50000")));
    return toPersist1;
}