Examples with IncrementalIndex org.apache.druid.segment.incremental.IncrementalIndex used on opensource projects

Example 86 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class SchemalessIndexTest method makeFilesToMap.

private List<File> makeFilesToMap(File tmpFile, Iterable<Pair<String, AggregatorFactory[]>> files) throws IOException {
    List<File> filesToMap = new ArrayList<>();
    for (Pair<String, AggregatorFactory[]> file : files) {
        IncrementalIndex index = makeIncrementalIndex(file.lhs, file.rhs);
        File theFile = new File(tmpFile, file.lhs);
        FileUtils.mkdirp(theFile);
        theFile.deleteOnExit();
        filesToMap.add(theFile);
        indexMerger.persist(index, theFile, INDEX_SPEC, null);
    }
    return filesToMap;
}

Example 87 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class SchemalessIndexTest method makeIncrementalIndex.

public static IncrementalIndex makeIncrementalIndex(final String resourceFilename, AggregatorFactory[] aggs) {
    URL resource = TestIndex.class.getClassLoader().getResource(resourceFilename);
    log.info("Realtime loading resource[%s]", resource);
    String filename = resource.getFile();
    log.info("Realtime loading index file[%s]", filename);
    final IncrementalIndex retVal = new OnheapIncrementalIndex.Builder().setIndexSchema(new IncrementalIndexSchema.Builder().withMinTimestamp(DateTimes.of("2011-01-12T00:00:00.000Z").getMillis()).withQueryGranularity(Granularities.MINUTE).withMetrics(aggs).build()).setMaxRowCount(1000).build();
    try {
        final List<Object> events = JSON_MAPPER.readValue(new File(filename), List.class);
        for (Object obj : events) {
            final Map<String, Object> event = JSON_MAPPER.convertValue(obj, Map.class);
            final List<String> dims = new ArrayList<>();
            for (Map.Entry<String, Object> entry : event.entrySet()) {
                if (!entry.getKey().equalsIgnoreCase(TIMESTAMP) && !METRICS.contains(entry.getKey())) {
                    dims.add(entry.getKey());
                }
            }
            retVal.add(new MapBasedInputRow(new DateTime(event.get(TIMESTAMP), ISOChronology.getInstanceUTC()).getMillis(), dims, event));
        }
    } catch (IOException e) {
        index = null;
        throw new RuntimeException(e);
    }
    return retVal;
}

Example 88 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class SchemalessTestSimpleTest method constructorFeeder.

@Parameterized.Parameters
public static Collection<?> constructorFeeder() {
    List<Object[]> argumentArrays = new ArrayList<>();
    for (SegmentWriteOutMediumFactory segmentWriteOutMediumFactory : SegmentWriteOutMediumFactory.builtInFactories()) {
        SchemalessIndexTest schemalessIndexTest = new SchemalessIndexTest(segmentWriteOutMediumFactory);
        final IncrementalIndex incrementalIndex = SchemalessIndexTest.getIncrementalIndex();
        final QueryableIndex persistedIncrementalIndex = TestIndex.persistRealtimeAndLoadMMapped(incrementalIndex);
        final QueryableIndex mergedIncrementalIndex = schemalessIndexTest.getMergedIncrementalIndex();
        argumentArrays.add(new Object[] { new IncrementalIndexSegment(incrementalIndex, null), false });
        argumentArrays.add(new Object[] { new QueryableIndexSegment(persistedIncrementalIndex, null), false });
        argumentArrays.add(new Object[] { new QueryableIndexSegment(mergedIncrementalIndex, null), true });
    }
    return argumentArrays;
}

Example 89 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method getIndexD3.

private IncrementalIndex getIndexD3() throws Exception {
    IncrementalIndex toPersist1 = new OnheapIncrementalIndex.Builder().setSimpleTestingIndexSchema(new CountAggregatorFactory("count")).setMaxRowCount(1000).build();
    toPersist1.add(new MapBasedInputRow(1, Arrays.asList("d3", "d1", "d2"), ImmutableMap.of("d1", "100", "d2", "4000", "d3", "30000")));
    toPersist1.add(new MapBasedInputRow(1, Arrays.asList("d3", "d1", "d2"), ImmutableMap.of("d1", "300", "d2", "2000", "d3", "40000")));
    toPersist1.add(new MapBasedInputRow(1, Arrays.asList("d3", "d1", "d2"), ImmutableMap.of("d1", "200", "d2", "3000", "d3", "50000")));
    return toPersist1;
}

Example 90 with IncrementalIndex

use of org.apache.druid.segment.incremental.IncrementalIndex in project druid by druid-io.

the class IndexMergerTestBase method getSingleDimIndex.

private IncrementalIndex getSingleDimIndex(String dimName, List<String> values) throws Exception {
    IncrementalIndex toPersist1 = new OnheapIncrementalIndex.Builder().setSimpleTestingIndexSchema(new CountAggregatorFactory("count")).setMaxRowCount(1000).build();
    addDimValuesToIndex(toPersist1, dimName, values);
    return toPersist1;
}