Search in sources :

Example 6 with IndexIO

use of org.apache.druid.segment.IndexIO in project druid by druid-io.

the class BroadcastSegmentIndexedTableTest method setup.

@Before
public void setup() throws IOException, SegmentLoadingException {
    final ObjectMapper mapper = new DefaultObjectMapper();
    mapper.registerModule(new SegmentizerModule());
    final IndexIO indexIO = new IndexIO(mapper, () -> 0);
    mapper.setInjectableValues(new InjectableValues.Std().addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE).addValue(ObjectMapper.class.getName(), mapper).addValue(IndexIO.class, indexIO).addValue(DataSegment.PruneSpecsHolder.class, DataSegment.PruneSpecsHolder.DEFAULT));
    final IndexMerger indexMerger = new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
    Interval testInterval = Intervals.of("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z");
    IncrementalIndex data = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv");
    File segment = new File(temporaryFolder.newFolder(), "segment");
    File persisted = indexMerger.persist(data, testInterval, segment, new IndexSpec(), null);
    File factoryJson = new File(persisted, "factory.json");
    Assert.assertTrue(factoryJson.exists());
    SegmentizerFactory factory = mapper.readValue(factoryJson, SegmentizerFactory.class);
    Assert.assertTrue(factory instanceof MMappedQueryableSegmentizerFactory);
    DataSegment dataSegment = new DataSegment(DATASOURCE, testInterval, DateTimes.nowUtc().toString(), ImmutableMap.of(), columnNames, ImmutableList.of(), null, null, segment.getTotalSpace());
    backingSegment = (QueryableIndexSegment) factory.factorize(dataSegment, segment, false, SegmentLazyLoadFailCallback.NOOP);
    columnNames = ImmutableList.<String>builder().add(ColumnHolder.TIME_COLUMN_NAME).addAll(backingSegment.asQueryableIndex().getColumnNames()).build();
    broadcastTable = new BroadcastSegmentIndexedTable(backingSegment, keyColumns, dataSegment.getVersion());
}
Also used : IndexMerger(org.apache.druid.segment.IndexMerger) IndexSpec(org.apache.druid.segment.IndexSpec) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) IndexMergerV9(org.apache.druid.segment.IndexMergerV9) MMappedQueryableSegmentizerFactory(org.apache.druid.segment.loading.MMappedQueryableSegmentizerFactory) MMappedQueryableSegmentizerFactory(org.apache.druid.segment.loading.MMappedQueryableSegmentizerFactory) SegmentizerFactory(org.apache.druid.segment.loading.SegmentizerFactory) InjectableValues(com.fasterxml.jackson.databind.InjectableValues) DataSegment(org.apache.druid.timeline.DataSegment) TestExprMacroTable(org.apache.druid.query.expression.TestExprMacroTable) ExprMacroTable(org.apache.druid.math.expr.ExprMacroTable) IndexIO(org.apache.druid.segment.IndexIO) SegmentizerModule(org.apache.druid.jackson.SegmentizerModule) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) File(java.io.File) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) Interval(org.joda.time.Interval) Before(org.junit.Before)

Example 7 with IndexIO

use of org.apache.druid.segment.IndexIO in project druid by druid-io.

the class BroadcastJoinableMMappedQueryableSegmentizerFactoryTest method testSegmentizer.

@Test
public void testSegmentizer() throws IOException, SegmentLoadingException {
    final ObjectMapper mapper = new DefaultObjectMapper();
    mapper.registerModule(new SegmentizerModule());
    final IndexIO indexIO = new IndexIO(mapper, () -> 0);
    mapper.setInjectableValues(new InjectableValues.Std().addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE).addValue(ObjectMapper.class.getName(), mapper).addValue(IndexIO.class, indexIO).addValue(DataSegment.PruneSpecsHolder.class, DataSegment.PruneSpecsHolder.DEFAULT));
    IndexMerger indexMerger = new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
    SegmentizerFactory expectedFactory = new BroadcastJoinableMMappedQueryableSegmentizerFactory(indexIO, KEY_COLUMNS);
    Interval testInterval = Intervals.of("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z");
    IncrementalIndex data = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv");
    List<String> columnNames = data.getColumnNames();
    File segment = new File(temporaryFolder.newFolder(), "segment");
    File persistedSegmentRoot = indexMerger.persist(data, testInterval, segment, new IndexSpec(null, null, null, null, expectedFactory), null);
    File factoryJson = new File(persistedSegmentRoot, "factory.json");
    Assert.assertTrue(factoryJson.exists());
    SegmentizerFactory factory = mapper.readValue(factoryJson, SegmentizerFactory.class);
    Assert.assertTrue(factory instanceof BroadcastJoinableMMappedQueryableSegmentizerFactory);
    Assert.assertEquals(expectedFactory, factory);
    // load a segment
    final DataSegment dataSegment = new DataSegment(TABLE_NAME, testInterval, DateTimes.nowUtc().toString(), ImmutableMap.of(), columnNames, ImmutableList.of(), null, null, persistedSegmentRoot.getTotalSpace());
    final Segment loaded = factory.factorize(dataSegment, persistedSegmentRoot, false, SegmentLazyLoadFailCallback.NOOP);
    final BroadcastSegmentIndexedTable table = (BroadcastSegmentIndexedTable) loaded.as(IndexedTable.class);
    Assert.assertNotNull(table);
}
Also used : IndexMerger(org.apache.druid.segment.IndexMerger) IndexSpec(org.apache.druid.segment.IndexSpec) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) IndexMergerV9(org.apache.druid.segment.IndexMergerV9) BroadcastSegmentIndexedTable(org.apache.druid.segment.join.table.BroadcastSegmentIndexedTable) IndexedTable(org.apache.druid.segment.join.table.IndexedTable) InjectableValues(com.fasterxml.jackson.databind.InjectableValues) DataSegment(org.apache.druid.timeline.DataSegment) TestExprMacroTable(org.apache.druid.query.expression.TestExprMacroTable) ExprMacroTable(org.apache.druid.math.expr.ExprMacroTable) Segment(org.apache.druid.segment.Segment) DataSegment(org.apache.druid.timeline.DataSegment) IndexIO(org.apache.druid.segment.IndexIO) SegmentizerModule(org.apache.druid.jackson.SegmentizerModule) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) BroadcastSegmentIndexedTable(org.apache.druid.segment.join.table.BroadcastSegmentIndexedTable) File(java.io.File) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) Interval(org.joda.time.Interval) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 8 with IndexIO

use of org.apache.druid.segment.IndexIO in project druid by druid-io.

the class SegmentManagerBroadcastJoinIndexedTableTest method setup.

@Before
public void setup() throws IOException {
    segmentPuller = new LocalDataSegmentPuller();
    objectMapper = new DefaultObjectMapper().registerModule(new SegmentizerModule()).registerModule(new SimpleModule().registerSubtypes(new NamedType(LocalLoadSpec.class, "local")));
    indexIO = new IndexIO(objectMapper, () -> 0);
    objectMapper.setInjectableValues(new InjectableValues.Std().addValue(LocalDataSegmentPuller.class, segmentPuller).addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE).addValue(ObjectMapper.class.getName(), objectMapper).addValue(IndexIO.class, indexIO));
    segmentCacheDir = temporaryFolder.newFolder();
    segmentDeepStorageDir = temporaryFolder.newFolder();
    segmentCacheManager = new SegmentLocalCacheManager(new SegmentLoaderConfig() {

        @Override
        public List<StorageLocationConfig> getLocations() {
            return Collections.singletonList(new StorageLocationConfig(segmentCacheDir, null, null));
        }
    }, objectMapper);
    segmentManager = new SegmentManager(new SegmentLocalCacheLoader(segmentCacheManager, indexIO, objectMapper));
    joinableFactory = new BroadcastTableJoinableFactory(segmentManager);
    EmittingLogger.registerEmitter(new NoopServiceEmitter());
}
Also used : LocalLoadSpec(org.apache.druid.segment.loading.LocalLoadSpec) StorageLocationConfig(org.apache.druid.segment.loading.StorageLocationConfig) NamedType(com.fasterxml.jackson.databind.jsontype.NamedType) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) BroadcastTableJoinableFactory(org.apache.druid.segment.join.BroadcastTableJoinableFactory) InjectableValues(com.fasterxml.jackson.databind.InjectableValues) IndexIO(org.apache.druid.segment.IndexIO) LocalDataSegmentPuller(org.apache.druid.segment.loading.LocalDataSegmentPuller) SegmentLocalCacheManager(org.apache.druid.segment.loading.SegmentLocalCacheManager) SegmentizerModule(org.apache.druid.jackson.SegmentizerModule) SegmentLocalCacheLoader(org.apache.druid.segment.loading.SegmentLocalCacheLoader) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) SegmentLoaderConfig(org.apache.druid.segment.loading.SegmentLoaderConfig) SimpleModule(com.fasterxml.jackson.databind.module.SimpleModule) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) Before(org.junit.Before)

Example 9 with IndexIO

use of org.apache.druid.segment.IndexIO in project druid by druid-io.

the class SegmentManagerThreadSafetyTest method setup.

@Before
public void setup() throws IOException {
    segmentPuller = new TestSegmentPuller();
    objectMapper = new DefaultObjectMapper().registerModule(new SimpleModule().registerSubtypes(new NamedType(LocalLoadSpec.class, "local"), new NamedType(TestSegmentizerFactory.class, "test"))).setInjectableValues(new Std().addValue(LocalDataSegmentPuller.class, segmentPuller));
    indexIO = new IndexIO(objectMapper, () -> 0);
    segmentCacheDir = temporaryFolder.newFolder();
    segmentDeepStorageDir = temporaryFolder.newFolder();
    segmentCacheManager = new SegmentLocalCacheManager(new SegmentLoaderConfig() {

        @Override
        public List<StorageLocationConfig> getLocations() {
            return Collections.singletonList(new StorageLocationConfig(segmentCacheDir, null, null));
        }
    }, objectMapper);
    segmentManager = new SegmentManager(new SegmentLocalCacheLoader(segmentCacheManager, indexIO, objectMapper));
    exec = Execs.multiThreaded(NUM_THREAD, "SegmentManagerThreadSafetyTest-%d");
    EmittingLogger.registerEmitter(new NoopServiceEmitter());
}
Also used : Std(com.fasterxml.jackson.databind.InjectableValues.Std) StorageLocationConfig(org.apache.druid.segment.loading.StorageLocationConfig) IndexIO(org.apache.druid.segment.IndexIO) NamedType(com.fasterxml.jackson.databind.jsontype.NamedType) SegmentLocalCacheManager(org.apache.druid.segment.loading.SegmentLocalCacheManager) SegmentLocalCacheLoader(org.apache.druid.segment.loading.SegmentLocalCacheLoader) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) SegmentLoaderConfig(org.apache.druid.segment.loading.SegmentLoaderConfig) SimpleModule(com.fasterxml.jackson.databind.module.SimpleModule) Before(org.junit.Before)

Example 10 with IndexIO

use of org.apache.druid.segment.IndexIO in project druid by druid-io.

the class CompactionTaskTest method setup.

@Before
public void setup() {
    final IndexIO testIndexIO = new TestIndexIO(OBJECT_MAPPER, SEGMENT_MAP);
    toolbox = new TestTaskToolbox(new TestTaskActionClient(new ArrayList<>(SEGMENT_MAP.keySet())), testIndexIO, SEGMENT_MAP);
    segmentCacheManagerFactory = new SegmentCacheManagerFactory(OBJECT_MAPPER);
}
Also used : IndexIO(org.apache.druid.segment.IndexIO) SegmentCacheManagerFactory(org.apache.druid.indexing.common.SegmentCacheManagerFactory) Before(org.junit.Before)

Aggregations

IndexIO (org.apache.druid.segment.IndexIO)16 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)8 File (java.io.File)8 DefaultObjectMapper (org.apache.druid.jackson.DefaultObjectMapper)6 IndexMergerV9 (org.apache.druid.segment.IndexMergerV9)6 ColumnConfig (org.apache.druid.segment.column.ColumnConfig)5 QueryableIndex (org.apache.druid.segment.QueryableIndex)4 Before (org.junit.Before)4 InjectableValues (com.fasterxml.jackson.databind.InjectableValues)3 ArrayList (java.util.ArrayList)3 SegmentizerModule (org.apache.druid.jackson.SegmentizerModule)3 Closer (org.apache.druid.java.util.common.io.Closer)3 IndexMerger (org.apache.druid.segment.IndexMerger)3 IndexSpec (org.apache.druid.segment.IndexSpec)3 IncrementalIndex (org.apache.druid.segment.incremental.IncrementalIndex)3 NamedType (com.fasterxml.jackson.databind.jsontype.NamedType)2 SimpleModule (com.fasterxml.jackson.databind.module.SimpleModule)2 Injector (com.google.inject.Injector)2 IAE (org.apache.druid.java.util.common.IAE)2 ExprMacroTable (org.apache.druid.math.expr.ExprMacroTable)2