Search in sources :

Example 6 with SegmentLoaderConfig

use of io.druid.segment.loading.SegmentLoaderConfig in project druid by druid-io.

the class RealtimeIndexTaskTest method makeToolbox.

private TaskToolbox makeToolbox(final Task task, final TaskStorage taskStorage, final IndexerMetadataStorageCoordinator mdc, final File directory) {
    final TaskConfig taskConfig = new TaskConfig(directory.getPath(), null, null, 50000, null, false, null, null);
    final TaskLockbox taskLockbox = new TaskLockbox(taskStorage);
    try {
        taskStorage.insert(task, TaskStatus.running(task.getId()));
    } catch (EntryExistsException e) {
    // suppress
    }
    taskLockbox.syncFromStorage();
    final TaskActionToolbox taskActionToolbox = new TaskActionToolbox(taskLockbox, mdc, emitter, EasyMock.createMock(SupervisorManager.class));
    final TaskActionClientFactory taskActionClientFactory = new LocalTaskActionClientFactory(taskStorage, taskActionToolbox);
    final QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate(ImmutableMap.<Class<? extends Query>, QueryRunnerFactory>of(TimeseriesQuery.class, new TimeseriesQueryRunnerFactory(new TimeseriesQueryQueryToolChest(new IntervalChunkingQueryRunnerDecorator(null, null, null) {

        @Override
        public <T> QueryRunner<T> decorate(QueryRunner<T> delegate, QueryToolChest<T, ? extends Query<T>> toolChest) {
            return delegate;
        }
    }), new TimeseriesQueryEngine(), new QueryWatcher() {

        @Override
        public void registerQuery(Query query, ListenableFuture future) {
        // do nothing
        }
    })));
    handOffCallbacks = Maps.newConcurrentMap();
    final SegmentHandoffNotifierFactory handoffNotifierFactory = new SegmentHandoffNotifierFactory() {

        @Override
        public SegmentHandoffNotifier createSegmentHandoffNotifier(String dataSource) {
            return new SegmentHandoffNotifier() {

                @Override
                public boolean registerSegmentHandoffCallback(SegmentDescriptor descriptor, Executor exec, Runnable handOffRunnable) {
                    handOffCallbacks.put(descriptor, new Pair<>(exec, handOffRunnable));
                    return true;
                }

                @Override
                public void start() {
                //Noop
                }

                @Override
                public void close() {
                //Noop
                }

                Map<SegmentDescriptor, Pair<Executor, Runnable>> getHandOffCallbacks() {
                    return handOffCallbacks;
                }
            };
        }
    };
    final TestUtils testUtils = new TestUtils();
    final TaskToolboxFactory toolboxFactory = new TaskToolboxFactory(taskConfig, taskActionClientFactory, emitter, new TestDataSegmentPusher(), new TestDataSegmentKiller(), // DataSegmentMover
    null, // DataSegmentArchiver
    null, new TestDataSegmentAnnouncer(), handoffNotifierFactory, conglomerate, // queryExecutorService
    MoreExecutors.sameThreadExecutor(), EasyMock.createMock(MonitorScheduler.class), new SegmentLoaderFactory(new SegmentLoaderLocalCacheManager(null, new SegmentLoaderConfig() {

        @Override
        public List<StorageLocationConfig> getLocations() {
            return Lists.newArrayList();
        }
    }, testUtils.getTestObjectMapper())), testUtils.getTestObjectMapper(), testUtils.getTestIndexMerger(), testUtils.getTestIndexIO(), MapCache.create(1024), new CacheConfig(), testUtils.getTestIndexMergerV9());
    return toolboxFactory.build(task);
}
Also used : TimeseriesQuery(io.druid.query.timeseries.TimeseriesQuery) Query(io.druid.query.Query) QueryWatcher(io.druid.query.QueryWatcher) LocalTaskActionClientFactory(io.druid.indexing.common.actions.LocalTaskActionClientFactory) TaskActionClientFactory(io.druid.indexing.common.actions.TaskActionClientFactory) TestDataSegmentAnnouncer(io.druid.indexing.test.TestDataSegmentAnnouncer) TaskConfig(io.druid.indexing.common.config.TaskConfig) TimeseriesQueryQueryToolChest(io.druid.query.timeseries.TimeseriesQueryQueryToolChest) QueryToolChest(io.druid.query.QueryToolChest) TimeseriesQueryQueryToolChest(io.druid.query.timeseries.TimeseriesQueryQueryToolChest) TestUtils(io.druid.indexing.common.TestUtils) DefaultQueryRunnerFactoryConglomerate(io.druid.query.DefaultQueryRunnerFactoryConglomerate) QueryRunnerFactoryConglomerate(io.druid.query.QueryRunnerFactoryConglomerate) TimeseriesQueryEngine(io.druid.query.timeseries.TimeseriesQueryEngine) Executor(java.util.concurrent.Executor) TaskToolboxFactory(io.druid.indexing.common.TaskToolboxFactory) SegmentDescriptor(io.druid.query.SegmentDescriptor) TaskActionToolbox(io.druid.indexing.common.actions.TaskActionToolbox) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) LocalTaskActionClientFactory(io.druid.indexing.common.actions.LocalTaskActionClientFactory) IntervalChunkingQueryRunnerDecorator(io.druid.query.IntervalChunkingQueryRunnerDecorator) SegmentLoaderConfig(io.druid.segment.loading.SegmentLoaderConfig) SegmentLoaderFactory(io.druid.indexing.common.SegmentLoaderFactory) SegmentLoaderLocalCacheManager(io.druid.segment.loading.SegmentLoaderLocalCacheManager) CacheConfig(io.druid.client.cache.CacheConfig) TestDataSegmentPusher(io.druid.indexing.test.TestDataSegmentPusher) Pair(io.druid.java.util.common.Pair) TimeseriesQuery(io.druid.query.timeseries.TimeseriesQuery) MonitorScheduler(com.metamx.metrics.MonitorScheduler) SegmentHandoffNotifier(io.druid.segment.realtime.plumber.SegmentHandoffNotifier) DefaultQueryRunnerFactoryConglomerate(io.druid.query.DefaultQueryRunnerFactoryConglomerate) EntryExistsException(io.druid.metadata.EntryExistsException) QueryRunner(io.druid.query.QueryRunner) TestDataSegmentKiller(io.druid.indexing.test.TestDataSegmentKiller) SegmentHandoffNotifierFactory(io.druid.segment.realtime.plumber.SegmentHandoffNotifierFactory) SupervisorManager(io.druid.indexing.overlord.supervisor.SupervisorManager) TimeseriesQueryRunnerFactory(io.druid.query.timeseries.TimeseriesQueryRunnerFactory) TaskLockbox(io.druid.indexing.overlord.TaskLockbox) ListenableFuture(com.google.common.util.concurrent.ListenableFuture)

Example 7 with SegmentLoaderConfig

use of io.druid.segment.loading.SegmentLoaderConfig in project druid by druid-io.

the class TaskToolboxTest method testFetchSegments.

@Test
public void testFetchSegments() throws SegmentLoadingException, IOException {
    File expectedFile = temporaryFolder.newFile();
    EasyMock.expect(mockSegmentLoaderLocalCacheManager.getSegmentFiles((DataSegment) EasyMock.anyObject())).andReturn(expectedFile).anyTimes();
    EasyMock.expect(mockSegmentLoaderLocalCacheManager.withConfig((SegmentLoaderConfig) EasyMock.anyObject())).andReturn(mockSegmentLoaderLocalCacheManager).anyTimes();
    EasyMock.replay(mockSegmentLoaderLocalCacheManager);
    DataSegment dataSegment = DataSegment.builder().dataSource("source").interval(new Interval("2012-01-01/P1D")).version("1").size(1).build();
    List<DataSegment> segments = ImmutableList.of(dataSegment);
    Map actualFetchedSegment = taskToolbox.build(task).fetchSegments(segments);
    Assert.assertEquals(expectedFile, actualFetchedSegment.get(dataSegment));
}
Also used : SegmentLoaderConfig(io.druid.segment.loading.SegmentLoaderConfig) File(java.io.File) DataSegment(io.druid.timeline.DataSegment) Map(java.util.Map) Interval(org.joda.time.Interval) Test(org.junit.Test)

Example 8 with SegmentLoaderConfig

use of io.druid.segment.loading.SegmentLoaderConfig in project druid by druid-io.

the class ZkCoordinatorTest method setUp.

@Before
public void setUp() throws Exception {
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    try {
        infoDir = new File(File.createTempFile("blah", "blah2").getParent(), "ZkCoordinatorTest");
        infoDir.mkdirs();
        for (File file : infoDir.listFiles()) {
            file.delete();
        }
        log.info("Creating tmp test files in [%s]", infoDir);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    scheduledRunnable = Lists.newArrayList();
    segmentLoader = new CacheTestSegmentLoader();
    serverManager = new ServerManager(segmentLoader, new NoopQueryRunnerFactoryConglomerate(), new NoopServiceEmitter(), MoreExecutors.sameThreadExecutor(), MoreExecutors.sameThreadExecutor(), new DefaultObjectMapper(), new LocalCacheProvider().get(), new CacheConfig());
    final ZkPathsConfig zkPaths = new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "/druid";
        }
    };
    segmentsAnnouncedByMe = new ConcurrentSkipListSet<>();
    announceCount = new AtomicInteger(0);
    announcer = new DataSegmentAnnouncer() {

        private final DataSegmentAnnouncer delegate = new BatchDataSegmentAnnouncer(me, new BatchDataSegmentAnnouncerConfig(), zkPaths, new Announcer(curator, Execs.singleThreaded("blah")), jsonMapper);

        @Override
        public void announceSegment(DataSegment segment) throws IOException {
            segmentsAnnouncedByMe.add(segment);
            announceCount.incrementAndGet();
            delegate.announceSegment(segment);
        }

        @Override
        public void unannounceSegment(DataSegment segment) throws IOException {
            segmentsAnnouncedByMe.remove(segment);
            announceCount.decrementAndGet();
            delegate.unannounceSegment(segment);
        }

        @Override
        public void announceSegments(Iterable<DataSegment> segments) throws IOException {
            for (DataSegment segment : segments) {
                segmentsAnnouncedByMe.add(segment);
            }
            announceCount.addAndGet(Iterables.size(segments));
            delegate.announceSegments(segments);
        }

        @Override
        public void unannounceSegments(Iterable<DataSegment> segments) throws IOException {
            for (DataSegment segment : segments) {
                segmentsAnnouncedByMe.remove(segment);
            }
            announceCount.addAndGet(-Iterables.size(segments));
            delegate.unannounceSegments(segments);
        }

        @Override
        public boolean isAnnounced(DataSegment segment) {
            return segmentsAnnouncedByMe.contains(segment);
        }
    };
    zkCoordinator = new ZkCoordinator(jsonMapper, new SegmentLoaderConfig() {

        @Override
        public File getInfoDir() {
            return infoDir;
        }

        @Override
        public int getNumLoadingThreads() {
            return 5;
        }

        @Override
        public int getAnnounceIntervalMillis() {
            return 50;
        }

        @Override
        public int getDropSegmentDelayMillis() {
            return 0;
        }
    }, zkPaths, me, announcer, curator, serverManager, new ScheduledExecutorFactory() {

        @Override
        public ScheduledExecutorService create(int corePoolSize, String nameFormat) {
            /*
               Override normal behavoir by adding the runnable to a list so that you can make sure
               all the shceduled runnables are executed by explicitly calling run() on each item in the list
             */
            return new ScheduledThreadPoolExecutor(corePoolSize, new ThreadFactoryBuilder().setDaemon(true).setNameFormat(nameFormat).build()) {

                @Override
                public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
                    scheduledRunnable.add(command);
                    return null;
                }
            };
        }
    });
}
Also used : ScheduledThreadPoolExecutor(java.util.concurrent.ScheduledThreadPoolExecutor) BatchDataSegmentAnnouncerConfig(io.druid.server.initialization.BatchDataSegmentAnnouncerConfig) DataSegment(io.druid.timeline.DataSegment) Announcer(io.druid.curator.announcement.Announcer) ZkPathsConfig(io.druid.server.initialization.ZkPathsConfig) ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) TimeUnit(java.util.concurrent.TimeUnit) SegmentLoaderConfig(io.druid.segment.loading.SegmentLoaderConfig) CacheConfig(io.druid.client.cache.CacheConfig) CacheTestSegmentLoader(io.druid.segment.loading.CacheTestSegmentLoader) NoopQueryRunnerFactoryConglomerate(io.druid.query.NoopQueryRunnerFactoryConglomerate) NoopServiceEmitter(io.druid.server.metrics.NoopServiceEmitter) IOException(java.io.IOException) ScheduledExecutorFactory(io.druid.java.util.common.concurrent.ScheduledExecutorFactory) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DefaultObjectMapper(io.druid.jackson.DefaultObjectMapper) LocalCacheProvider(io.druid.client.cache.LocalCacheProvider) File(java.io.File) Before(org.junit.Before)

Example 9 with SegmentLoaderConfig

use of io.druid.segment.loading.SegmentLoaderConfig in project druid by druid-io.

the class IngestSegmentFirehoseFactoryTest method constructorFeeder.

@Parameterized.Parameters(name = "{1}")
public static Collection<Object[]> constructorFeeder() throws IOException {
    final IndexSpec indexSpec = new IndexSpec();
    final HeapMemoryTaskStorage ts = new HeapMemoryTaskStorage(new TaskStorageConfig(null) {
    });
    final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder().withQueryGranularity(Granularities.NONE).withMinTimestamp(JodaUtils.MIN_INSTANT).withDimensionsSpec(ROW_PARSER).withMetrics(new AggregatorFactory[] { new LongSumAggregatorFactory(METRIC_LONG_NAME, DIM_LONG_NAME), new DoubleSumAggregatorFactory(METRIC_FLOAT_NAME, DIM_FLOAT_NAME) }).build();
    final OnheapIncrementalIndex index = new OnheapIncrementalIndex(schema, true, MAX_ROWS * MAX_SHARD_NUMBER);
    for (Integer i = 0; i < MAX_ROWS; ++i) {
        index.add(ROW_PARSER.parse(buildRow(i.longValue())));
    }
    if (!persistDir.mkdirs() && !persistDir.exists()) {
        throw new IOException(String.format("Could not create directory at [%s]", persistDir.getAbsolutePath()));
    }
    INDEX_MERGER.persist(index, persistDir, indexSpec);
    final TaskLockbox tl = new TaskLockbox(ts);
    final IndexerSQLMetadataStorageCoordinator mdc = new IndexerSQLMetadataStorageCoordinator(null, null, null) {

        private final Set<DataSegment> published = Sets.newHashSet();

        private final Set<DataSegment> nuked = Sets.newHashSet();

        @Override
        public List<DataSegment> getUsedSegmentsForInterval(String dataSource, Interval interval) throws IOException {
            return ImmutableList.copyOf(segmentSet);
        }

        @Override
        public List<DataSegment> getUsedSegmentsForIntervals(String dataSource, List<Interval> interval) throws IOException {
            return ImmutableList.copyOf(segmentSet);
        }

        @Override
        public List<DataSegment> getUnusedSegmentsForInterval(String dataSource, Interval interval) {
            return ImmutableList.of();
        }

        @Override
        public Set<DataSegment> announceHistoricalSegments(Set<DataSegment> segments) {
            Set<DataSegment> added = Sets.newHashSet();
            for (final DataSegment segment : segments) {
                if (published.add(segment)) {
                    added.add(segment);
                }
            }
            return ImmutableSet.copyOf(added);
        }

        @Override
        public void deleteSegments(Set<DataSegment> segments) {
            nuked.addAll(segments);
        }
    };
    final LocalTaskActionClientFactory tac = new LocalTaskActionClientFactory(ts, new TaskActionToolbox(tl, mdc, newMockEmitter(), EasyMock.createMock(SupervisorManager.class)));
    SegmentHandoffNotifierFactory notifierFactory = EasyMock.createNiceMock(SegmentHandoffNotifierFactory.class);
    EasyMock.replay(notifierFactory);
    final TaskToolboxFactory taskToolboxFactory = new TaskToolboxFactory(new TaskConfig(tmpDir.getAbsolutePath(), null, null, 50000, null, false, null, null), tac, newMockEmitter(), new DataSegmentPusher() {

        @Deprecated
        @Override
        public String getPathForHadoop(String dataSource) {
            return getPathForHadoop();
        }

        @Override
        public String getPathForHadoop() {
            throw new UnsupportedOperationException();
        }

        @Override
        public DataSegment push(File file, DataSegment segment) throws IOException {
            return segment;
        }
    }, new DataSegmentKiller() {

        @Override
        public void kill(DataSegment segments) throws SegmentLoadingException {
        }

        @Override
        public void killAll() throws IOException {
            throw new UnsupportedOperationException("not implemented");
        }
    }, new DataSegmentMover() {

        @Override
        public DataSegment move(DataSegment dataSegment, Map<String, Object> targetLoadSpec) throws SegmentLoadingException {
            return dataSegment;
        }
    }, new DataSegmentArchiver() {

        @Override
        public DataSegment archive(DataSegment segment) throws SegmentLoadingException {
            return segment;
        }

        @Override
        public DataSegment restore(DataSegment segment) throws SegmentLoadingException {
            return segment;
        }
    }, // segment announcer
    null, notifierFactory, // query runner factory conglomerate corporation unionized collective
    null, // query executor service
    null, // monitor scheduler
    null, new SegmentLoaderFactory(new SegmentLoaderLocalCacheManager(null, new SegmentLoaderConfig() {

        @Override
        public List<StorageLocationConfig> getLocations() {
            return Lists.newArrayList();
        }
    }, MAPPER)), MAPPER, INDEX_MERGER, INDEX_IO, null, null, INDEX_MERGER_V9);
    Collection<Object[]> values = new LinkedList<>();
    for (InputRowParser parser : Arrays.<InputRowParser>asList(ROW_PARSER, new MapInputRowParser(new JSONParseSpec(new TimestampSpec(TIME_COLUMN, "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.<String>of()), ImmutableList.of(DIM_FLOAT_NAME, DIM_LONG_NAME), ImmutableList.<SpatialDimensionSchema>of()), null, null)))) {
        for (List<String> dim_names : Arrays.<List<String>>asList(null, ImmutableList.of(DIM_NAME))) {
            for (List<String> metric_names : Arrays.<List<String>>asList(null, ImmutableList.of(METRIC_LONG_NAME, METRIC_FLOAT_NAME))) {
                values.add(new Object[] { new IngestSegmentFirehoseFactory(DATA_SOURCE_NAME, FOREVER, new SelectorDimFilter(DIM_NAME, DIM_VALUE, null), dim_names, metric_names, Guice.createInjector(new Module() {

                    @Override
                    public void configure(Binder binder) {
                        binder.bind(TaskToolboxFactory.class).toInstance(taskToolboxFactory);
                    }
                }), INDEX_IO), String.format("DimNames[%s]MetricNames[%s]ParserDimNames[%s]", dim_names == null ? "null" : "dims", metric_names == null ? "null" : "metrics", parser == ROW_PARSER ? "dims" : "null"), parser });
            }
        }
    }
    return values;
}
Also used : DataSegmentArchiver(io.druid.segment.loading.DataSegmentArchiver) OnheapIncrementalIndex(io.druid.segment.incremental.OnheapIncrementalIndex) DataSegmentMover(io.druid.segment.loading.DataSegmentMover) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) LinkedList(java.util.LinkedList) LocalTaskActionClientFactory(io.druid.indexing.common.actions.LocalTaskActionClientFactory) SegmentLoaderConfig(io.druid.segment.loading.SegmentLoaderConfig) SegmentLoaderFactory(io.druid.indexing.common.SegmentLoaderFactory) SegmentLoaderLocalCacheManager(io.druid.segment.loading.SegmentLoaderLocalCacheManager) JSONParseSpec(io.druid.data.input.impl.JSONParseSpec) IndexerSQLMetadataStorageCoordinator(io.druid.metadata.IndexerSQLMetadataStorageCoordinator) DoubleSumAggregatorFactory(io.druid.query.aggregation.DoubleSumAggregatorFactory) HeapMemoryTaskStorage(io.druid.indexing.overlord.HeapMemoryTaskStorage) LinkedList(java.util.LinkedList) MapInputRowParser(io.druid.data.input.impl.MapInputRowParser) InputRowParser(io.druid.data.input.impl.InputRowParser) Module(com.google.inject.Module) SimpleModule(com.fasterxml.jackson.databind.module.SimpleModule) File(java.io.File) Interval(org.joda.time.Interval) IndexSpec(io.druid.segment.IndexSpec) DataSegmentPusher(io.druid.segment.loading.DataSegmentPusher) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) MapInputRowParser(io.druid.data.input.impl.MapInputRowParser) LongSumAggregatorFactory(io.druid.query.aggregation.LongSumAggregatorFactory) DataSegmentKiller(io.druid.segment.loading.DataSegmentKiller) TaskConfig(io.druid.indexing.common.config.TaskConfig) DataSegment(io.druid.timeline.DataSegment) Binder(com.google.inject.Binder) TaskToolboxFactory(io.druid.indexing.common.TaskToolboxFactory) SelectorDimFilter(io.druid.query.filter.SelectorDimFilter) TaskActionToolbox(io.druid.indexing.common.actions.TaskActionToolbox) TimestampSpec(io.druid.data.input.impl.TimestampSpec) IncrementalIndexSchema(io.druid.segment.incremental.IncrementalIndexSchema) SegmentLoadingException(io.druid.segment.loading.SegmentLoadingException) TaskStorageConfig(io.druid.indexing.common.config.TaskStorageConfig) IOException(java.io.IOException) DoubleSumAggregatorFactory(io.druid.query.aggregation.DoubleSumAggregatorFactory) AggregatorFactory(io.druid.query.aggregation.AggregatorFactory) LongSumAggregatorFactory(io.druid.query.aggregation.LongSumAggregatorFactory) SegmentHandoffNotifierFactory(io.druid.segment.realtime.plumber.SegmentHandoffNotifierFactory) SpatialDimensionSchema(io.druid.data.input.impl.SpatialDimensionSchema) TaskLockbox(io.druid.indexing.overlord.TaskLockbox) DimensionsSpec(io.druid.data.input.impl.DimensionsSpec)

Aggregations

SegmentLoaderConfig (io.druid.segment.loading.SegmentLoaderConfig)9 SegmentLoaderFactory (io.druid.indexing.common.SegmentLoaderFactory)6 TaskToolboxFactory (io.druid.indexing.common.TaskToolboxFactory)6 TaskConfig (io.druid.indexing.common.config.TaskConfig)6 SegmentLoaderLocalCacheManager (io.druid.segment.loading.SegmentLoaderLocalCacheManager)6 SegmentHandoffNotifierFactory (io.druid.segment.realtime.plumber.SegmentHandoffNotifierFactory)5 DataSegment (io.druid.timeline.DataSegment)5 File (java.io.File)5 List (java.util.List)5 ImmutableList (com.google.common.collect.ImmutableList)4 LocalTaskActionClientFactory (io.druid.indexing.common.actions.LocalTaskActionClientFactory)4 TaskActionClientFactory (io.druid.indexing.common.actions.TaskActionClientFactory)4 TaskActionToolbox (io.druid.indexing.common.actions.TaskActionToolbox)4 Binder (com.google.inject.Binder)3 Module (com.google.inject.Module)3 CacheConfig (io.druid.client.cache.CacheConfig)3 TaskLockbox (io.druid.indexing.overlord.TaskLockbox)3 SupervisorManager (io.druid.indexing.overlord.supervisor.SupervisorManager)3 Injector (com.google.inject.Injector)2 TestUtils (io.druid.indexing.common.TestUtils)2