Examples with TaskConfig io.druid.indexing.common.config.TaskConfig used on opensource projects

Example 1 with TaskConfig

use of io.druid.indexing.common.config.TaskConfig in project druid by druid-io.

the class HadoopTaskTest method testBuildClassLoader.

@Test
public void testBuildClassLoader() throws Exception {
    final HadoopTask task = new HadoopTask("taskId", "dataSource", ImmutableList.<String>of(), ImmutableMap.<String, Object>of()) {

        @Override
        public String getType() {
            return null;
        }

        @Override
        public boolean isReady(TaskActionClient taskActionClient) throws Exception {
            return false;
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            return null;
        }
    };
    final TaskToolbox toolbox = EasyMock.createStrictMock(TaskToolbox.class);
    EasyMock.expect(toolbox.getConfig()).andReturn(new TaskConfig(temporaryFolder.newFolder().toString(), null, null, null, ImmutableList.of("something:hadoop:1"), false, null, null)).once();
    EasyMock.replay(toolbox);
    final ClassLoader classLoader = task.buildClassLoader(toolbox);
    assertClassLoaderIsSingular(classLoader);
    final Class<?> hadoopClazz = Class.forName("org.apache.hadoop.fs.FSDataInputStream", false, classLoader);
    assertClassLoaderIsSingular(hadoopClazz.getClassLoader());
    final Class<?> druidHadoopConfigClazz = Class.forName("io.druid.indexer.HadoopDruidIndexerConfig", false, classLoader);
    assertClassLoaderIsSingular(druidHadoopConfigClazz.getClassLoader());
}

Example 2 with TaskConfig

use of io.druid.indexing.common.config.TaskConfig in project druid by druid-io.

the class IngestSegmentFirehoseFactoryTimelineTest method constructorFeeder.

@Parameterized.Parameters(name = "{0}")
public static Collection<Object[]> constructorFeeder() {
    final List<TestCase> testCases = ImmutableList.of(TC("2000/2000T02", 3, 7, DS("2000/2000T01", "v1", 0, IR("2000", 1), IR("2000T00:01", 2)), DS("2000T01/2000T02", "v1", 0, IR("2000T01", 4))), /* Adjacent segments */
    TC("2000/2000T02", 3, 7, DS("2000/2000T02", "v1", 0, IR("2000", 1), IR("2000T00:01", 2), IR("2000T01", 8)), DS("2000T01/2000T02", "v2", 0, IR("2000T01:01", 4))), /* 1H segment overlaid on top of 2H segment */
    TC("2000/2000-01-02", 4, 23, DS("2000/2000-01-02", "v1", 0, IR("2000", 1), IR("2000T00:01", 2), IR("2000T01", 8), IR("2000T02", 16)), DS("2000T01/2000T02", "v2", 0, IR("2000T01:01", 4))), /* 1H segment overlaid on top of 1D segment */
    TC("2000/2000T02", 4, 15, DS("2000/2000T02", "v1", 0, IR("2000", 1), IR("2000T00:01", 2), IR("2000T01", 8)), DS("2000/2000T02", "v1", 1, IR("2000T01:01", 4))), /* Segment set with two segments for the same interval */
    TC("2000T01/2000T02", 1, 2, DS("2000/2000T03", "v1", 0, IR("2000", 1), IR("2000T01", 2), IR("2000T02", 4))), /* Segment wider than desired interval */
    TC("2000T02/2000T04", 2, 12, DS("2000/2000T03", "v1", 0, IR("2000", 1), IR("2000T01", 2), IR("2000T02", 4)), DS("2000T03/2000T04", "v1", 0, IR("2000T03", 8))));
    final List<Object[]> constructors = Lists.newArrayList();
    for (final TestCase testCase : testCases) {
        final TaskActionClient taskActionClient = new TaskActionClient() {

            @Override
            public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
                if (taskAction instanceof SegmentListUsedAction) {
                    // Expect the interval we asked for
                    final SegmentListUsedAction action = (SegmentListUsedAction) taskAction;
                    if (action.getIntervals().equals(ImmutableList.of(testCase.interval))) {
                        return (RetType) ImmutableList.copyOf(testCase.segments);
                    } else {
                        throw new IllegalArgumentException("WTF");
                    }
                } else {
                    throw new UnsupportedOperationException();
                }
            }
        };
        SegmentHandoffNotifierFactory notifierFactory = EasyMock.createNiceMock(SegmentHandoffNotifierFactory.class);
        EasyMock.replay(notifierFactory);
        final TaskToolboxFactory taskToolboxFactory = new TaskToolboxFactory(new TaskConfig(testCase.tmpDir.getAbsolutePath(), null, null, 50000, null, false, null, null), new TaskActionClientFactory() {

            @Override
            public TaskActionClient create(Task task) {
                return taskActionClient;
            }
        }, new NoopServiceEmitter(), // segment pusher
        null, // segment killer
        null, // segment mover
        null, // segment archiver
        null, // segment announcer,
        null, notifierFactory, // query runner factory conglomerate corporation unionized collective
        null, // query executor service
        null, // monitor scheduler
        null, new SegmentLoaderFactory(new SegmentLoaderLocalCacheManager(null, new SegmentLoaderConfig() {

            @Override
            public List<StorageLocationConfig> getLocations() {
                return Lists.newArrayList();
            }
        }, MAPPER)), MAPPER, INDEX_MERGER, INDEX_IO, null, null, INDEX_MERGER_V9);
        final Injector injector = Guice.createInjector(new Module() {

            @Override
            public void configure(Binder binder) {
                binder.bind(TaskToolboxFactory.class).toInstance(taskToolboxFactory);
            }
        });
        final IngestSegmentFirehoseFactory factory = new IngestSegmentFirehoseFactory(DATA_SOURCE, testCase.interval, new NoopDimFilter(), Arrays.asList(DIMENSIONS), Arrays.asList(METRICS), injector, INDEX_IO);
        constructors.add(new Object[] { testCase.toString(), factory, testCase.tmpDir, testCase.expectedCount, testCase.expectedSum });
    }
    return constructors;
}

Example 3 with TaskConfig

use of io.druid.indexing.common.config.TaskConfig in project druid by druid-io.

the class TaskLifecycleTest method setUpTaskToolboxFactory.

private TaskToolboxFactory setUpTaskToolboxFactory(DataSegmentPusher dataSegmentPusher, SegmentHandoffNotifierFactory handoffNotifierFactory, TestIndexerMetadataStorageCoordinator mdc) throws IOException {
    Preconditions.checkNotNull(queryRunnerFactoryConglomerate);
    Preconditions.checkNotNull(monitorScheduler);
    Preconditions.checkNotNull(taskStorage);
    Preconditions.checkNotNull(emitter);
    taskLockbox = new TaskLockbox(taskStorage);
    tac = new LocalTaskActionClientFactory(taskStorage, new TaskActionToolbox(taskLockbox, mdc, emitter, EasyMock.createMock(SupervisorManager.class)));
    File tmpDir = temporaryFolder.newFolder();
    taskConfig = new TaskConfig(tmpDir.toString(), null, null, 50000, null, false, null, null);
    return new TaskToolboxFactory(taskConfig, tac, emitter, dataSegmentPusher, new LocalDataSegmentKiller(new LocalDataSegmentPusherConfig()), new DataSegmentMover() {

        @Override
        public DataSegment move(DataSegment dataSegment, Map<String, Object> targetLoadSpec) throws SegmentLoadingException {
            return dataSegment;
        }
    }, new DataSegmentArchiver() {

        @Override
        public DataSegment archive(DataSegment segment) throws SegmentLoadingException {
            return segment;
        }

        @Override
        public DataSegment restore(DataSegment segment) throws SegmentLoadingException {
            return segment;
        }
    }, new DataSegmentAnnouncer() {

        @Override
        public void announceSegment(DataSegment segment) throws IOException {
            announcedSinks++;
        }

        @Override
        public void unannounceSegment(DataSegment segment) throws IOException {
        }

        @Override
        public void announceSegments(Iterable<DataSegment> segments) throws IOException {
        }

        @Override
        public void unannounceSegments(Iterable<DataSegment> segments) throws IOException {
        }

        @Override
        public boolean isAnnounced(DataSegment segment) {
            return false;
        }
    }, // segment announcer
    handoffNotifierFactory, // query runner factory conglomerate corporation unionized collective
    queryRunnerFactoryConglomerate, // query executor service
    MoreExecutors.sameThreadExecutor(), // monitor scheduler
    monitorScheduler, new SegmentLoaderFactory(new SegmentLoaderLocalCacheManager(null, new SegmentLoaderConfig() {

        @Override
        public List<StorageLocationConfig> getLocations() {
            return Lists.newArrayList();
        }
    }, new DefaultObjectMapper())), MAPPER, INDEX_MERGER, INDEX_IO, MapCache.create(0), FireDepartmentTest.NO_CACHE_CONFIG, INDEX_MERGER_V9);
}

Example 4 with TaskConfig

use of io.druid.indexing.common.config.TaskConfig in project druid by druid-io.

the class WorkerTaskMonitorTest method createTaskMonitor.

private WorkerTaskMonitor createTaskMonitor() {
    final TaskConfig taskConfig = new TaskConfig(Files.createTempDir().toString(), null, null, 0, null, false, null, null);
    TaskActionClientFactory taskActionClientFactory = EasyMock.createNiceMock(TaskActionClientFactory.class);
    TaskActionClient taskActionClient = EasyMock.createNiceMock(TaskActionClient.class);
    EasyMock.expect(taskActionClientFactory.create(EasyMock.<Task>anyObject())).andReturn(taskActionClient).anyTimes();
    SegmentHandoffNotifierFactory notifierFactory = EasyMock.createNiceMock(SegmentHandoffNotifierFactory.class);
    EasyMock.replay(taskActionClientFactory, taskActionClient, notifierFactory);
    return new WorkerTaskMonitor(jsonMapper, cf, workerCuratorCoordinator, new ThreadPoolTaskRunner(new TaskToolboxFactory(taskConfig, taskActionClientFactory, null, null, null, null, null, null, notifierFactory, null, null, null, new SegmentLoaderFactory(new SegmentLoaderLocalCacheManager(null, new SegmentLoaderConfig() {

        @Override
        public List<StorageLocationConfig> getLocations() {
            return Lists.newArrayList();
        }
    }, jsonMapper)), jsonMapper, indexMerger, indexIO, null, null, indexMergerV9), taskConfig, new NoopServiceEmitter(), DUMMY_NODE));
}

Example 5 with TaskConfig

use of io.druid.indexing.common.config.TaskConfig in project druid by druid-io.

the class ResetCluster method deleteIndexerHadoopWorkingDir.

private void deleteIndexerHadoopWorkingDir(Injector injector) {
    try {
        log.info("===========================================================================");
        log.info("Deleting hadoopWorkingPath.");
        log.info("===========================================================================");
        TaskConfig taskConfig = injector.getInstance(TaskConfig.class);
        HadoopTask.invokeForeignLoader("io.druid.indexer.HadoopWorkingDirCleaner", new String[] { taskConfig.getHadoopWorkingPath() }, HadoopTask.buildClassLoader(null, taskConfig.getDefaultHadoopCoordinates()));
    } catch (Exception ex) {
        log.error(ex, "Failed to cleanup indexer hadoop working directory.");
    }
}